From: Fabian Groffen Date: Wed, 30 Jul 2008 13:55:41 +0000 (-0000) Subject: Merged from trunk 11248:11267 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=bfa04da65d2b392dfb75aa7eb1da234fdc2927a7;p=portage.git Merged from trunk 11248:11267 | 11249 | Bug #233165 - When waiting for jobs and merges to finish in | | zmedico | Scheduler._main_loop(), keep scheduling the merge queue | | | since it doesn't autoschedule, and skip the poll loop if | | | there no event handlers due to synchronous merge tasks being | | | the only things left to do. | | 11250 | Fix broken reference to "categories" in | | zmedico | CategorySet.singleBuilder(). Thanks to Thargor for | | | reporting. | | 11251 | Make use of the new config.iteritems() method. | | zmedico | | | 11252 | Document the doman language code path translation extension | | zmedico | from bug #222439. | | 11253 | Tweak the conditional USE deps examples. | | zmedico | | | 11254 | Fix alignment. | | zmedico | | | 11255 | Add syntax examples for unconditional USE deps. | | zmedico | | | 11256 | Redirect the FEATURES=buildsyspkg message to the log when in | | zmedico | background mode. | | 11257 | Fix _use_dep.__str__() to work correctly in the case when | | zmedico | conditional USE deps have evaluated to nothing (empty string | | | rather than []). Thanks to ABCD for reporting. | | 11258 | Implement _use_dep.__nonzero__(). | | zmedico | | | 11259 | Tweak table alignment. | | zmedico | | | 11260 | In apply_recursive_permissions(), ignore InvalidLocation | | zmedico | exceptions such as FileNotFound and DirectoryNotFound since | | | sometimes things disappear, like when adjusting permissions | | | on DISTCC_DIR. | | 11261 | Add support for the --jobs option to be specified without an | | zmedico | argument, and also support -j as a short option. Since | | | optparse doesn't natively support options with non-required | | | args, create an insert_optional_args() function that inserts | | | the required argument into the args so that optparse is | | | happy. The function inserts the string True as a substitute | | | for the argument that is required. This string is later | | | converted to the True constant when stored in the emerge | | | opts dict (similar to how normal boolean options are | | | stored). The PollScheduler and SequentialTaskQueue classes | | | recognize the meaning of the True constant to mean unlimited | | | concurrent jobs. | | 11262 | Fix slightly broken loop logic in insert_optional_args() by | | zmedico | converting it to pop args off of a stack. | | 11263 | During the first minute of entering the main scheduler loop, | | zmedico | if --load-average is enabled then limit the rate that new | | | jobs are spawned, so that the load average measurement has | | | time to respond to the new load introduced by the new jobs. | | | The time between spawning new jobs is proportional to the | | | number of currently running jobs. | | 11264 | Enable Scheduler._job_delay() whenever --load-average is | | zmedico | enabled, for whole time the scheduler is running. This | | | protects against too many jobs being sheduled if the load | | | average temporarily drops. | | 11265 | Fix Scheduler._choose_pkg() so that it doesn't choose | | zmedico | packages prematurely in some cases. | | 11266 | Make sure Scheduler._choose_pkg() doesn't return a package | | zmedico | too early when there's no digraph and the previous merge | | | hasn't completed yet. | | 11267 | Fix Scheduler._set_digraph() to correctly handle cases when | | zmedico | max_jobs is True. | svn path=/main/branches/prefix/; revision=11286 --- diff --git a/doc/package/ebuild/eapi/2.docbook b/doc/package/ebuild/eapi/2.docbook index 594eceabe..08386cc2f 100644 --- a/doc/package/ebuild/eapi/2.docbook +++ b/doc/package/ebuild/eapi/2.docbook @@ -1,7 +1,67 @@ EAPI 2_pre0 + + Helpers + + doman + + Language codes in file names are now used for path translation. + + Man Page Path Translation + + + + + + Source + Destination + + + + + foo.1 + /usr/share/man/man1/foo.1 + + + foo.lang.1 + /usr/share/man/lang/man1/foo.1 + + + +
+ + USE Dependencies + + Unconditional USE Dependencies + Syntax Examples + + + + + + Example + Meaning + + + + + foo[bar] + foo must have bar enabled + + + foo[bar,baz] + foo must have both bar and baz enabled + + + foo[-bar,baz] + foo must have bar disabled and baz enabled + + + +
+ Conditional USE Dependencies Syntax Examples @@ -17,19 +77,19 @@ foo[bar?] - foo bar? ( foo[bar] ) + bar? ( foo[bar] ) !bar? ( foo ) foo[-bar?] - foo !bar? ( foo[-bar] ) + bar? ( foo ) !bar? ( foo[-bar] ) foo[bar=] - foo bar? ( foo[bar] ) !bar? ( foo[-bar] ) + bar? ( foo[bar] ) !bar? ( foo[-bar] ) foo[bar!=] - foo bar? ( foo[-bar] ) !bar? ( foo[bar] ) + bar? ( foo[-bar] ) !bar? ( foo[bar] ) diff --git a/man/emerge.1 b/man/emerge.1 index 59d335f62..e252c19b1 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -323,9 +323,10 @@ directory. .BR "\-\-ignore-default-opts" Causes \fIEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) to be ignored. .TP -.BR \-\-jobs=JOBS -Specifies the number of packages to build simultaneously. Also see -the related \fB\-\-load\-average\fR option. +.BR "-j [JOBS], \-\-jobs[=JOBS]" +Specifies the number of packages to build simultaneously. If this option is +given without an argument, emerge will not limit the number of jobs that can +run simultaneously. Also see the related \fB\-\-load\-average\fR option. .TP .BR "\-\-keep\-going" Continue as much as possible after an error. When an error occurs, diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index b4756cf9e..d142b498b 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2348,7 +2348,7 @@ class EbuildBuild(CompositeTask): __slots__ = ("args_set", "background", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ - ("_build_dir", "_buildpkg", "_ebuild_path", "_tree") + ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): @@ -2460,16 +2460,13 @@ class EbuildBuild(CompositeTask): logger.log(msg, short_msg=short_msg) #buildsyspkg: Check if we need to _force_ binary package creation - issyspkg = "buildsyspkg" in features and \ + self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ not opts.buildpkg - if opts.buildpkg or issyspkg: + if opts.buildpkg or self._issyspkg: self._buildpkg = True - if issyspkg: - portage.writemsg_stdout(">>> This is a system package, " + \ - "let's pack a rescue tarball.\n", noiselevel=-1) msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) @@ -2506,6 +2503,21 @@ class EbuildBuild(CompositeTask): self.wait() return + if self._issyspkg: + msg = ">>> This is a system package, " + \ + "let's pack a rescue tarball.\n" + + log_path = self.settings.get("PORTAGE_LOG_FILE") + if log_path is not None: + log_file = open(log_path, 'a') + try: + log_file.write(msg) + finally: + log_file.close() + + if not self.background: + portage.writemsg_stdout(msg, noiselevel=-1) + packager = EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) @@ -3286,7 +3298,7 @@ class BinpkgFetcher(SpawnProcess): "FILE" : os.path.basename(pkg_path) } - fetch_env = dict((k, settings[k]) for k in settings) + fetch_env = dict(settings.iteritems()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ for x in shlex.split(fcmd)] @@ -8182,7 +8194,8 @@ class SequentialTaskQueue(SlotObject): if task.poll() is not None: state_changed = True - while task_queue and (len(running_tasks) < max_jobs): + while task_queue and \ + (max_jobs is True or len(running_tasks) < max_jobs): task = task_queue.popleft() cancelled = getattr(task, "cancelled", None) if not cancelled: @@ -8300,10 +8313,12 @@ class PollScheduler(object): max_jobs = self._max_jobs max_load = self._max_load - if self._running_job_count() >= self._max_jobs: + if self._max_jobs is not True and \ + self._running_job_count() >= self._max_jobs: return False - if max_load is not None and max_jobs > 1 and \ + if max_load is not None and \ + (max_jobs is True or max_jobs > 1) and \ self._running_job_count() > 1: try: avg1, avg5, avg15 = os.getloadavg() @@ -8886,6 +8901,12 @@ class Scheduler(PollScheduler): self._max_load = myopts.get("--load-average") + # The load average takes some time to respond when new + # jobs are added, so we need to limit the rate of adding + # new jobs. + self._job_delay_factor = 0.5 + self._previous_job_start_time = None + self._set_digraph(digraph) # This is used to memoize the _choose_pkg() result when @@ -8940,7 +8961,8 @@ class Scheduler(PollScheduler): @rtype: bool @returns: True if background mode is enabled, False otherwise. """ - background = (self._max_jobs > 1 or "--quiet" in self.myopts) and \ + background = (self._max_jobs is True or \ + self._max_jobs > 1 or "--quiet" in self.myopts) and \ not bool(self._opts_no_background.intersection(self.myopts)) self._status_display.quiet = \ @@ -8955,7 +8977,8 @@ class Scheduler(PollScheduler): return background def _set_digraph(self, digraph): - if self._max_jobs < 2: + if self._max_jobs is not True and \ + self._max_jobs < 2: # save some memory self._digraph = None return @@ -9587,7 +9610,10 @@ class Scheduler(PollScheduler): if self._choose_pkg_return_early: return None - if self._max_jobs < 2 or self._jobs == 0: + if self._digraph is None: + if self._jobs or self._task_queues.merge: + self._choose_pkg_return_early = True + return None return self._pkg_queue.pop(0) self._prune_digraph() @@ -9664,6 +9690,9 @@ class Scheduler(PollScheduler): self._poll_loop() while self._jobs or merge_queue: + if merge_queue.schedule() and \ + not self._poll_event_handlers: + continue self._poll_loop() def _schedule_tasks(self): @@ -9679,6 +9708,22 @@ class Scheduler(PollScheduler): self._status_display.display() return remaining + def _job_delay(self): + """ + @rtype: bool + @returns: True if job scheduling should be delayed, False otherwise. + """ + + if self._jobs and self._max_load is not None: + + current_time = time.time() + + if current_time - self._previous_job_start_time < \ + self._job_delay_factor * self._jobs: + return True + + return False + def _schedule_tasks_imp(self): """ @rtype: bool @@ -9693,7 +9738,8 @@ class Scheduler(PollScheduler): return (False, state_change) if self._choose_pkg_return_early or \ - not self._can_add_job(): + not self._can_add_job() or \ + self._job_delay(): return (True, state_change) pkg = self._choose_pkg() @@ -9714,12 +9760,14 @@ class Scheduler(PollScheduler): elif pkg.built: self._jobs += 1 + self._previous_job_start_time = time.time() self._status_display.running = self._jobs task.addExitListener(self._extract_exit) self._task_queues.jobs.add(task) else: self._jobs += 1 + self._previous_job_start_time = time.time() self._status_display.running = self._jobs task.addExitListener(self._build_exit) self._task_queues.jobs.add(task) @@ -12798,6 +12846,63 @@ def multiple_actions(action1, action2): sys.stderr.write("!!! '%s' or '%s'\n\n" % (action1, action2)) sys.exit(1) +def insert_optional_args(args): + """ + Parse optional arguments and insert a value if one has + not been provided. This is done before feeding the args + to the optparse parser since that parser does not support + this feature natively. + """ + + new_args = [] + jobs_opts = ("-j", "--jobs") + arg_stack = args[:] + arg_stack.reverse() + while arg_stack: + arg = arg_stack.pop() + + short_job_opt = bool("j" in arg and arg[:1] == "-" and arg[:2] != "--") + if not (short_job_opt or arg in jobs_opts): + new_args.append(arg) + continue + + # Insert an empty placeholder in order to + # satisfy the requirements of optparse. + + new_args.append("--jobs") + job_count = None + saved_opts = None + if short_job_opt and len(arg) > 2: + if arg[:2] == "-j": + try: + job_count = int(arg[2:]) + except ValueError: + saved_opts = arg[2:] + else: + job_count = "True" + saved_opts = arg[1:].replace("j", "") + + if job_count is None and arg_stack: + try: + job_count = int(arg_stack[-1]) + except ValueError: + pass + else: + # Discard the job count from the stack + # since we're consuming it here. + arg_stack.pop() + + if job_count is None: + # unlimited number of jobs + new_args.append("True") + else: + new_args.append(str(job_count)) + + if saved_opts is not None: + new_args.append("-" + saved_opts) + + return new_args + def parse_opts(tmpcmdline, silent=False): myaction=None myopts = {} @@ -12868,15 +12973,22 @@ def parse_opts(tmpcmdline, silent=False): parser.add_option(myopt, dest=myopt.lstrip("--").replace("-", "_"), **kwargs) + tmpcmdline = insert_optional_args(tmpcmdline) + myoptions, myargs = parser.parse_args(args=tmpcmdline) if myoptions.jobs: - try: - jobs = int(myoptions.jobs) - except ValueError: - jobs = 0 + jobs = None + if myoptions.jobs == "True": + jobs = True + else: + try: + jobs = int(myoptions.jobs) + except ValueError: + jobs = -1 - if jobs < 1: + if jobs is not True and \ + jobs < 1: jobs = None if not silent: writemsg("!!! Invalid --jobs parameter: '%s'\n" % \ diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index b2c032b21..bb4077aa5 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -14,7 +14,7 @@ def shorthelp(): print " "+turquoise("emerge")+" < "+turquoise("--sync")+" | "+turquoise("--metadata")+" | "+turquoise("--info")+" >" print " "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]" print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] " - print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"]" + print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhjkKlnNoOpqPsStuvV")+"]" print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--columns")+" ]" print " [ "+green("--complete-graph")+" ] [ "+green("--deep")+" ]" print " [ "+green("--jobs") + " " + turquoise("JOBS")+" ] [ "+green("--keep-going")+" ] [ " + green("--load-average")+" " + turquoise("LOAD") + " ]" @@ -305,9 +305,12 @@ def help(myaction,myopts,havecolor=1): print " downloaded from the remote server without consulting packages" print " existing in the packages directory." print - print " " + green("--jobs") + " " + turquoise("JOBS") + print " " + green("--jobs") + " " + turquoise("[JOBS]") + " ("+green("-j")+" short option)" desc = "Specifies the number of packages " + \ - "to build simultaneously. Also see " + \ + "to build simultaneously. If this option is " + \ + "given without an argument, emerge will not " + \ + "limit the number of jobs that " + \ + "can run simultaneously. Also see " + \ "the related --load-average option." for line in wrap(desc, desc_width): print desc_indent + line diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 49f6bc045..8528da4f8 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -395,7 +395,12 @@ class _use_dep(object): self.conditional = conditional break + def __nonzero__(self): + return bool(self.tokens) + def __str__(self): + if not self.tokens: + return "" return "[%s]" % (",".join(self.tokens),) def evaluate_conditionals(self, use): @@ -413,17 +418,17 @@ class _use_dep(object): x x= x -x x= -x - x x!= -x - -x x!= x + x x!= -x + -x x!= x Conditional syntax examples: compact form equivalent expanded form - foo[bar?] foo bar? ( foo[bar] ) - foo[-bar?] foo !bar? ( foo[-bar] ) - foo[bar=] foo bar? ( foo[bar] ) !bar? ( foo[-bar] ) - foo[bar!=] foo bar? ( foo[-bar] ) !bar? ( foo[bar] ) + foo[bar?] bar? ( foo[bar] ) !bar? ( foo ) + foo[-bar?] bar? ( foo ) !bar? ( foo[-bar] ) + foo[bar=] bar? ( foo[bar] ) !bar? ( foo[-bar] ) + foo[bar!=] bar? ( foo[-bar] ) !bar? ( foo[bar] ) """ tokens = [] diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py index bacc3a5c5..5352a32f3 100644 --- a/pym/portage/sets/dbapi.py +++ b/pym/portage/sets/dbapi.py @@ -70,7 +70,7 @@ class CategorySet(PackageSet): raise SetConfigError("no category given") category = options["category"] - if not category in categories: + if not category in settings.categories: raise SetConfigError("invalid category name '%s'" % category) repository = cls._builderGetRepository(options, trees.keys()) diff --git a/pym/portage/util.py b/pym/portage/util.py index 7aa333399..79fe7cc6f 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -801,8 +801,12 @@ def apply_recursive_permissions(top, uid=-1, gid=-1, if not applied: all_applied = False except PortageException, e: - all_applied = False - onerror(e) + # Ignore InvalidLocation exceptions such as FileNotFound + # and DirectoryNotFound since sometimes things disappear, + # like when adjusting permissions on DISTCC_DIR. + if not isinstance(e, portage.exception.InvalidLocation): + all_applied = False + onerror(e) return all_applied def apply_secpass_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1,