From: Fabian Groffen Date: Fri, 25 Jul 2008 07:34:22 +0000 (-0000) Subject: Merged from trunk 11169:11185 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=6c3d2880508c79c2a37eb5ee57682385bcee7a8a;p=portage.git Merged from trunk 11169:11185 | 11170 | Make --fetchonly disable background mode. | | zmedico | | | 11173 | Add GENTOO_MIRRORS to the variables exported by | | zmedico | EbuildFetcher. | | 11175 | Add sandbox write access to PORTAGE_PYM_PATH before | | zmedico | importing portage in order to avoid sandbox violations after | | | python upgrade. | | 11176 | Fix --depclean/--prune regressions reported by selkin: * | | zmedico | Make --with-bdeps default to "y" for removal actions. * | | | Treat unsatisfied build time dependencies as "soft" for | | | removal actions, even when --with-bdeps=y. | | 11177 | Avoid "0 files checked ..." messages reported by Pesa. | | zmedico | | | 11178 | Fix typo, found by pyflakes. | | zmedico | | | 11179 | Fix bug in get_repo_path() when called with multiple args. | | zmedico | | | 11181 | Bug #228117 - Insert a trailing newline in the getconfig() | | zmedico | input file as necessary to avoid a silent error in shlex | | | that is triggered by a source statement at the end of the | | | file without a trailing newline after the source statement. | | | To solve this problem, derive a special file class that | | | inserts a newline just before eof when necessary. | | 11182 | Bug #206773 - Add a new PORTAGE_IONICE_COMMAND variable that | | zmedico | emerge uses to adjust ionice priority, similar to | | | PORTAGE_NICENESS but used more like FETCHCOMMAND since so | | | that portage doesn't have to know anything about ionice | | | options. The command should include a \${PID} place-holder | | | to be substituted with an integer pid. | | 11183 | * Refer user to make.conf(5) if PORTAGE_IONICE_COMMAND | | zmedico | fails. * Document that PORTAGE_IONICE_COMMAND can be | | | disabled by setting it to an empty string. | | 11184 | When in background mode and there is a single failure, dump | | zmedico | the whole build log for convenience. | | 11185 | Cancel prefetchers if they're the only reason the main poll | | zmedico | loop is still running. | svn path=/main/branches/prefix/; revision=11186 --- diff --git a/bin/portageq b/bin/portageq index 2a0cf6f3f..fe52c81fc 100755 --- a/bin/portageq +++ b/bin/portageq @@ -383,8 +383,8 @@ def get_repo_path(argv): if len(argv) < 2: print "ERROR: insufficient parameters!" sys.exit(2) - for arg in arvg[1:]: - print portage.db[argv[0]]["porttree"].dbapi.getRepositoryPath(argv[1]) + for arg in argv[1:]: + print portage.db[argv[0]]["porttree"].dbapi.getRepositoryPath(arg) def list_preserved_libs(argv): """ @@ -476,17 +476,24 @@ def main(): sys.exit(os.EX_USAGE) os.environ["ROOT"] = sys.argv[2] + # Avoid sandbox violations after python upgrade. + from os import path as osp + # for an explanation on this logic, see pym/_emerge/__init__.py + if os.environ.__contains__("PORTAGE_PYTHONPATH"): + pym_path = os.environ["PORTAGE_PYTHONPATH"]) + else: + pym_path = osp.join(osp.dirname( + osp.dirname(osp.realpath(__file__))), "pym") + if os.environ.get("SANDBOX_ON") == "1": + sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":") + if pym_path not in sandbox_write: + sandbox_write.append(pym_path) + os.environ["SANDBOX_WRITE"] = \ + ":".join(filter(None, sandbox_write)) + global portage - try: - import portage - except ImportError: - from os import path as osp - # for an explanation on this logic, see pym/_emerge/__init__.py - if os.environ.__contains__("PORTAGE_PYTHONPATH"): - sys.path.insert(0, os.environ["PORTAGE_PYTHONPATH"]) - else: - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) - import portage + sys.path.insert(0, pym_path) + import portage try: if uses_root: diff --git a/cnf/make.globals b/cnf/make.globals index e5e949adc..06a28bbf4 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -62,6 +62,9 @@ PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5" # Minimum size of existing file for RESUMECOMMAND to be called. PORTAGE_FETCH_RESUME_MIN_SIZE="350K" +# Command called to adjust the io priority of portage and it's subprocesses. +PORTAGE_IONICE_COMMAND="ionice -c 2 -n 7 -p \${PID}" + # Number of times 'emerge --sync' will run before giving up. PORTAGE_RSYNC_RETRIES="3" diff --git a/man/make.conf.5 b/man/make.conf.5 index b3dc3c49d..875060da2 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -459,6 +459,16 @@ that small garbage files such as html 404 pages are properly discarded. The variable should contain an integer number of bytes and may have a suffix such as K, M, or G. .TP +\fBPORTAGE_IONICE_COMMAND\fR = \fI[ionice command string]\fR +This variable should contain a command for portage to call in order +to adjust the io priority of portage and it's subprocesses. The command +string should contain a \\${PID} place-holder that will be substituted +with an integer pid. For more information about ionice, see \fBionice\fR(1). +Set this variable to an empty string if you do not want portage to call +ionice. +.br +Defaults to "ionice -c 2 -n 7 -p \\${PID}". +.TP \fBPORTAGE_NICENESS\fR = \fI[number]\fR The value of this variable will be added to the current nice level that emerge is running at. In other words, this will not set the nice level, diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index fc9b9588d..e91306cf6 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2239,7 +2239,7 @@ class EbuildFetcher(SpawnProcess): __slots__ = ("fetchonly", "pkg",) - _env_vars = ("FETCHCOMMAND", "RESUMECOMMAND") + _env_vars = ("FETCHCOMMAND", "GENTOO_MIRRORS", "RESUMECOMMAND") def _start(self): @@ -4512,6 +4512,7 @@ class depgraph(object): myuse = pkg.use.enabled jbigkey = pkg depth = pkg.depth + 1 + removal_action = "remove" in self.myparams edepend={} depkeys = ["DEPEND","RDEPEND","PDEPEND"] @@ -4525,7 +4526,8 @@ class depgraph(object): edepend["RDEPEND"] = "" edepend["PDEPEND"] = "" bdeps_satisfied = False - if mytype in ("installed", "binary"): + + if pkg.built and not removal_action: if self.myopts.get("--with-bdeps", "n") == "y": # Pull in build time deps as requested, but marked them as # "satisfied" since they are not strictly required. This allows @@ -4539,6 +4541,9 @@ class depgraph(object): # built packages do not have build time dependencies. edepend["DEPEND"] = "" + if removal_action and self.myopts.get("--with-bdeps", "y") == "n": + edepend["DEPEND"] = "" + deps = ( ("/", edepend["DEPEND"], self._priority(buildtime=True, satisfied=bdeps_satisfied)), @@ -8685,6 +8690,10 @@ class Scheduler(PollScheduler): "--fetchonly", "--fetch-all-uri", "--nodeps", "--pretend"]) + _opts_no_background = \ + frozenset(["--pretend", + "--fetchonly", "--fetch-all-uri"]) + _opts_no_restart = frozenset(["--buildpkgonly", "--fetchonly", "--fetch-all-uri", "--pretend"]) @@ -8723,6 +8732,9 @@ class Scheduler(PollScheduler): kwargs.pop("short_msg", None) emergelog(self.xterm_titles, *pargs, **kwargs) + class _failed_pkg(SlotObject): + __slots__ = ("log_path", "pkg", "returncode") + def __init__(self, settings, trees, mtimedb, myopts, spinner, mergelist, favorites, digraph): PollScheduler.__init__(self) @@ -8860,7 +8872,7 @@ class Scheduler(PollScheduler): @returns: True if background mode is enabled, False otherwise. """ background = (self._max_jobs > 1 or "--quiet" in self.myopts) and \ - "--pretend" not in self.myopts + not bool(self._opts_no_background.intersection(self.myopts)) self._status_display.quiet = \ not background or \ @@ -9272,8 +9284,8 @@ class Scheduler(PollScheduler): if not failed_pkgs: break - for failed_pkg, returncode in failed_pkgs: - mergelist.remove(list(failed_pkg)) + for failed_pkg in failed_pkgs: + mergelist.remove(list(failed_pkg.pkg)) self._failed_pkgs_all.extend(failed_pkgs) del failed_pkgs[:] @@ -9293,7 +9305,7 @@ class Scheduler(PollScheduler): def _eerror(lines): for l in lines: - eerror(l, phase="other", key=failed_pkg.cpv) + eerror(l, phase="other", key=failed_pkg.pkg.cpv) msg = [] msg.append("One or more packages have been " + \ @@ -9312,8 +9324,32 @@ class Scheduler(PollScheduler): self._logger.log(" *** Finished. Cleaning up...") + if failed_pkgs: + self._failed_pkgs_all.extend(failed_pkgs) + del failed_pkgs[:] + background = self._background - if self._failed_pkgs_all and background and \ + failure_log_shown = False + if background and len(self._failed_pkgs_all) == 1: + # If only one package failed then just show it's + # whole log for easy viewing. + failed_pkg = self._failed_pkgs_all[-1] + log_path = failed_pkg.log_path + if log_path is not None: + try: + log_file = open(log_path, 'rb') + except IOError: + pass + else: + try: + for line in log_file: + writemsg_level(line, noiselevel=-1) + finally: + log_file.close() + failure_log_shown = True + + if background and not failure_log_shown and \ + self._failed_pkgs_all and \ self._failed_pkgs_die_msgs and \ not _flush_elog_mod_echo(): @@ -9345,9 +9381,9 @@ class Scheduler(PollScheduler): for line in wrap(msg, 72): writemsg("%s%s\n" % (prefix, line), noiselevel=-1) writemsg(prefix + "\n", noiselevel=-1) - for pkg, returncode in self._failed_pkgs_all: + for failed_pkg in self._failed_pkgs_all: writemsg("%s\t%s\n" % (prefix, - colorize("INFORM", str(pkg))), + colorize("INFORM", str(failed_pkg.pkg))), noiselevel=-1) writemsg(prefix + "\n", noiselevel=-1) @@ -9379,7 +9415,9 @@ class Scheduler(PollScheduler): def _do_merge_exit(self, merge): pkg = merge.merge.pkg if merge.returncode != os.EX_OK: - self._failed_pkgs.append((pkg, merge.returncode)) + log_path = merge.merge.settings.get("PORTAGE_LOG_FILE") + self._failed_pkgs.append(self._failed_pkg( + log_path=log_path, pkg=pkg, returncode=merge.returncode)) self._status_display.failed = len(self._failed_pkgs) return @@ -9414,7 +9452,9 @@ class Scheduler(PollScheduler): self._task_queues.merge.add(merge) self._status_display.merges = len(self._task_queues.merge) else: - self._failed_pkgs.append((build.pkg, build.returncode)) + log_path = build.settings.get("PORTAGE_LOG_FILE") + self._failed_pkgs.append(self._failed_pkg( + log_path=log_path, pkg=build.pkg, returncode=build.returncode)) self._status_display.failed = len(self._failed_pkgs) self._deallocate_config(build.settings) self._jobs -= 1 @@ -9443,7 +9483,7 @@ class Scheduler(PollScheduler): self._main_loop_cleanup() portage.elog._emerge_elog_listener = None if failed_pkgs: - pkg, rval = failed_pkgs[-1] + rval = failed_pkgs[-1].returncode return rval @@ -9531,7 +9571,7 @@ class Scheduler(PollScheduler): # Only allow 1 job max if a restart is scheduled # due to portage update. if self._is_restart_scheduled() or \ - "--pretend" in self.myopts: + self._opts_no_background.intersection(self.myopts): self._set_max_jobs(1) merge_queue = self._task_queues.merge @@ -9546,6 +9586,13 @@ class Scheduler(PollScheduler): remaining, state_change = self._schedule_tasks_imp() self._task_queues.merge.schedule() self._status_display.display() + + # Cancel prefetchers if they're the only reason + # the main poll loop is still running. + if self._failed_pkgs and \ + not (self._jobs or self._task_queues.merge): + self._task_queues.fetch.clear() + return remaining def _schedule_tasks_imp(self): @@ -11767,8 +11814,10 @@ def action_depclean(settings, trees, ldpath_mtimes, unresolvable = set() for dep in resolver._initially_unsatisfied_deps: - if isinstance(dep.parent, Package): + if isinstance(dep.parent, Package) and \ + (dep.priority > UnmergeDepPriority.SOFT): unresolvable.add((dep.atom, dep.parent.cpv)) + if not unresolvable: return False @@ -12925,6 +12974,30 @@ def adjust_config(myopts, settings): settings["NOCOLOR"] = "true" settings.backup_changes("NOCOLOR") +def ionice(settings): + + ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND") + if ionice_cmd: + ionice_cmd = shlex.split(ionice_cmd) + if not ionice_cmd: + return + + from portage.util import varexpand + variables = {"PID" : str(os.getpid())} + cmd = [varexpand(x, mydict=variables) for x in ionice_cmd] + + try: + rval = portage.process.spawn(cmd, env=os.environ) + except portage.exception.CommandNotFound: + # The OS kernel probably doesn't support ionice, + # so return silently. + return + + if rval != os.EX_OK: + out = portage.output.EOutput() + out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,)) + out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.") + def emerge_main(): global portage # NFC why this is necessary now - genone portage._disable_legacy_globals() @@ -12946,6 +13019,8 @@ def emerge_main(): settings, trees, mtimedb = load_emerge_config() portdb = trees[settings["ROOT"]]["porttree"].dbapi + ionice(settings) + try: os.nice(int(settings.get("PORTAGE_NICENESS", "0"))) except (OSError, ValueError), e: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index e9651184f..4e97519ec 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1003,7 +1003,8 @@ class config(object): "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE", "PORTAGE_GPG_DIR", - "PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT", + "PORTAGE_GPG_KEY", "PORTAGE_IONICE_COMMAND", + "PORTAGE_PACKAGE_EMPTY_ABORT", "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR", diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 7321a3a8a..4b648a915 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -2408,7 +2408,7 @@ class dblink(object): showMessage("%s checking %d files for package collisions\n" % \ (green("*"), len(mycontents))) for i, f in enumerate(mycontents): - if i % 1000 == 0: + if i % 1000 == 0 and i != 0: showMessage("%d files checked ...\n" % i) if scheduler is not None and \ diff --git a/pym/portage/util.py b/pym/portage/util.py index 998878ce6..7aa333399 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -340,6 +340,44 @@ class _tolerant_shlex(shlex.shlex): (self.infile, str(e)), noiselevel=-1) return (newfile, StringIO.StringIO()) +class _insert_newline_eof(file): + """ + Read functions insert anywhere from 0 and 2 newlines just before eof. + This is useful as a workaround for avoiding a silent error in shlex that + is triggered by a source statement at the end of the file without a + trailing newline after the source statement. + """ + + def read(self, *args): + if hasattr(self, "_got_eof"): + return "" + rval = file.read(self, *args) + if rval and not args and rval[-1:] != "\n": + rval += "\n" + if not rval: + self._got_eof = True + return "\n" + return rval + + def readline(self, *args): + if hasattr(self, "_got_eof"): + return "" + rval = file.readline(self, *args) + if rval and rval[-1:] != "\n": + rval += "\n" + if not rval: + self._got_eof = True + rval = "\n" + return rval + + def readlines(self, *args): + if hasattr(self, "_got_eof"): + return [] + lines = file.readlines(self, *args) + if lines and lines[-1][-1:] != "\n": + lines[-1] += "\n" + return lines + def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): if isinstance(expand, dict): # Some existing variable definitions have been @@ -350,7 +388,7 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): expand_map = {} mykeys = {} try: - f=open(mycfg,'r') + f = _insert_newline_eof(mycfg, 'rb') except IOError, e: if e.errno == PermissionDenied.errno: raise PermissionDenied(mycfg)