From cbf98c85d6bdd1eb0789bff2f8d1eb4d014c3ffa Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 16 Jul 2008 11:11:49 +0000 Subject: [PATCH] Merged from trunk 11060:11076 | 11061 | In Scheduler._restart_if_necessary(), use list.remove() | | zmedico | instead of assuming that portage is first in the resume | | | list. | | 11062 | Make EbuildFetcher copy FETCHCOMMAND and RESUMECOMMAND to | | zmedico | the fetch environment, in case the default values have been | | | overridden. | | 11063 | Add some cross references between emerge's new parallel | | zmedico | build options and MAKEOPTS. | | 11064 | When --jobs > 1, display a status message each time that | | zmedico | Scheduler._schedule_tasks() is called. For example: >>> | | | Jobs: 2 running, 0 merges, load average: 0.34, 0.46, 0.27 | | 11065 | Redirect dblink.unmerge() output to the log file when --jobs | | zmedico | > 1. | | 11066 | Adapt the dblink merge display callback to use log levels | | zmedico | from the logging module so that it can be used as an | | | interface for both writemsg() and writemsg_stdout(). | | 11067 | Make dblink._display_merge() detect scheduler presence | | zmedico | internally. | | 11068 | Redirect dblink.merge() output to the log file when --jobs > | | zmedico | 1. | | 11069 | Pass the whole dblink instance into | | zmedico | Scheduler._dblink_display_merge() in case it's ever needed. | | 11070 | Show LDFLAGS in emerge --info output. Thanks to dirtyepic | | zmedico | for this patch. | | 11071 | Remove the config._filter_calling_env attribute since it's | | zmedico | not really needed. Replace it with an simple | | | os.path.exists() call inside config.environ(). | | 11072 | Add async support for calling functions in | | zmedico | misc-functions.sh: * Create a new MiscFunctionsProcess class | | | derived from SpawnProcess. * Add "background" support to the | | | SpawnProcess class and split out a _spawn() template | | | function so MiscFunctionsProcess can override it. * Rename | | | the existing EbuildPhase class to EbuildProcess. * Create a | | | new EbuildPhase class derived from CompositeTask. This class | | | spawns the EbuildProcess and MiscFunctionsProcess classes as | | | sub-tasks. | | 11073 | * Add xterm titles support to Scheduler._display_status(). * | | zmedico | Tweak emergelog() so that it won't interfere with the | | | scheduler summary. | | 11075 | Add a progress counter showing "x of y complete" to | | zmedico | Scheduler._display_status(). | | 11076 | Pass background=True into prefetcher constructors. | | zmedico | | svn path=/main/branches/prefix/; revision=11077 --- man/emerge.1 | 4 + man/make.conf.5 | 6 +- pym/_emerge/__init__.py | 257 ++++++++++++++++++++++++++--------- pym/_emerge/help.py | 9 +- pym/portage/__init__.py | 57 +++++--- pym/portage/dbapi/vartree.py | 84 ++++++++---- 6 files changed, 303 insertions(+), 114 deletions(-) diff --git a/man/emerge.1 b/man/emerge.1 index 226d28a31..b928ab53f 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -335,6 +335,10 @@ the related \fB\-\-skipfirst\fR option. .BR \-\-load\-average=LOAD Specifies that no new builds should be started if there are other builds running and the load average is at least LOAD (a floating-point number). +This option is recommended for use in combination with \fB\-\-jobs\fR in +order to avoid excess load. See \fBmake\fR(1) for information about +analogous options that should be configured via \fBMAKEOPTS\fR in +\fBmake.conf\fR(5). .TP .BR "\-\-newuse " (\fB\-N\fR) Tells emerge to include installed packages where USE flags have changed since diff --git a/man/make.conf.5 b/man/make.conf.5 index 1a32e46f6..f05ed33e8 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -378,8 +378,10 @@ package. Use this variable if you want to use parallel make. For example, if you have a dual\-processor system, set this variable to "\-j2" or "\-j3" for enhanced build performance with many packages. Suggested settings are -between \fICPUs+1\fR and \fI2*CPUs+1\fR. -For more information, see \fBmake\fR(1). +between \fICPUs+1\fR and \fI2*CPUs+1\fR. In order to avoid +excess load, the \fB\-\-load\-average\fR option is recommended. +For more information, see \fBmake\fR(1). Also see \fBemerge\fR(1) for +information about analogous \fB\-\-jobs\fR and \fB\-\-load\-average\fR options. .TP \fBNOCOLOR\fR = \fI["true" | "false"]\fR Defines if color should be disabled by default. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 08056e4eb..c1c481597 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -23,6 +23,8 @@ except KeyboardInterrupt: import array from collections import deque import fcntl +import fpformat +import logging import select import shlex import shutil @@ -259,9 +261,7 @@ shortmapping={ } def emergelog(xterm_titles, mystr, short_msg=None): - if xterm_titles: - if short_msg == None: - short_msg = mystr + if xterm_titles and short_msg: if "HOSTNAME" in os.environ: short_msg = os.environ["HOSTNAME"]+": "+short_msg xtermTitle(short_msg) @@ -2041,7 +2041,7 @@ class SpawnProcess(SubProcess): __slots__ = ("args",) + \ _spawn_kwarg_names - _file_names = ("process", "out") + _file_names = ("log", "process", "stdout") _files_dict = slot_dict_class(_file_names, prefix="") _bufsize = 4096 @@ -2072,6 +2072,7 @@ class SpawnProcess(SubProcess): fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + files.process = os.fdopen(master_fd, 'r') if logfile is not None: fd_pipes_orig = fd_pipes.copy() @@ -2079,11 +2080,14 @@ class SpawnProcess(SubProcess): fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd - files.out = open(logfile, "a") + files.log = open(logfile, "a") portage.util.apply_secpass_permissions(logfile, uid=portage.portage_uid, gid=portage.portage_gid, mode=0660) + if not self.background: + files.stdout = os.fdopen(os.dup(fd_pipes_orig[1]), 'w') + output_handler = self._output_handler else: @@ -2106,13 +2110,15 @@ class SpawnProcess(SubProcess): kwargs["returnpid"] = True kwargs.pop("logfile", None) - retval = portage.process.spawn(self.args, **kwargs) + retval = self._spawn(self.args, **kwargs) os.close(slave_fd) if isinstance(retval, int): # spawn failed os.close(master_fd) + for f in self.files.values(): + f.close() self.returncode = retval self.wait() return @@ -2120,12 +2126,13 @@ class SpawnProcess(SubProcess): self.pid = retval[0] portage.process.spawned_pids.remove(self.pid) - - files.process = os.fdopen(master_fd, 'r') self._reg_id = self.scheduler.register(files.process.fileno(), PollConstants.POLLIN, output_handler) self._registered = True + def _spawn(self, args, **kwargs): + return portage.process.spawn(args, **kwargs) + def _output_handler(self, fd, event): files = self._files buf = array.array('B') @@ -2134,8 +2141,11 @@ class SpawnProcess(SubProcess): except EOFError: pass if buf: - buf.tofile(files.out) - files.out.flush() + if not self.background: + buf.tofile(files.stdout) + files.stdout.flush() + buf.tofile(files.log) + files.log.flush() else: self._unregister() self.wait() @@ -2160,10 +2170,44 @@ class SpawnProcess(SubProcess): self.wait() return self._registered +class MiscFunctionsProcess(SpawnProcess): + """ + Spawns misc-functions.sh with an existing ebuild environment. + """ + + __slots__ = ("commands", "phase", "pkg", "settings") + + def _start(self): + settings = self.settings + portage_bin_path = settings["PORTAGE_BIN_PATH"] + misc_sh_binary = os.path.join(portage_bin_path, + os.path.basename(portage.const.MISC_SH_BINARY)) + + self.args = [portage._shell_quote(misc_sh_binary)] + self.commands + self.logfile = settings.get("PORTAGE_LOG_FILE") + + portage._doebuild_exit_status_unlink( + settings.get("EBUILD_EXIT_STATUS_FILE")) + + SpawnProcess._start(self) + + def _spawn(self, args, **kwargs): + settings = self.settings + debug = settings.get("PORTAGE_DEBUG") == "1" + return portage.spawn(" ".join(args), settings, + debug=debug, **kwargs) + + def _set_returncode(self, wait_retval): + SpawnProcess._set_returncode(self, wait_retval) + self.returncode = portage._doebuild_exit_status_check_and_log( + self.settings, self.phase, self.returncode) + class EbuildFetcher(SpawnProcess): __slots__ = ("fetchonly", "pkg",) + _env_vars = ("FETCHCOMMAND", "RESUMECOMMAND") + def _start(self): root_config = self.pkg.root_config @@ -2173,6 +2217,10 @@ class EbuildFetcher(SpawnProcess): fetch_env = settings.environ() fetch_env["PORTAGE_NICENESS"] = "0" + for k in self._env_vars: + v = settings.get(k) + if v is not None: + fetch_env[k] = v if self.fetchonly: fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1" @@ -2528,17 +2576,10 @@ class EbuildExecuter(CompositeTask): # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(pkg.root, settings, cleanup) - fd_pipes = { - 0 : sys.stdin.fileno(), - 1 : sys.stdout.fileno(), - 2 : sys.stderr.fileno(), - } - ebuild_phases = TaskSequence(scheduler=scheduler) for phase in self._phases: ebuild_phases.add(EbuildPhase(background=self.background, - fd_pipes=fd_pipes, pkg=pkg, phase=phase, scheduler=scheduler, settings=settings, tree=tree)) @@ -2632,7 +2673,7 @@ class EbuildMetadataPhase(SubProcess): return self._registered -class EbuildPhase(SubProcess): +class EbuildProcess(SubProcess): __slots__ = ("fd_pipes", "phase", "pkg", "settings", "tree") @@ -2777,41 +2818,57 @@ class EbuildPhase(SubProcess): settings, self.phase, self.returncode) portage._post_phase_userpriv_perms(settings) + +class EbuildPhase(CompositeTask): + + __slots__ = ("background", "pkg", "phase", + "scheduler", "settings", "tree") + + _post_phase_cmds = portage._post_phase_cmds + + def _start(self): + + ebuild_process = EbuildProcess(background=self.background, + pkg=self.pkg, phase=self.phase, scheduler=self.scheduler, + settings=self.settings, tree=self.tree) + + self._start_task(ebuild_process, self._ebuild_exit) + + def _ebuild_exit(self, ebuild_process): + if self.phase == "install": - portage._check_build_log(settings) - if self.returncode == os.EX_OK: - self.returncode = portage._post_src_install_checks(settings) + portage._check_build_log(self.settings) - elif self.phase == "preinst": - if self.returncode == os.EX_OK: - portage._doebuild_exit_status_unlink( - settings.get("EBUILD_EXIT_STATUS_FILE")) - phase_retval = portage.spawn( - " ".join(portage._post_pkg_preinst_cmd(settings)), - settings, debug=debug, free=1, logfile=log_path) - phase_retval = portage._doebuild_exit_status_check_and_log( - settings, self.phase, phase_retval) - if phase_retval != os.EX_OK: - writemsg("!!! post preinst failed; exiting.\n", - noiselevel=-1) - self.returncode = phase_retval + if self._default_exit(ebuild_process) != os.EX_OK: + self.wait() + return - elif self.phase == "postinst": + settings = self.settings - if self.returncode == os.EX_OK: - portage._doebuild_exit_status_unlink( - settings.get("EBUILD_EXIT_STATUS_FILE")) - phase_retval = portage.spawn( - " ".join(portage._post_pkg_postinst_cmd(settings)), - settings, debug=debug, free=1, logfile=log_path) - phase_retval = portage._doebuild_exit_status_check_and_log( - settings, self.phase, phase_retval) - if phase_retval != os.EX_OK: - writemsg("!!! post postinst failed; exiting.\n", - noiselevel=-1) - self.returncode = phase_retval + if self.phase == "install": + portage._post_src_install_uid_fix(settings) + + post_phase_cmds = self._post_phase_cmds.get(self.phase) + if post_phase_cmds is not None: + post_phase = MiscFunctionsProcess(background=self.background, + commands=post_phase_cmds, phase=self.phase, pkg=self.pkg, + scheduler=self.scheduler, settings=settings) + self._start_task(post_phase, self._post_phase_exit) + return -class EbuildBinpkg(EbuildPhase): + self.returncode = ebuild_process.returncode + self._current_task = None + self.wait() + + def _post_phase_exit(self, post_phase): + if self._final_exit(post_phase) != os.EX_OK: + writemsg("!!! post %s failed; exiting.\n" % self.phase, + noiselevel=-1) + self._current_task = None + self.wait() + return + +class EbuildBinpkg(EbuildProcess): """ This assumes that src_install() has successfully completed. """ @@ -2836,12 +2893,12 @@ class EbuildBinpkg(EbuildPhase): settings.backup_changes("PORTAGE_BINPKG_TMPFILE") try: - EbuildPhase._start(self) + EbuildProcess._start(self) finally: settings.pop("PORTAGE_BINPKG_TMPFILE", None) def _set_returncode(self, wait_retval): - EbuildPhase._set_returncode(self, wait_retval) + EbuildProcess._set_returncode(self, wait_retval) pkg = self.pkg bintree = pkg.root_config.trees["bintree"] @@ -8354,7 +8411,7 @@ class Scheduler(PollScheduler): _fetch_log = EPREFIX + "/var/log/emerge-fetch.log" class _iface_class(SlotObject): - __slots__ = ("dblinkEbuildPhase", "fetch", + __slots__ = ("dblinkEbuildPhase", "dblinkDisplayMerge", "fetch", "register", "schedule", "unregister") class _fetch_iface_class(SlotObject): @@ -8374,9 +8431,12 @@ class Scheduler(PollScheduler): __slots__ = ("curval", "maxval") class _emerge_log_class(SlotObject): - __slots__ = ("xterm_titles",) + __slots__ = ("parallel", "xterm_titles",) def log(self, *pargs, **kwargs): + if self.parallel: + # Avoid interference with the scheduler's status display. + kwargs.pop("short_msg", None) emergelog(self.xterm_titles, *pargs, **kwargs) def __init__(self, settings, trees, mtimedb, myopts, @@ -8419,6 +8479,7 @@ class Scheduler(PollScheduler): schedule=self._schedule_fetch) self._sched_iface = self._iface_class( dblinkEbuildPhase=self._dblink_ebuild_phase, + dblinkDisplayMerge=self._dblink_display_merge, fetch=fetch_iface, register=self._register, schedule=self._schedule_wait, unregister=self._unregister) @@ -8430,6 +8491,9 @@ class Scheduler(PollScheduler): self._prefetchers = weakref.WeakValueDictionary() self._pkg_queue = [] self._completed_tasks = set() + # Number of completed package tasks, excluding uninstalls. + self._completed_pkg_count = 0 + self._summary_prev_pkg_count = 0 self._failed_pkgs = [] self._failed_pkgs_all = [] self._failed_pkgs_die_msgs = [] @@ -8608,6 +8672,24 @@ class Scheduler(PollScheduler): finally: f.close() + def _dblink_display_merge(self, pkg_dblink, msg, level=0): + log_path = pkg_dblink.settings.get("PORTAGE_LOG_FILE") + background = self._max_jobs > 1 + + if level >= logging.WARNING: + noiselevel = -1 + msg_func = writemsg + else: + noiselevel = 0 + msg_func = portage.writemsg_stdout + + if log_path is None: + msg_func(msg, noiselevel=noiselevel) + else: + if not background: + msg_func(msg, noiselevel=noiselevel) + self._append_to_log_path(log_path, msg) + def _dblink_ebuild_phase(self, pkg_dblink, pkg_dbapi, ebuild_path, phase): """ @@ -8706,16 +8788,17 @@ class Scheduler(PollScheduler): elif pkg.type_name == "ebuild": - prefetcher = EbuildFetcher(fetchonly=1, - logfile=self._fetch_log, pkg=pkg, - scheduler=self._sched_iface) + prefetcher = EbuildFetcher(background=True, + fetchonly=1, logfile=self._fetch_log, + pkg=pkg, scheduler=self._sched_iface) elif pkg.type_name == "binary" and \ "--getbinpkg" in self.myopts and \ pkg.root_config.trees["bintree"].isremote(pkg.cpv): - prefetcher = BinpkgFetcher(logfile=self._fetch_log, - pkg=pkg, scheduler=self._sched_iface) + prefetcher = BinpkgFetcher(background=True, + logfile=self._fetch_log, pkg=pkg, + scheduler=self._sched_iface) return prefetcher @@ -8802,7 +8885,7 @@ class Scheduler(PollScheduler): "emerge via exec() after change of " + \ "portage version.") - del mtimedb["resume"]["mergelist"][0] + mtimedb["resume"]["mergelist"].remove(list(pkg)) mtimedb.commit() portage.run_exitfuncs() mynewargv = [sys.argv[0], "--resume"] @@ -8978,6 +9061,7 @@ class Scheduler(PollScheduler): if not mtimedb["resume"]["mergelist"]: del mtimedb["resume"] mtimedb.commit() + self._completed_pkg_count += 1 def _build_exit(self, build): if build.returncode == os.EX_OK: @@ -9016,6 +9100,8 @@ class Scheduler(PollScheduler): def _main_loop_cleanup(self): del self._pkg_queue[:] self._completed_tasks.clear() + self._completed_pkg_count = 0 + self._summary_prev_pkg_count = 0 self._digraph = None self._task_queues.fetch.clear() @@ -9095,6 +9181,15 @@ class Scheduler(PollScheduler): self._poll_loop() def _schedule_tasks(self): + remaining, state_change = self._schedule_tasks_imp() + + if state_change or \ + self._summary_prev_pkg_count != self._completed_pkg_count: + self._display_status() + self._summary_prev_pkg_count = self._completed_pkg_count + return remaining + + def _schedule_tasks_imp(self): """ @rtype: bool @returns: True if tasks remain to schedule, False otherwise. @@ -9102,15 +9197,19 @@ class Scheduler(PollScheduler): task_queues = self._task_queues background = self._max_jobs > 1 + self._logger.parallel = background + state_change = 0 while self._can_add_job(): if not self._pkg_queue or self._failed_pkgs: - return False + return (False, state_change) pkg = self._choose_pkg() if pkg is None: - return True + return (True, state_change) + + state_change += 1 if not pkg.installed: self._pkg_count.curval += 1 @@ -9129,7 +9228,41 @@ class Scheduler(PollScheduler): self._jobs += 1 task.addExitListener(self._build_exit) task_queues.jobs.add(task) - return True + return (True, state_change) + + def _load_avg_str(self, digits=1): + try: + avg = os.getloadavg() + except OSError, e: + return str(e) + return ", ".join(fpformat.fix(x, digits) for x in avg) + + def _display_status(self): + if self._max_jobs < 2: + return + + # Don't use len(self._completed_tasks) here since that also + # can include uninstall tasks. + completed_str = str(self._completed_pkg_count) + maxval_str = str(self._pkg_count.maxval) + jobs_str = str(self._jobs) + merges_str = str(len(self._task_queues.merge)) + load_avg_str = self._load_avg_str() + + msg = ("Jobs: %s of %s complete, %s running, %s merges, " + \ + "load average: %s") % \ + (colorize("INFORM", completed_str), colorize("INFORM", maxval_str), + colorize("INFORM", jobs_str), colorize("INFORM", merges_str), + load_avg_str) + noiselevel = 0 + if "--verbose" in self.myopts: + noiselevel = -1 + portage.writemsg_stdout(">>> %s\n" % msg, noiselevel=noiselevel) + + short_msg = ("Jobs: %s of %s complete, %s running, %s merges, " + \ + "load average: %s") % \ + (completed_str, maxval_str, jobs_str, merges_str, load_avg_str) + xtermTitle(short_msg) def _task(self, pkg, background): @@ -10955,7 +11088,7 @@ def action_info(settings, trees, myopts, myfiles): if mypkgs: # Get our global settings (we only print stuff if it varies from # the current config) - mydesiredvars = [ 'CHOST', 'CFLAGS', 'CXXFLAGS', 'EPREFIX' ] + mydesiredvars = [ 'CHOST', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'EPREFIX' ] auxkeys = mydesiredvars + [ "USE", "IUSE"] global_vals = {} pkgsettings = portage.config(clone=settings) diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index 756f9156b..bf6a13834 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -319,7 +319,14 @@ def help(myaction,myopts,havecolor=1): desc = "Specifies that no new builds should " + \ "be started if there are other builds " + \ "running and the load average is at " + \ - "least LOAD (a floating-point number)." + "least LOAD (a floating-point number). " + \ + "This option is recommended for use " + \ + "in combination with --jobs in " + \ + "order to avoid excess load. See " + \ + "make(1) for information about " + \ + "analogous options that should be " + \ + "configured via MAKEOPTS in " + \ + "make.conf(5)." for line in wrap(desc, desc_width): print desc_indent + line print diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index fbd3d1950..6da2e7ffd 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1041,7 +1041,6 @@ class config(object): self.already_in_regenerate = 0 - self._filter_calling_env = False self.locked = 0 self.mycpv = None self.puse = [] @@ -1067,7 +1066,6 @@ class config(object): self.local_config = local_config if clone: - self._filter_calling_env = copy.deepcopy(clone._filter_calling_env) self.incrementals = copy.deepcopy(clone.incrementals) self.profile_path = copy.deepcopy(clone.profile_path) self.user_profile_dir = copy.deepcopy(clone.user_profile_dir) @@ -2828,7 +2826,13 @@ class config(object): "return our locally-maintained environment" mydict={} environ_filter = self._environ_filter - filter_calling_env = self._filter_calling_env + + filter_calling_env = False + temp_dir = self.get("T") + if temp_dir is not None and \ + os.path.exists(os.path.join(temp_dir, "environment")): + filter_calling_env = True + environ_whitelist = self._environ_whitelist env_d = self.configdict["env.d"] for x in self: @@ -4250,16 +4254,14 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, mysettings["EBUILD_PHASE"] = mydo _doebuild_exit_status_unlink( mysettings.get("EBUILD_EXIT_STATUS_FILE")) - filter_calling_env_state = mysettings._filter_calling_env - if os.path.exists(os.path.join(mysettings["T"], "environment")): - mysettings._filter_calling_env = True + try: phase_retval = spawn(actionmap[mydo]["cmd"] % mydo, mysettings, debug=debug, logfile=logfile, fd_pipes=fd_pipes, returnpid=returnpid, **kwargs) finally: mysettings["EBUILD_PHASE"] = "" - mysettings._filter_calling_env = filter_calling_env_state + if returnpid: return phase_retval msg = _doebuild_exit_status_check(mydo, mysettings) @@ -4277,6 +4279,23 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, phase_retval = _post_src_install_checks(mysettings) return phase_retval +_post_phase_cmds = { + + "install" : [ + "install_qa_check", + "install_symlink_html_docs"], + + "preinst" : [ + "preinst_bsdflags", + "preinst_sfperms", + "preinst_selinux_labels", + "preinst_suid_scan", + "preinst_mask"], + + "postinst" : [ + "postinst_bsdflags"] +} + def _post_phase_userpriv_perms(mysettings): if "userpriv" in mysettings.features and secpass >= 2: """ Privileged phases may have left files that need to be made @@ -4287,8 +4306,8 @@ def _post_phase_userpriv_perms(mysettings): def _post_src_install_checks(mysettings): _post_src_install_uid_fix(mysettings) - retval = _spawn_misc_sh(mysettings, ["install_qa_check", - "install_symlink_html_docs"]) + global _post_phase_cmds + retval = _spawn_misc_sh(mysettings, post_phase_cmds["install"]) if retval != os.EX_OK: writemsg("!!! install_qa_check failed; exiting.\n", noiselevel=-1) @@ -4414,10 +4433,8 @@ def _post_pkg_preinst_cmd(mysettings): os.path.basename(MISC_SH_BINARY)) mysettings["EBUILD_PHASE"] = "" - myargs = [_shell_quote(misc_sh_binary), - "preinst_bsdflags", - "preinst_sfperms", "preinst_selinux_labels", - "preinst_suid_scan", "preinst_mask"] + global _post_phase_cmds + myargs = [_shell_quote(misc_sh_binary)] + _post_phase_cmds["preinst"] return myargs @@ -4432,7 +4449,8 @@ def _post_pkg_postinst_cmd(mysettings): os.path.basename(MISC_SH_BINARY)) mysettings["EBUILD_PHASE"] = "" - myargs = [_shell_quote(misc_sh_binary), "postinst_bsdflags"] + global _post_phase_cmds + myargs = [_shell_quote(misc_sh_binary)] + _post_phase_cmds["postinst"] return myargs @@ -4454,9 +4472,6 @@ def _spawn_misc_sh(mysettings, commands, **kwargs): mycommand = " ".join([_shell_quote(misc_sh_binary)] + commands) _doebuild_exit_status_unlink( mysettings.get("EBUILD_EXIT_STATUS_FILE")) - filter_calling_env_state = mysettings._filter_calling_env - if os.path.exists(os.path.join(mysettings["T"], "environment")): - mysettings._filter_calling_env = True debug = mysettings.get("PORTAGE_DEBUG") == "1" logfile = mysettings.get("PORTAGE_LOG_FILE") mydo = mysettings["EBUILD_PHASE"] @@ -4464,7 +4479,7 @@ def _spawn_misc_sh(mysettings, commands, **kwargs): rval = spawn(mycommand, mysettings, debug=debug, logfile=logfile, **kwargs) finally: - mysettings._filter_calling_env = filter_calling_env_state + pass msg = _doebuild_exit_status_check(mydo, mysettings) if msg: rval = 1 @@ -5111,7 +5126,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, builddir_lock = None tmpdir = None tmpdir_orig = None - filter_calling_env_state = mysettings._filter_calling_env + try: if mydo in ("digest", "manifest", "help"): # Temporarily exempt the depend phase from manifest checks, in case @@ -5327,7 +5342,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, del e env_stat = None if env_stat: - mysettings._filter_calling_env = True + pass else: for var in ("ARCH", ): value = mysettings.get(var) @@ -5636,7 +5651,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return retval finally: - mysettings._filter_calling_env = filter_calling_env_state + if tmpdir: mysettings["PORTAGE_TMPDIR"] = tmpdir_orig shutil.rmtree(tmpdir) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 360d303ad..4bc7ab87b 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -28,6 +28,7 @@ from portage.elog.messages import ewarn from portage.elog.filtering import filter_mergephases, filter_unmergephases import os, re, sys, stat, errno, commands, copy, time, subprocess +import logging from itertools import izip try: @@ -1878,6 +1879,19 @@ class dblink(object): contents=contents, env=self.settings.environ()) return os.EX_OK + def _display_merge(self, msg, level=0): + if self._scheduler is not None: + self._scheduler.dblinkDisplayMerge(self, msg, level=level) + return + + if level >= logging.WARNING: + noiselevel = -1 + msg_func = writemsg + else: + noiselevel = 0 + msg_func = writemsg_stdout + msg_func(msg, noiselevel=noiselevel) + def _unmerge_pkgfiles(self, pkgfiles, others_in_slot): """ @@ -1891,8 +1905,10 @@ class dblink(object): @rtype: None """ + showMessage = self._display_merge + if not pkgfiles: - writemsg_stdout("No package files given... Grabbing a set.\n") + showMessage("No package files given... Grabbing a set.\n") pkgfiles = self.getcontents() if others_in_slot is None: @@ -1956,7 +1972,7 @@ class dblink(object): bsd_chflags.chflags(parent_name, pflags) def show_unmerge(zing, desc, file_type, file_name): - writemsg_stdout("%s %s %s %s\n" % \ + showMessage("%s %s %s %s\n" % \ (zing, desc.ljust(8), file_type, file_name)) for objkey in mykeys: obj = normalize_path(objkey) @@ -2334,17 +2350,18 @@ class dblink(object): collision_ignore = set([normalize_path(myignore) for myignore in \ self.settings.get("COLLISION_IGNORE", "").split()]) + showMessage = self._display_merge stopmerge = False i=0 collisions = [] destroot = normalize_path(destroot).rstrip(os.path.sep) + \ os.path.sep - writemsg_stdout("%s checking %d files for package collisions\n" % \ + showMessage("%s checking %d files for package collisions\n" % \ (green("*"), len(mycontents))) for f in mycontents: i = i + 1 if i % 1000 == 0: - writemsg_stdout("%d files checked ...\n" % i) + showMessage("%d files checked ...\n" % i) dest_path = normalize_path( os.path.join(destroot, f.lstrip(os.path.sep))) try: @@ -2402,6 +2419,9 @@ class dblink(object): def _security_check(self, installed_instances): if not installed_instances: return 0 + + showMessage = self._display_merge + file_paths = set() for dblnk in installed_instances: file_paths.update(dblnk.getcontents()) @@ -2436,13 +2456,13 @@ class dblink(object): return 0 from portage.output import colorize prefix = colorize("SECURITY_WARN", "*") + " WARNING: " - writemsg(prefix + "suid/sgid file(s) " + \ - "with suspicious hardlink(s):\n", noiselevel=-1) + showMessage(prefix + "suid/sgid file(s) " + \ + "with suspicious hardlink(s):\n", level=logging.ERROR) for path_list in suspicious_hardlinks: for path, s in path_list: - writemsg(prefix + " '%s'\n" % path, noiselevel=-1) - writemsg(prefix + "See the Gentoo Security Handbook " + \ - "guide for advice on how to proceed.\n", noiselevel=-1) + showMessage(prefix + " '%s'\n" % path, level=logging.ERROR) + showMessage(prefix + "See the Gentoo Security Handbook " + \ + "guide for advice on how to proceed.\n", level=logging.ERROR) return 1 def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0, @@ -2481,12 +2501,14 @@ class dblink(object): not existing; we will merge these symlinks at a later time. """ + showMessage = self._display_merge + srcroot = normalize_path(srcroot).rstrip(os.path.sep) + os.path.sep destroot = normalize_path(destroot).rstrip(os.path.sep) + os.path.sep if not os.path.isdir(srcroot): - writemsg("!!! Directory Not Found: D='%s'\n" % srcroot, - noiselevel=-1) + showMessage("!!! Directory Not Found: D='%s'\n" % srcroot, + level=logging.ERROR) return 1 inforoot_slot_file = os.path.join(inforoot, "SLOT") @@ -2511,8 +2533,8 @@ class dblink(object): _eerror(l, phase="preinst", key=self.settings.mycpv) if slot != self.settings["SLOT"]: - writemsg("!!! WARNING: Expected SLOT='%s', got '%s'\n" % \ - (self.settings["SLOT"], slot)) + showMessage("!!! WARNING: Expected SLOT='%s', got '%s'\n" % \ + (self.settings["SLOT"], slot), level=logging.WARN) if not os.path.exists(self.dbcatdir): os.makedirs(self.dbcatdir) @@ -2729,7 +2751,7 @@ class dblink(object): # run preinst script if scheduler is None: - writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot)) + showMessage(">>> Merging %s to %s\n" % (self.mycpv, destroot)) a = doebuild(myebuild, "preinst", destroot, self.settings, use_cache=0, tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) @@ -2739,7 +2761,8 @@ class dblink(object): # XXX: Decide how to handle failures here. if a != os.EX_OK: - writemsg("!!! FAILED preinst: "+str(a)+"\n", noiselevel=-1) + showMessage("!!! FAILED preinst: "+str(a)+"\n", + level=logging.ERROR) return a # copy "info" files (like SLOT, CFLAGS, etc.) into the database @@ -2849,19 +2872,20 @@ class dblink(object): continue if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self): continue - writemsg_stdout(">>> Safely unmerging already-installed instance...\n") + showMessage(">>> Safely unmerging already-installed instance...\n") others_in_slot.remove(dblnk) # dblnk will unmerge itself now dblnk.unmerge(trimworld=0, ldpath_mtimes=prev_mtimes, others_in_slot=others_in_slot) # TODO: Check status and abort if necessary. dblnk.delete() - writemsg_stdout(">>> Original instance of package unmerged safely.\n") + showMessage(">>> Original instance of package unmerged safely.\n") if len(others_in_slot) > 1: from portage.output import colorize - writemsg_stdout(colorize("WARN", "WARNING:") + showMessage(colorize("WARN", "WARNING:") + " AUTOCLEAN is disabled. This can cause serious" - + " problems due to overlapping packages.\n") + + " problems due to overlapping packages.\n", + level=logging.WARN) # We hold both directory locks. self.dbdir = self.dbpkgdir @@ -2894,7 +2918,7 @@ class dblink(object): use_cache=0, tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) if a == os.EX_OK: - writemsg_stdout(">>> %s %s\n" % (self.mycpv, "merged.")) + showMessage(">>> %s %s\n" % (self.mycpv, "merged.")) else: a = scheduler.dblinkEbuildPhase( self, mydbapi, myebuild, "postinst") @@ -2903,7 +2927,8 @@ class dblink(object): # XXX: Decide how to handle failures here. if a != os.EX_OK: - writemsg("!!! FAILED postinst: "+str(a)+"\n", noiselevel=-1) + showMessage("!!! FAILED postinst: "+str(a)+"\n", + level=logging.ERROR) return a downgrade = False @@ -2945,6 +2970,9 @@ class dblink(object): 2. None otherwise """ + + showMessage = self._display_merge + from os.path import sep, join srcroot = normalize_path(srcroot).rstrip(sep) + sep destroot = normalize_path(destroot).rstrip(sep) + sep @@ -3045,7 +3073,7 @@ class dblink(object): # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) if mymtime != None: - writemsg_stdout(">>> %s -> %s\n" % (mydest, myto)) + showMessage(">>> %s -> %s\n" % (mydest, myto)) outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") else: print "!!! Failed to move file." @@ -3073,7 +3101,7 @@ class dblink(object): if stat.S_ISLNK(mydmode) or stat.S_ISDIR(mydmode): # a symlink to an existing directory will work for us; keep it: - writemsg_stdout("--- %s/\n" % mydest) + showMessage("--- %s/\n" % mydest) if bsd_chflags: bsd_chflags.lchflags(mydest, dflags) else: @@ -3092,7 +3120,7 @@ class dblink(object): bsd_chflags.lchflags(mydest, dflags) os.chmod(mydest, mystat[0]) os.chown(mydest, mystat[4], mystat[5]) - writemsg_stdout(">>> %s/\n" % mydest) + showMessage(">>> %s/\n" % mydest) else: #destination doesn't exist if self.settings.selinux_enabled(): @@ -3103,7 +3131,7 @@ class dblink(object): os.mkdir(mydest) os.chmod(mydest, mystat[0]) os.chown(mydest, mystat[4], mystat[5]) - writemsg_stdout(">>> %s/\n" % mydest) + showMessage(">>> %s/\n" % mydest) outfile.write("dir "+myrealdest+"\n") # recurse and merge this directory if self.mergeme(srcroot, destroot, outfile, secondhand, @@ -3122,7 +3150,7 @@ class dblink(object): if stat.S_ISDIR(mydmode): # install of destination is blocked by an existing directory with the same name moveme = 0 - writemsg_stdout("!!! %s\n" % mydest) + showMessage("!!! %s\n" % mydest, level=logging.ERROR) elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])): cfgprot = 0 # install of destination is blocked by an existing regular file, @@ -3168,7 +3196,7 @@ class dblink(object): if mymtime != None: outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") - writemsg_stdout("%s %s\n" % (zing,mydest)) + showMessage("%s %s\n" % (zing,mydest)) else: # we are merging a fifo or device node zing = "!!!" @@ -3182,7 +3210,7 @@ class dblink(object): outfile.write("fif %s\n" % myrealdest) else: outfile.write("dev %s\n" % myrealdest) - writemsg_stdout(zing + " " + mydest + "\n") + showMessage(zing + " " + mydest + "\n") def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, mydbapi=None, prev_mtimes=None): -- 2.26.2