From c94e394b1d4c78792ed3d27832790879e755fe4d Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 17 Jul 2008 14:03:12 +0000 Subject: [PATCH] Merged from trunk 11087:11105 | 11088 | Avoid raising an IndexError from deprecated_profile_check() | | zmedico | if the deprecated file happens to be empty. Instead just | | | refer to the "Gentoo Upgrading Guide". Thanks to armin76 for | | | reporting. | | 11089 | Remove TODO since it no longer exists. | | zmedico | | | 11090 | Add --anon option for anonsvn.gentoo.org users. | | zmedico | | | 11091 | Add --anon to the usage message. | | zmedico | | | 11092 | Eliminate duplicate code by making EbuildProcess inherit | | zmedico | from SpawnProcess instead of SubProcess. | | 11093 | When SpawnProcess._start() spawns a process in the | | zmedico | background, use /dev/null so that any attempts to read from | | | stdin will immediately return EOF instead of blocking | | | indefinitely. TODO: Use job control functions like | | | tcsetpgrp() to control access to stdin. | | 11094 | Add the new --anon option to the getopt args. | | zmedico | | | 11095 | In SequentialTaskQueue.clear(), remove the exit listener | | zmedico | from each task before cancelling it. | | 11096 | Add a JobStatusDisplay class to encapsulate the status | | zmedico | display. This object tracks whether or not any of it's | | | attributes have changed since the last time it was | | | displayed, so the it's only displayed when something has | | | changed. Unlike before, now the display should always update | | | whenever a relevant change occurs. | | 11097 | Don't increment the "complete" count for uninstall tasks. | | zmedico | | | 11098 | * Add a new portage.util.writemsg_level() function which | | zmedico | uses a "level" parameter to control whether the message is | | | sent to stdout or stderr. When level >= logging.WARNING then | | | the message is sent to stderr, otherwise it is sent to | | | stdout. The noiselevel is passed directly to writemsg(). * | | | Use writemsg_level() for handling merge messages from dblink | | | and add appropriate noiselevel values to each call. * Add a | | | writemsg_level parameter to portage.env_update() so that | | | dblink can pass in dblink._display_merge() as a means to | | | redirect output back to the scheduler for appropriate | | | handling when running in the background. | | 11099 | Tweak interaction between Scheduler and status display so | | zmedico | that it doesn't display more often than necessary and avoids | | | showing inconsistent counts. | | 11100 | Handle the case in dblink.unmerge() when the log file is a | | zmedico | temp file and so it doesn't exist after postrm cleaning. | | 11101 | Don't show a blank newline before ">>> Emerging" when in | | zmedico | background mode. | | 11102 | Make BinpkgVerifier redirect output to the fetch log when in | | zmedico | background mode. | | 11103 | Update the status display in _build_exit(), after | | zmedico | decrementing the job count. | | 11104 | Make Binpkg send "Extracting" messages to the log file when | | zmedico | in background mode, instead of stdout. | | 11105 | Don't show "Merging" and "merged." messages when in | | zmedico | background mode. | svn path=/main/branches/prefix/; revision=11108 --- pym/_emerge/__init__.py | 426 ++++++++++++++++++----------------- pym/portage/__init__.py | 26 ++- pym/portage/dbapi/vartree.py | 56 +++-- pym/portage/util.py | 20 ++ 4 files changed, 285 insertions(+), 243 deletions(-) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index b72f3a8a0..565c302b0 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2068,15 +2068,25 @@ class SpawnProcess(SubProcess): self._files = self._files_dict() files = self._files - master_fd, slave_fd = os.pipe() + master_fd, slave_fd = self._pipe(fd_pipes) fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + null_input = None + fd_pipes_orig = fd_pipes.copy() + if self.background: + # TODO: Use job control functions like tcsetpgrp() to control + # access to stdin. Until then, use /dev/null so that any + # attempts to read from stdin will immediately return EOF + # instead of blocking indefinitely. + null_input = open('/dev/null', 'rb') + fd_pipes[0] = null_input.fileno() + else: + fd_pipes[0] = fd_pipes_orig[0] + files.process = os.fdopen(master_fd, 'r') if logfile is not None: - fd_pipes_orig = fd_pipes.copy() - fd_pipes[0] = fd_pipes_orig[0] fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd @@ -2113,6 +2123,8 @@ class SpawnProcess(SubProcess): retval = self._spawn(self.args, **kwargs) os.close(slave_fd) + if null_input is not None: + null_input.close() if isinstance(retval, int): # spawn failed @@ -2130,6 +2142,13 @@ class SpawnProcess(SubProcess): PollConstants.POLLIN, output_handler) self._registered = True + def _pipe(self, fd_pipes): + """ + @type fd_pipes: dict + @param fd_pipes: pipes from which to copy terminal size if desired. + """ + return os.pipe() + def _spawn(self, args, **kwargs): return portage.process.spawn(args, **kwargs) @@ -2673,151 +2692,43 @@ class EbuildMetadataPhase(SubProcess): return self._registered -class EbuildProcess(SubProcess): +class EbuildProcess(SpawnProcess): - __slots__ = ("fd_pipes", "phase", "pkg", - "settings", "tree") - - _file_names = ("log", "stdout", "ebuild") - _files_dict = slot_dict_class(_file_names, prefix="") - _bufsize = 4096 + __slots__ = ("phase", "pkg", "settings", "tree") def _start(self): + self.logfile = self.settings.get("PORTAGE_LOG_FILE") + SpawnProcess._start(self) + + def _pipe(self, fd_pipes): + stdout_pipe = fd_pipes.get(1) + got_pty, master_fd, slave_fd = \ + portage._create_pty_or_pipe(copy_term_size=stdout_pipe) + return (master_fd, slave_fd) + + def _spawn(self, args, **kwargs): + root_config = self.pkg.root_config tree = self.tree mydbapi = root_config.trees[tree].dbapi settings = self.settings ebuild_path = settings["EBUILD"] debug = settings.get("PORTAGE_DEBUG") == "1" - logfile = settings.get("PORTAGE_LOG_FILE") - master_fd = None - slave_fd = None - fd_pipes = None - if self.fd_pipes is not None: - fd_pipes = self.fd_pipes.copy() - else: - fd_pipes = {} - - fd_pipes.setdefault(0, sys.stdin.fileno()) - fd_pipes.setdefault(1, sys.stdout.fileno()) - fd_pipes.setdefault(2, sys.stderr.fileno()) - - # flush any pending output - for fd in fd_pipes.itervalues(): - if fd == sys.stdout.fileno(): - sys.stdout.flush() - if fd == sys.stderr.fileno(): - sys.stderr.flush() - - fd_pipes_orig = fd_pipes.copy() - self._files = self._files_dict() - files = self._files - got_pty = False - - portage._doebuild_exit_status_unlink( - settings.get("EBUILD_EXIT_STATUS_FILE")) - - if logfile: - got_pty, master_fd, slave_fd = \ - portage._create_pty_or_pipe(copy_term_size=fd_pipes_orig[1]) - - fcntl.fcntl(master_fd, fcntl.F_SETFL, - fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - - fd_pipes[0] = fd_pipes_orig[0] - fd_pipes[1] = slave_fd - fd_pipes[2] = slave_fd - else: - # Create a dummy pipe so the scheduler can monitor - # the process from inside a poll() loop. - master_fd, slave_fd = os.pipe() - fcntl.fcntl(master_fd, fcntl.F_SETFL, - fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - fd_pipes[self._dummy_pipe_fd] = slave_fd - if self.background: - fd_pipes[1] = slave_fd - fd_pipes[2] = slave_fd - - retval = portage.doebuild(ebuild_path, self.phase, + rval = portage.doebuild(ebuild_path, self.phase, root_config.root, settings, debug, - mydbapi=mydbapi, tree=tree, - fd_pipes=fd_pipes, returnpid=True) - - os.close(slave_fd) - - if isinstance(retval, int): - # doebuild failed before spawning - os.close(master_fd) - self.returncode = retval - self.wait() - return - - self.pid = retval[0] - portage.process.spawned_pids.remove(self.pid) - - if logfile: - files.log = open(logfile, 'a') - if not self.background: - files.stdout = os.fdopen(os.dup(fd_pipes_orig[1]), 'w') - output_handler = self._output_handler - else: - output_handler = self._dummy_handler - - files.ebuild = os.fdopen(master_fd, 'r') - self._reg_id = self.scheduler.register(files.ebuild.fileno(), - PollConstants.POLLIN, output_handler) - self._registered = True - - def _output_handler(self, fd, event): - files = self._files - buf = array.array('B') - try: - buf.fromfile(files.ebuild, self._bufsize) - except EOFError: - pass - if buf: - if not self.background: - buf.tofile(files.stdout) - files.stdout.flush() - buf.tofile(files.log) - files.log.flush() - else: - self._unregister() - self.wait() - return self._registered + mydbapi=mydbapi, tree=tree, **kwargs) - def _dummy_handler(self, fd, event): - """ - This method is mainly interested in detecting EOF, since - the only purpose of the pipe is to allow the scheduler to - monitor the process from inside a poll() loop. - """ - files = self._files - buf = array.array('B') - try: - buf.fromfile(files.ebuild, self._bufsize) - except EOFError: - pass - if buf: - pass - else: - self._unregister() - self.wait() - return self._registered + return rval def _set_returncode(self, wait_retval): - SubProcess._set_returncode(self, wait_retval) - - settings = self.settings - debug = settings.get("PORTAGE_DEBUG") == "1" - log_path = settings.get("PORTAGE_LOG_FILE") + SpawnProcess._set_returncode(self, wait_retval) if self.phase != "clean": self.returncode = portage._doebuild_exit_status_check_and_log( - settings, self.phase, self.returncode) + self.settings, self.phase, self.returncode) - portage._post_phase_userpriv_perms(settings) + portage._post_phase_userpriv_perms(self.settings) class EbuildPhase(CompositeTask): @@ -2975,6 +2886,20 @@ class Binpkg(CompositeTask): ("_bintree", "_build_dir", "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") + def _writemsg_level(self, msg, level=0, noiselevel=0): + + if not self.background: + portage.util.writemsg_level(msg, + level=level, noiselevel=noiselevel) + + log_path = self.settings.get("PORTAGE_LOG_FILE") + if log_path is not None: + f = open(log_path, 'a') + try: + f.write(msg) + finally: + f.close() + def _start(self): pkg = self.pkg @@ -3071,8 +2996,15 @@ class Binpkg(CompositeTask): verifier = None if self._verify: - verifier = BinpkgVerifier(background=self.background, pkg=self.pkg) - self._start_task(verifier, self._verifier_exit) + verifier = BinpkgVerifier(background=self.background, + logfile=self.scheduler.fetch.log_file, pkg=self.pkg) + + if self.background: + verifier.addExitListener(self._verifier_exit) + self._current_task = verifier + self.scheduler.fetch.schedule(verifier) + else: + self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) @@ -3133,10 +3065,9 @@ class Binpkg(CompositeTask): portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid, gid=portage.data.portage_gid, mode=dir_mode) - portage.writemsg_stdout(">>> Extracting info\n") - # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) + self._writemsg_level(">>> Extracting info\n") pkg_xpak = portage.xpak.tbz2(self._pkg_path) check_missing_metadata = ("CATEGORY", "PF") @@ -3191,7 +3122,7 @@ class Binpkg(CompositeTask): extractor = BinpkgExtractorAsync(background=self.background, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, scheduler=self.scheduler) - portage.writemsg_stdout(">>> Extracting %s\n" % self.pkg.cpv) + self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) self._start_task(extractor, self._extractor_exit) def _extractor_exit(self, extractor): @@ -3327,7 +3258,7 @@ class BinpkgFetcher(SpawnProcess): self.locked = False class BinpkgVerifier(AsynchronousTask): - __slots__ = ("pkg",) + __slots__ = ("logfile", "pkg",) def _start(self): """ @@ -3340,24 +3271,38 @@ class BinpkgVerifier(AsynchronousTask): root_config = pkg.root_config bintree = root_config.trees["bintree"] rval = os.EX_OK + stdout_orig = sys.stdout + stderr_orig = sys.stderr + log_file = None + if self.background and self.logfile is not None: + log_file = open(self.logfile, 'a') try: - bintree.digestCheck(pkg) - except portage.exception.FileNotFound: - writemsg("!!! Fetching Binary failed " + \ - "for '%s'\n" % pkg.cpv, noiselevel=-1) - rval = 1 - except portage.exception.DigestException, e: - writemsg("\n!!! Digest verification failed:\n", - noiselevel=-1) - writemsg("!!! %s\n" % e.value[0], - noiselevel=-1) - writemsg("!!! Reason: %s\n" % e.value[1], - noiselevel=-1) - writemsg("!!! Got: %s\n" % e.value[2], - noiselevel=-1) - writemsg("!!! Expected: %s\n" % e.value[3], - noiselevel=-1) - rval = 1 + if log_file is not None: + sys.stdout = log_file + sys.stderr = log_file + try: + bintree.digestCheck(pkg) + except portage.exception.FileNotFound: + writemsg("!!! Fetching Binary failed " + \ + "for '%s'\n" % pkg.cpv, noiselevel=-1) + rval = 1 + except portage.exception.DigestException, e: + writemsg("\n!!! Digest verification failed:\n", + noiselevel=-1) + writemsg("!!! %s\n" % e.value[0], + noiselevel=-1) + writemsg("!!! Reason: %s\n" % e.value[1], + noiselevel=-1) + writemsg("!!! Got: %s\n" % e.value[2], + noiselevel=-1) + writemsg("!!! Expected: %s\n" % e.value[3], + noiselevel=-1) + rval = 1 + finally: + sys.stdout = stdout_orig + sys.stderr = stderr_orig + if log_file is not None: + log_file.close() self.returncode = rval self.wait() @@ -3413,8 +3358,11 @@ class MergeListItem(CompositeTask): ldpath_mtimes = mtimedb["ldpath"] if not build_opts.pretend: + extra_newline = "\n" + if self.background: + extra_newline = "" portage.writemsg_stdout( - "\n>>> Emerging (%s of %s) %s to %s\n" % \ + extra_newline + ">>> Emerging (%s of %s) %s to %s\n" % \ (colorize("MERGE_LIST_PROGRESS", str(pkg_count.curval)), colorize("MERGE_LIST_PROGRESS", str(pkg_count.maxval)), colorize("GOOD", pkg.cpv), pkg.root), noiselevel=-1) @@ -8119,6 +8067,7 @@ class SequentialTaskQueue(SlotObject): running_tasks = self.running_tasks while running_tasks: task = running_tasks.pop() + task.removeExitListener(self._task_exit) task.cancel() def __nonzero__(self): @@ -8395,6 +8344,87 @@ class TaskScheduler(object): def run(self): self._scheduler.schedule() +class JobStatusDisplay(object): + + _bound_properties = ("curval", "merges", "running") + _msg_template = "Jobs: %(curval)s of %(maxval)s complete, " + \ + "%(running)s running, %(merges)s merge%(merges_plural)s, " + \ + "load average: %(load_avg)s" + + def __init__(self, quiet=False): + object.__setattr__(self, "quiet", quiet) + object.__setattr__(self, "maxval", 0) + object.__setattr__(self, "_changed", False) + self.reset() + + def reset(self): + self.maxval = 0 + for name in self._bound_properties: + object.__setattr__(self, name, 0) + + def __setattr__(self, name, value): + old_value = getattr(self, name) + if value == old_value: + return + object.__setattr__(self, name, value) + if name in self._bound_properties: + self._property_change(name, old_value, value) + + def _property_change(self, name, old_value, new_value): + self._changed = True + + def _load_avg_str(self, digits=1): + try: + avg = os.getloadavg() + except OSError, e: + return str(e) + return ", ".join(fpformat.fix(x, digits) for x in avg) + + def display(self): + """ + Display status on stdout, but only if something has + changed since the last call. + """ + + if self.quiet: + return + if not self._changed: + return + self._changed = False + + # Don't use len(self._completed_tasks) here since that also + # can include uninstall tasks. + curval_str = str(self.curval) + maxval_str = str(self.maxval) + running_str = str(self.running) + merges_str = str(self.merges) + load_avg_str = self._load_avg_str() + if self.merges == 1: + merges_plural = "" + else: + merges_plural = "s" + + msg = self._msg_template % { + "curval" : colorize("INFORM", curval_str), + "maxval" : colorize("INFORM", maxval_str), + "running" : colorize("INFORM", running_str), + "merges" : colorize("INFORM", merges_str), + "merges_plural" : merges_plural, + "load_avg" : load_avg_str, + } + portage.writemsg_stdout(">>> %s\n" % (msg,), noiselevel=-1) + + xterm_msg = self._msg_template % { + "curval" : curval_str, + "maxval" : maxval_str, + "running" : running_str, + "merges" : merges_str, + "merges_plural" : merges_plural, + "load_avg" : load_avg_str, + } + + xtermTitle(xterm_msg) + class Scheduler(PollScheduler): _opts_ignore_blockers = \ @@ -8491,9 +8521,9 @@ class Scheduler(PollScheduler): self._prefetchers = weakref.WeakValueDictionary() self._pkg_queue = [] self._completed_tasks = set() - # Number of completed package tasks, excluding uninstalls. - self._completed_pkg_count = 0 - self._summary_prev_pkg_count = 0 + + self._status_display = JobStatusDisplay() + self._failed_pkgs = [] self._failed_pkgs_all = [] self._failed_pkgs_die_msgs = [] @@ -8503,6 +8533,7 @@ class Scheduler(PollScheduler): if isinstance(x, Package) and x.operation == "merge"]) self._pkg_count = self._pkg_count_class( curval=0, maxval=merge_count) + self._status_display.maxval = self._pkg_count.maxval max_jobs = myopts.get("--jobs") if max_jobs is None: @@ -8672,22 +8703,17 @@ class Scheduler(PollScheduler): finally: f.close() - def _dblink_display_merge(self, pkg_dblink, msg, level=0): + def _dblink_display_merge(self, pkg_dblink, msg, level=0, noiselevel=0): log_path = pkg_dblink.settings.get("PORTAGE_LOG_FILE") background = self._max_jobs > 1 - if level >= logging.WARNING: - noiselevel = -1 - msg_func = writemsg - else: - noiselevel = 0 - msg_func = portage.writemsg_stdout - if log_path is None: - msg_func(msg, noiselevel=noiselevel) + portage.util.writemsg_level(msg, + level=level, noiselevel=noiselevel) else: if not background: - msg_func(msg, noiselevel=noiselevel) + portage.util.writemsg_level(msg, + level=level, noiselevel=noiselevel) self._append_to_log_path(log_path, msg) def _dblink_ebuild_phase(self, @@ -8706,7 +8732,8 @@ class Scheduler(PollScheduler): if phase == "preinst": msg = ">>> Merging %s to %s\n" % (pkg.cpv, pkg.root) - portage.writemsg_stdout(msg) + if not background: + portage.writemsg_stdout(msg) if log_path is not None: self._append_to_log_path(log_path, msg) @@ -8719,7 +8746,8 @@ class Scheduler(PollScheduler): if phase == "postinst" and \ ebuild_phase.returncode == os.EX_OK: msg = ">>> %s %s\n" % (pkg.cpv, "merged.") - portage.writemsg_stdout(msg) + if not background: + portage.writemsg_stdout(msg) if log_path is not None: self._append_to_log_path(log_path, msg) @@ -8970,6 +8998,7 @@ class Scheduler(PollScheduler): self._pkg_count.curval = 0 self._pkg_count.maxval = len([x for x in self._mergelist \ if isinstance(x, Package) and x.operation == "merge"]) + self._status_display.maxval = self._pkg_count.maxval self._logger.log(" *** Finished. Cleaning up...") @@ -9032,6 +9061,10 @@ class Scheduler(PollScheduler): self._do_merge_exit(merge) self._deallocate_config(merge.merge.settings) self._schedule() + if merge.returncode == os.EX_OK and \ + not merge.merge.pkg.installed: + self._status_display.curval += 1 + self._status_display.merges = len(self._task_queues.merge) def _do_merge_exit(self, merge): pkg = merge.merge.pkg @@ -9061,7 +9094,6 @@ class Scheduler(PollScheduler): if not mtimedb["resume"]["mergelist"]: del mtimedb["resume"] mtimedb.commit() - self._completed_pkg_count += 1 def _build_exit(self, build): if build.returncode == os.EX_OK: @@ -9069,10 +9101,13 @@ class Scheduler(PollScheduler): merge = PackageMerge(merge=build) merge.addExitListener(self._merge_exit) self._task_queues.merge.add(merge) + self._status_display.merges = len(self._task_queues.merge) else: self._failed_pkgs.append((build.pkg, build.returncode)) self._deallocate_config(build.settings) self._jobs -= 1 + self._status_display.running = self._jobs + self._status_display.display() self._schedule() def _extract_exit(self, build): @@ -9100,8 +9135,7 @@ class Scheduler(PollScheduler): def _main_loop_cleanup(self): del self._pkg_queue[:] self._completed_tasks.clear() - self._completed_pkg_count = 0 - self._summary_prev_pkg_count = 0 + self._status_display.reset() self._digraph = None self._task_queues.fetch.clear() @@ -9182,11 +9216,8 @@ class Scheduler(PollScheduler): def _schedule_tasks(self): remaining, state_change = self._schedule_tasks_imp() - - if state_change or \ - self._summary_prev_pkg_count != self._completed_pkg_count: - self._display_status() - self._summary_prev_pkg_count = self._completed_pkg_count + if state_change: + self._status_display.display() return remaining def _schedule_tasks_imp(self): @@ -9198,6 +9229,11 @@ class Scheduler(PollScheduler): task_queues = self._task_queues background = self._max_jobs > 1 self._logger.parallel = background + self._status_display.quiet = \ + not background or \ + ("--quiet" in self.myopts and \ + "--verbose" not in self.myopts) + state_change = 0 while self._can_add_job(): @@ -9222,48 +9258,16 @@ class Scheduler(PollScheduler): task_queues.merge.add(merge) elif pkg.built: self._jobs += 1 + self._status_display.running = self._jobs task.addExitListener(self._extract_exit) task_queues.jobs.add(task) else: self._jobs += 1 + self._status_display.running = self._jobs task.addExitListener(self._build_exit) task_queues.jobs.add(task) return (True, state_change) - def _load_avg_str(self, digits=1): - try: - avg = os.getloadavg() - except OSError, e: - return str(e) - return ", ".join(fpformat.fix(x, digits) for x in avg) - - def _display_status(self): - if self._max_jobs < 2: - return - - # Don't use len(self._completed_tasks) here since that also - # can include uninstall tasks. - completed_str = str(self._completed_pkg_count) - maxval_str = str(self._pkg_count.maxval) - jobs_str = str(self._jobs) - merges_str = str(len(self._task_queues.merge)) - load_avg_str = self._load_avg_str() - - msg = ("Jobs: %s of %s complete, %s running, %s merges, " + \ - "load average: %s") % \ - (colorize("INFORM", completed_str), colorize("INFORM", maxval_str), - colorize("INFORM", jobs_str), colorize("INFORM", merges_str), - load_avg_str) - noiselevel = 0 - if "--verbose" in self.myopts: - noiselevel = -1 - portage.writemsg_stdout(">>> %s\n" % msg, noiselevel=noiselevel) - - short_msg = ("Jobs: %s of %s complete, %s running, %s merges, " + \ - "load average: %s") % \ - (completed_str, maxval_str, jobs_str, merges_str, load_avg_str) - xtermTitle(short_msg) - def _task(self, pkg, background): pkg_to_replace = None diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 69f496e66..8aae5b44d 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -541,11 +541,10 @@ class digraph(object): print " ",child, print "(%s)" % self.nodes[node][0][child] - #parse /etc/env.d and generate /etc/profile.env def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, - env=None): + env=None, writemsg_level=portage.util.writemsg_level): if target_root is None: global settings target_root = settings["ROOT"] @@ -752,13 +751,14 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, # an older package installed ON TOP of a newer version will cause ldconfig # to overwrite the symlinks we just made. -X means no links. After 'clean' # we can safely create links. - writemsg(">>> Regenerating %s/etc/ld.so.cache...\n" % (target_root + EPREFIX_LSTRIP)) + writemsg_level(">>> Regenerating %s/etc/ld.so.cache...\n" % \ + (target_root + EPREFIX_LSTRIP,)) if makelinks: os.system("cd / ; %s -r '%s'" % (ldconfig, target_root)) else: os.system("cd / ; %s -X -r '%s'" % (ldconfig, target_root)) elif ostype in ("FreeBSD","DragonFly"): - writemsg(">>> Regenerating %svar/run/ld-elf.so.hints...\n" % \ + writemsg_level(">>> Regenerating %svar/run/ld-elf.so.hints...\n" % \ target_root + EPREFIX_LSTRIP) os.system(("cd / ; %s -elf -i " + \ "-f '%svar/run/ld-elf.so.hints' '%setc/ld.so.conf'") % \ @@ -3021,10 +3021,6 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero env=mysettings.environ() keywords["opt_name"]="[%s]" % mysettings["PF"] - if keywords.get("returnpid"): - # emerge handles logging externally - keywords.pop("logfile", None) - fd_pipes = keywords.get("fd_pipes") if fd_pipes is None: fd_pipes = { @@ -5294,8 +5290,12 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if mystatus: return mystatus have_build_dirs = True - # PORTAGE_LOG_FILE is set above by the prepare_build_dirs() call. - logfile = mysettings.get("PORTAGE_LOG_FILE") + + # emerge handles logging externally + if not returnpid: + # PORTAGE_LOG_FILE is set by the + # above prepare_build_dirs() call. + logfile = mysettings.get("PORTAGE_LOG_FILE") if have_build_dirs: env_file = os.path.join(mysettings["T"], "environment") @@ -6894,9 +6894,13 @@ def deprecated_profile_check(): deprecatedfile = open(DEPRECATED_PROFILE_FILE, "r") dcontent = deprecatedfile.readlines() deprecatedfile.close() - newprofile = dcontent[0] writemsg(red("\n!!! Your current profile is deprecated and not supported anymore.\n"), noiselevel=-1) + if not dcontent: + writemsg(red("!!! Please refer to the Gentoo Upgrading Guide.\n"), + noiselevel=-1) + return True + newprofile = dcontent[0] writemsg(red("!!! Please upgrade to the following profile if possible:\n"), noiselevel=-1) writemsg(8*" "+green(newprofile)+"\n", noiselevel=-1) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index a42bacf2c..aa3e9a013 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -15,7 +15,8 @@ from portage.locks import lockdir, unlockdir from portage.output import bold, red, green from portage.update import fixdbentries from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, \ - writemsg, writemsg_stdout, write_atomic, atomic_ofstream, writedict, \ + writemsg, writemsg_stdout, writemsg_level, \ + write_atomic, atomic_ofstream, writedict, \ grabfile, grabdict, normalize_path, new_protect_filename, getlibpaths from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp @@ -1676,6 +1677,7 @@ class dblink(object): # name of the dir; the package may have been moved. myebuildpath = None ebuild_phase = "prerm" + log_path = None mystuff = os.listdir(self.dbdir) for x in mystuff: if x.endswith(".ebuild"): @@ -1719,6 +1721,7 @@ class dblink(object): catdir_lock = None prepare_build_dirs(self.myroot, self.settings, 1) + log_path = self.settings.get("PORTAGE_LOG_FILE") if scheduler is None: retval = doebuild(myebuildpath, ebuild_phase, self.myroot, @@ -1875,22 +1878,28 @@ class dblink(object): raise del e unlockdir(catdir_lock) + + if log_path is not None and os.path.exists(log_path): + # Restore this since it gets lost somewhere above and it + # needs to be set for _display_merge() to be able to log. + # Note that the log isn't necessarily supposed to exist + # since if PORT_LOGDIR is unset then it's a temp file + # so it gets cleaned above. + self.settings["PORTAGE_LOG_FILE"] = log_path + else: + self.settings.pop("PORTAGE_LOG_FILE", None) + env_update(target_root=self.myroot, prev_mtimes=ldpath_mtimes, - contents=contents, env=self.settings.environ()) + contents=contents, env=self.settings.environ(), + writemsg_level=self._display_merge) return os.EX_OK - def _display_merge(self, msg, level=0): + def _display_merge(self, msg, level=0, noiselevel=0): if self._scheduler is not None: - self._scheduler.dblinkDisplayMerge(self, msg, level=level) + self._scheduler.dblinkDisplayMerge(self, msg, + level=level, noiselevel=noiselevel) return - - if level >= logging.WARNING: - noiselevel = -1 - msg_func = writemsg - else: - noiselevel = 0 - msg_func = writemsg_stdout - msg_func(msg, noiselevel=noiselevel) + writemsg_level(msg, level=level, noiselevel=noiselevel) def _unmerge_pkgfiles(self, pkgfiles, others_in_slot): """ @@ -2457,12 +2466,15 @@ class dblink(object): from portage.output import colorize prefix = colorize("SECURITY_WARN", "*") + " WARNING: " showMessage(prefix + "suid/sgid file(s) " + \ - "with suspicious hardlink(s):\n", level=logging.ERROR) + "with suspicious hardlink(s):\n", + level=logging.ERROR, noiselevel=-1) for path_list in suspicious_hardlinks: for path, s in path_list: - showMessage(prefix + " '%s'\n" % path, level=logging.ERROR) + showMessage(prefix + " '%s'\n" % path, + level=logging.ERROR, noiselevel=-1) showMessage(prefix + "See the Gentoo Security Handbook " + \ - "guide for advice on how to proceed.\n", level=logging.ERROR) + "guide for advice on how to proceed.\n", + level=logging.ERROR, noiselevel=-1) return 1 def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0, @@ -2508,7 +2520,7 @@ class dblink(object): if not os.path.isdir(srcroot): showMessage("!!! Directory Not Found: D='%s'\n" % srcroot, - level=logging.ERROR) + level=logging.ERROR, noiselevel=-1) return 1 inforoot_slot_file = os.path.join(inforoot, "SLOT") @@ -2762,7 +2774,7 @@ class dblink(object): # XXX: Decide how to handle failures here. if a != os.EX_OK: showMessage("!!! FAILED preinst: "+str(a)+"\n", - level=logging.ERROR) + level=logging.ERROR, noiselevel=-1) return a # copy "info" files (like SLOT, CFLAGS, etc.) into the database @@ -2885,7 +2897,7 @@ class dblink(object): showMessage(colorize("WARN", "WARNING:") + " AUTOCLEAN is disabled. This can cause serious" + " problems due to overlapping packages.\n", - level=logging.WARN) + level=logging.WARN, noiselevel=-1) # We hold both directory locks. self.dbdir = self.dbpkgdir @@ -2928,7 +2940,7 @@ class dblink(object): # XXX: Decide how to handle failures here. if a != os.EX_OK: showMessage("!!! FAILED postinst: "+str(a)+"\n", - level=logging.ERROR) + level=logging.ERROR, noiselevel=-1) return a downgrade = False @@ -2939,7 +2951,8 @@ class dblink(object): #update environment settings, library paths. DO NOT change symlinks. env_update(makelinks=(not downgrade), target_root=self.settings["ROOT"], prev_mtimes=prev_mtimes, - contents=contents, env=self.settings.environ()) + contents=contents, env=self.settings.environ(), + writemsg_level=self._display_merge) return os.EX_OK @@ -3150,7 +3163,8 @@ class dblink(object): if stat.S_ISDIR(mydmode): # install of destination is blocked by an existing directory with the same name moveme = 0 - showMessage("!!! %s\n" % mydest, level=logging.ERROR) + showMessage("!!! %s\n" % mydest, + level=logging.ERROR, noiselevel=-1) elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])): cfgprot = 0 # install of destination is blocked by an existing regular file, diff --git a/pym/portage/util.py b/pym/portage/util.py index 58dde032e..998878ce6 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -51,6 +51,26 @@ def writemsg_stdout(mystr,noiselevel=0): """Prints messages stdout based on the noiselimit setting""" writemsg(mystr, noiselevel=noiselevel, fd=sys.stdout) +def writemsg_level(msg, level=0, noiselevel=0): + """ + Show a message for the given level as defined by the logging module + (default is 0). When level >= logging.WARNING then the message is + sent to stderr, otherwise it is sent to stdout. The noiselevel is + passed directly to writemsg(). + + @type msg: str + @param msg: a message string, including newline if appropriate + @type level: int + @param level: a numeric logging level (see the logging module) + @type noiselevel: int + @param noiselevel: passed directly to writemsg + """ + if level >= logging.WARNING: + fd = sys.stderr + else: + fd = sys.stdout + writemsg(msg, noiselevel=noiselevel, fd=fd) + def normalize_path(mypath): """ os.path.normpath("//foo") returns "//foo" instead of "/foo" -- 2.26.2