Merged from trunk 11060:11076
authorFabian Groffen <grobian@gentoo.org>
Wed, 16 Jul 2008 11:11:49 +0000 (11:11 -0000)
committerFabian Groffen <grobian@gentoo.org>
Wed, 16 Jul 2008 11:11:49 +0000 (11:11 -0000)
   | 11061   | In Scheduler._restart_if_necessary(), use list.remove()      |
   | zmedico | instead of assuming that portage is first in the resume      |
   |         | list.                                                        |

   | 11062   | Make EbuildFetcher copy FETCHCOMMAND and RESUMECOMMAND to    |
   | zmedico | the fetch environment, in case the default values have been  |
   |         | overridden.                                                  |

   | 11063   | Add some cross references between emerge's new parallel      |
   | zmedico | build options and MAKEOPTS.                                  |

   | 11064   | When --jobs > 1, display a status message each time that     |
   | zmedico | Scheduler._schedule_tasks() is called. For example: >>>      |
   |         | Jobs: 2 running, 0 merges, load average: 0.34, 0.46, 0.27    |

   | 11065   | Redirect dblink.unmerge() output to the log file when --jobs |
   | zmedico | > 1.                                                         |

   | 11066   | Adapt the dblink merge display callback to use log levels    |
   | zmedico | from the logging module so that it can be used as an         |
   |         | interface for both writemsg() and writemsg_stdout().         |

   | 11067   | Make dblink._display_merge() detect scheduler presence       |
   | zmedico | internally.                                                  |

   | 11068   | Redirect dblink.merge() output to the log file when --jobs > |
   | zmedico | 1.                                                           |

   | 11069   | Pass the whole dblink instance into                          |
   | zmedico | Scheduler._dblink_display_merge() in case it's ever needed.  |

   | 11070   | Show LDFLAGS in emerge --info output. Thanks to dirtyepic    |
   | zmedico | for this patch.                                              |

   | 11071   | Remove the config._filter_calling_env attribute since it's   |
   | zmedico | not really needed. Replace it with an simple                 |
   |         | os.path.exists() call inside config.environ().               |

   | 11072   | Add async support for calling functions in                   |
   | zmedico | misc-functions.sh: * Create a new MiscFunctionsProcess class |
   |         | derived from SpawnProcess. * Add "background" support to the |
   |         | SpawnProcess class and split out a _spawn() template         |
   |         | function so MiscFunctionsProcess can override it. * Rename   |
   |         | the existing EbuildPhase class to EbuildProcess. * Create a  |
   |         | new EbuildPhase class derived from CompositeTask. This class |
   |         | spawns the EbuildProcess and MiscFunctionsProcess classes as |
   |         | sub-tasks.                                                   |

   | 11073   | * Add xterm titles support to Scheduler._display_status(). * |
   | zmedico | Tweak emergelog() so that it won't interfere with the        |
   |         | scheduler summary.                                           |

   | 11075   | Add a progress counter showing "x of y complete" to          |
   | zmedico | Scheduler._display_status().                                 |

   | 11076   | Pass background=True into prefetcher constructors.           |
   | zmedico |                                                              |

svn path=/main/branches/prefix/; revision=11077

man/emerge.1
man/make.conf.5
pym/_emerge/__init__.py
pym/_emerge/help.py
pym/portage/__init__.py
pym/portage/dbapi/vartree.py

index 226d28a31db9410bdab66f34bbaa0894b2c5f58d..b928ab53ff30236218329ef7065703f078459965 100644 (file)
@@ -335,6 +335,10 @@ the related \fB\-\-skipfirst\fR option.
 .BR \-\-load\-average=LOAD
 Specifies that no new builds should be started if there are other builds
 running and the load average is at least LOAD (a floating-point number).
+This option is recommended for use in combination with \fB\-\-jobs\fR in
+order to avoid excess load. See \fBmake\fR(1) for information about
+analogous options that should be configured via \fBMAKEOPTS\fR in
+\fBmake.conf\fR(5).
 .TP
 .BR "\-\-newuse " (\fB\-N\fR)
 Tells emerge to include installed packages where USE flags have changed since 
index 1a32e46f6a68174539fbad8915049cdd7d6bcf4d..f05ed33e8ef7e9bc0487c07d2f4e65c8400b0b15 100644 (file)
@@ -378,8 +378,10 @@ package.
 Use this variable if you want to use parallel make.  For example, if you
 have a dual\-processor system, set this variable to "\-j2" or "\-j3" for 
 enhanced build performance with many packages. Suggested settings are
-between \fICPUs+1\fR and \fI2*CPUs+1\fR.
-For more information, see \fBmake\fR(1).
+between \fICPUs+1\fR and \fI2*CPUs+1\fR. In order to avoid
+excess load, the \fB\-\-load\-average\fR option is recommended.
+For more information, see \fBmake\fR(1). Also see \fBemerge\fR(1) for
+information about analogous \fB\-\-jobs\fR and \fB\-\-load\-average\fR options.
 .TP
 \fBNOCOLOR\fR = \fI["true" | "false"]\fR
 Defines if color should be disabled by default.
index 08056e4ebf884e3df0bb10f514c50183f4b34406..c1c481597b6dfb851c1e4831f887eeb39a4be0a8 100644 (file)
@@ -23,6 +23,8 @@ except KeyboardInterrupt:
 import array
 from collections import deque
 import fcntl
+import fpformat
+import logging
 import select
 import shlex
 import shutil
@@ -259,9 +261,7 @@ shortmapping={
 }
 
 def emergelog(xterm_titles, mystr, short_msg=None):
-       if xterm_titles:
-               if short_msg == None:
-                       short_msg = mystr
+       if xterm_titles and short_msg:
                if "HOSTNAME" in os.environ:
                        short_msg = os.environ["HOSTNAME"]+": "+short_msg
                xtermTitle(short_msg)
@@ -2041,7 +2041,7 @@ class SpawnProcess(SubProcess):
        __slots__ = ("args",) + \
                _spawn_kwarg_names
 
-       _file_names = ("process", "out")
+       _file_names = ("log", "process", "stdout")
        _files_dict = slot_dict_class(_file_names, prefix="")
        _bufsize = 4096
 
@@ -2072,6 +2072,7 @@ class SpawnProcess(SubProcess):
                fcntl.fcntl(master_fd, fcntl.F_SETFL,
                        fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
 
+               files.process = os.fdopen(master_fd, 'r')
                if logfile is not None:
 
                        fd_pipes_orig = fd_pipes.copy()
@@ -2079,11 +2080,14 @@ class SpawnProcess(SubProcess):
                        fd_pipes[1] = slave_fd
                        fd_pipes[2] = slave_fd
 
-                       files.out = open(logfile, "a")
+                       files.log = open(logfile, "a")
                        portage.util.apply_secpass_permissions(logfile,
                                uid=portage.portage_uid, gid=portage.portage_gid,
                                mode=0660)
 
+                       if not self.background:
+                               files.stdout = os.fdopen(os.dup(fd_pipes_orig[1]), 'w')
+
                        output_handler = self._output_handler
 
                else:
@@ -2106,13 +2110,15 @@ class SpawnProcess(SubProcess):
                kwargs["returnpid"] = True
                kwargs.pop("logfile", None)
 
-               retval = portage.process.spawn(self.args, **kwargs)
+               retval = self._spawn(self.args, **kwargs)
 
                os.close(slave_fd)
 
                if isinstance(retval, int):
                        # spawn failed
                        os.close(master_fd)
+                       for f in self.files.values():
+                               f.close()
                        self.returncode = retval
                        self.wait()
                        return
@@ -2120,12 +2126,13 @@ class SpawnProcess(SubProcess):
                self.pid = retval[0]
                portage.process.spawned_pids.remove(self.pid)
 
-               
-               files.process = os.fdopen(master_fd, 'r')
                self._reg_id = self.scheduler.register(files.process.fileno(),
                        PollConstants.POLLIN, output_handler)
                self._registered = True
 
+       def _spawn(self, args, **kwargs):
+               return portage.process.spawn(args, **kwargs)
+
        def _output_handler(self, fd, event):
                files = self._files
                buf = array.array('B')
@@ -2134,8 +2141,11 @@ class SpawnProcess(SubProcess):
                except EOFError:
                        pass
                if buf:
-                       buf.tofile(files.out)
-                       files.out.flush()
+                       if not self.background:
+                               buf.tofile(files.stdout)
+                               files.stdout.flush()
+                       buf.tofile(files.log)
+                       files.log.flush()
                else:
                        self._unregister()
                        self.wait()
@@ -2160,10 +2170,44 @@ class SpawnProcess(SubProcess):
                        self.wait()
                return self._registered
 
+class MiscFunctionsProcess(SpawnProcess):
+       """
+       Spawns misc-functions.sh with an existing ebuild environment.
+       """
+
+       __slots__ = ("commands", "phase", "pkg", "settings")
+
+       def _start(self):
+               settings = self.settings
+               portage_bin_path = settings["PORTAGE_BIN_PATH"]
+               misc_sh_binary = os.path.join(portage_bin_path,
+                       os.path.basename(portage.const.MISC_SH_BINARY))
+
+               self.args = [portage._shell_quote(misc_sh_binary)] + self.commands
+               self.logfile = settings.get("PORTAGE_LOG_FILE")
+
+               portage._doebuild_exit_status_unlink(
+                       settings.get("EBUILD_EXIT_STATUS_FILE"))
+
+               SpawnProcess._start(self)
+
+       def _spawn(self, args, **kwargs):
+               settings = self.settings
+               debug = settings.get("PORTAGE_DEBUG") == "1"
+               return portage.spawn(" ".join(args), settings,
+                       debug=debug, **kwargs)
+
+       def _set_returncode(self, wait_retval):
+               SpawnProcess._set_returncode(self, wait_retval)
+               self.returncode = portage._doebuild_exit_status_check_and_log(
+                       self.settings, self.phase, self.returncode)
+
 class EbuildFetcher(SpawnProcess):
 
        __slots__ = ("fetchonly", "pkg",)
 
+       _env_vars = ("FETCHCOMMAND", "RESUMECOMMAND")
+
        def _start(self):
 
                root_config = self.pkg.root_config
@@ -2173,6 +2217,10 @@ class EbuildFetcher(SpawnProcess):
 
                fetch_env = settings.environ()
                fetch_env["PORTAGE_NICENESS"] = "0"
+               for k in self._env_vars:
+                       v = settings.get(k)
+                       if v is not None:
+                               fetch_env[k] = v
                if self.fetchonly:
                        fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
 
@@ -2528,17 +2576,10 @@ class EbuildExecuter(CompositeTask):
                # This initializes PORTAGE_LOG_FILE.
                portage.prepare_build_dirs(pkg.root, settings, cleanup)
 
-               fd_pipes = {
-                       0 : sys.stdin.fileno(),
-                       1 : sys.stdout.fileno(),
-                       2 : sys.stderr.fileno(),
-               }
-
                ebuild_phases = TaskSequence(scheduler=scheduler)
 
                for phase in self._phases:
                        ebuild_phases.add(EbuildPhase(background=self.background,
-                               fd_pipes=fd_pipes,
                                pkg=pkg, phase=phase, scheduler=scheduler,
                                settings=settings, tree=tree))
 
@@ -2632,7 +2673,7 @@ class EbuildMetadataPhase(SubProcess):
 
                return self._registered
 
-class EbuildPhase(SubProcess):
+class EbuildProcess(SubProcess):
 
        __slots__ = ("fd_pipes", "phase", "pkg",
                "settings", "tree")
@@ -2777,41 +2818,57 @@ class EbuildPhase(SubProcess):
                                settings, self.phase, self.returncode)
 
                portage._post_phase_userpriv_perms(settings)
+
+class EbuildPhase(CompositeTask):
+
+       __slots__ = ("background", "pkg", "phase",
+               "scheduler", "settings", "tree")
+
+       _post_phase_cmds = portage._post_phase_cmds
+
+       def _start(self):
+
+               ebuild_process = EbuildProcess(background=self.background,
+                       pkg=self.pkg, phase=self.phase, scheduler=self.scheduler,
+                       settings=self.settings, tree=self.tree)
+
+               self._start_task(ebuild_process, self._ebuild_exit)
+
+       def _ebuild_exit(self, ebuild_process):
+
                if self.phase == "install":
-                       portage._check_build_log(settings)
-                       if self.returncode == os.EX_OK:
-                               self.returncode = portage._post_src_install_checks(settings)
+                       portage._check_build_log(self.settings)
 
-               elif self.phase == "preinst":
-                       if self.returncode == os.EX_OK:
-                               portage._doebuild_exit_status_unlink(
-                                       settings.get("EBUILD_EXIT_STATUS_FILE"))
-                               phase_retval = portage.spawn(
-                                       " ".join(portage._post_pkg_preinst_cmd(settings)),
-                                       settings, debug=debug, free=1, logfile=log_path)
-                               phase_retval = portage._doebuild_exit_status_check_and_log(
-                                       settings, self.phase, phase_retval)
-                               if phase_retval != os.EX_OK:
-                                       writemsg("!!! post preinst failed; exiting.\n",
-                                               noiselevel=-1)
-                                       self.returncode = phase_retval
+               if self._default_exit(ebuild_process) != os.EX_OK:
+                       self.wait()
+                       return
 
-               elif self.phase == "postinst":
+               settings = self.settings
 
-                       if self.returncode == os.EX_OK:
-                               portage._doebuild_exit_status_unlink(
-                                       settings.get("EBUILD_EXIT_STATUS_FILE"))
-                               phase_retval = portage.spawn(
-                                       " ".join(portage._post_pkg_postinst_cmd(settings)),
-                                       settings, debug=debug, free=1, logfile=log_path)
-                               phase_retval = portage._doebuild_exit_status_check_and_log(
-                                       settings, self.phase, phase_retval)
-                               if phase_retval != os.EX_OK:
-                                       writemsg("!!! post postinst failed; exiting.\n",
-                                               noiselevel=-1)
-                                       self.returncode = phase_retval
+               if self.phase == "install":
+                       portage._post_src_install_uid_fix(settings)
+
+               post_phase_cmds = self._post_phase_cmds.get(self.phase)
+               if post_phase_cmds is not None:
+                       post_phase = MiscFunctionsProcess(background=self.background,
+                               commands=post_phase_cmds, phase=self.phase, pkg=self.pkg,
+                               scheduler=self.scheduler, settings=settings)
+                       self._start_task(post_phase, self._post_phase_exit)
+                       return
 
-class EbuildBinpkg(EbuildPhase):
+               self.returncode = ebuild_process.returncode
+               self._current_task = None
+               self.wait()
+
+       def _post_phase_exit(self, post_phase):
+               if self._final_exit(post_phase) != os.EX_OK:
+                       writemsg("!!! post %s failed; exiting.\n" % self.phase,
+                               noiselevel=-1)
+               self._current_task = None
+               self.wait()
+               return
+
+class EbuildBinpkg(EbuildProcess):
        """
        This assumes that src_install() has successfully completed.
        """
@@ -2836,12 +2893,12 @@ class EbuildBinpkg(EbuildPhase):
                settings.backup_changes("PORTAGE_BINPKG_TMPFILE")
 
                try:
-                       EbuildPhase._start(self)
+                       EbuildProcess._start(self)
                finally:
                        settings.pop("PORTAGE_BINPKG_TMPFILE", None)
 
        def _set_returncode(self, wait_retval):
-               EbuildPhase._set_returncode(self, wait_retval)
+               EbuildProcess._set_returncode(self, wait_retval)
 
                pkg = self.pkg
                bintree = pkg.root_config.trees["bintree"]
@@ -8354,7 +8411,7 @@ class Scheduler(PollScheduler):
        _fetch_log = EPREFIX + "/var/log/emerge-fetch.log"
 
        class _iface_class(SlotObject):
-               __slots__ = ("dblinkEbuildPhase", "fetch",
+               __slots__ = ("dblinkEbuildPhase", "dblinkDisplayMerge", "fetch",
                        "register", "schedule", "unregister")
 
        class _fetch_iface_class(SlotObject):
@@ -8374,9 +8431,12 @@ class Scheduler(PollScheduler):
                __slots__ = ("curval", "maxval")
 
        class _emerge_log_class(SlotObject):
-               __slots__ = ("xterm_titles",)
+               __slots__ = ("parallel", "xterm_titles",)
 
                def log(self, *pargs, **kwargs):
+                       if self.parallel:
+                               # Avoid interference with the scheduler's status display.
+                               kwargs.pop("short_msg", None)
                        emergelog(self.xterm_titles, *pargs, **kwargs)
 
        def __init__(self, settings, trees, mtimedb, myopts,
@@ -8419,6 +8479,7 @@ class Scheduler(PollScheduler):
                        schedule=self._schedule_fetch)
                self._sched_iface = self._iface_class(
                        dblinkEbuildPhase=self._dblink_ebuild_phase,
+                       dblinkDisplayMerge=self._dblink_display_merge,
                        fetch=fetch_iface, register=self._register,
                        schedule=self._schedule_wait, unregister=self._unregister)
 
@@ -8430,6 +8491,9 @@ class Scheduler(PollScheduler):
                self._prefetchers = weakref.WeakValueDictionary()
                self._pkg_queue = []
                self._completed_tasks = set()
+               # Number of completed package tasks, excluding uninstalls.
+               self._completed_pkg_count = 0
+               self._summary_prev_pkg_count = 0
                self._failed_pkgs = []
                self._failed_pkgs_all = []
                self._failed_pkgs_die_msgs = []
@@ -8608,6 +8672,24 @@ class Scheduler(PollScheduler):
                finally:
                        f.close()
 
+       def _dblink_display_merge(self, pkg_dblink, msg, level=0):
+               log_path = pkg_dblink.settings.get("PORTAGE_LOG_FILE")
+               background = self._max_jobs > 1
+
+               if level >= logging.WARNING:
+                       noiselevel = -1
+                       msg_func = writemsg
+               else:
+                       noiselevel = 0
+                       msg_func = portage.writemsg_stdout
+
+               if log_path is None:
+                       msg_func(msg, noiselevel=noiselevel)
+               else:
+                       if not background:
+                               msg_func(msg, noiselevel=noiselevel)
+                       self._append_to_log_path(log_path, msg)
+
        def _dblink_ebuild_phase(self,
                pkg_dblink, pkg_dbapi, ebuild_path, phase):
                """
@@ -8706,16 +8788,17 @@ class Scheduler(PollScheduler):
 
                elif pkg.type_name == "ebuild":
 
-                       prefetcher = EbuildFetcher(fetchonly=1,
-                               logfile=self._fetch_log, pkg=pkg,
-                               scheduler=self._sched_iface)
+                       prefetcher = EbuildFetcher(background=True,
+                               fetchonly=1, logfile=self._fetch_log,
+                               pkg=pkg, scheduler=self._sched_iface)
 
                elif pkg.type_name == "binary" and \
                        "--getbinpkg" in self.myopts and \
                        pkg.root_config.trees["bintree"].isremote(pkg.cpv):
 
-                       prefetcher = BinpkgFetcher(logfile=self._fetch_log,
-                               pkg=pkg, scheduler=self._sched_iface)
+                       prefetcher = BinpkgFetcher(background=True,
+                               logfile=self._fetch_log, pkg=pkg,
+                               scheduler=self._sched_iface)
 
                return prefetcher
 
@@ -8802,7 +8885,7 @@ class Scheduler(PollScheduler):
                        "emerge via exec() after change of " + \
                        "portage version.")
 
-               del mtimedb["resume"]["mergelist"][0]
+               mtimedb["resume"]["mergelist"].remove(list(pkg))
                mtimedb.commit()
                portage.run_exitfuncs()
                mynewargv = [sys.argv[0], "--resume"]
@@ -8978,6 +9061,7 @@ class Scheduler(PollScheduler):
                if not mtimedb["resume"]["mergelist"]:
                        del mtimedb["resume"]
                mtimedb.commit()
+               self._completed_pkg_count += 1
 
        def _build_exit(self, build):
                if build.returncode == os.EX_OK:
@@ -9016,6 +9100,8 @@ class Scheduler(PollScheduler):
        def _main_loop_cleanup(self):
                del self._pkg_queue[:]
                self._completed_tasks.clear()
+               self._completed_pkg_count = 0
+               self._summary_prev_pkg_count = 0
                self._digraph = None
                self._task_queues.fetch.clear()
 
@@ -9095,6 +9181,15 @@ class Scheduler(PollScheduler):
                        self._poll_loop()
 
        def _schedule_tasks(self):
+               remaining, state_change = self._schedule_tasks_imp()
+
+               if state_change or \
+                       self._summary_prev_pkg_count != self._completed_pkg_count:
+                       self._display_status()
+                       self._summary_prev_pkg_count = self._completed_pkg_count
+               return remaining
+
+       def _schedule_tasks_imp(self):
                """
                @rtype: bool
                @returns: True if tasks remain to schedule, False otherwise.
@@ -9102,15 +9197,19 @@ class Scheduler(PollScheduler):
 
                task_queues = self._task_queues
                background = self._max_jobs > 1
+               self._logger.parallel = background
+               state_change = 0
 
                while self._can_add_job():
 
                        if not self._pkg_queue or self._failed_pkgs:
-                               return False
+                               return (False, state_change)
 
                        pkg = self._choose_pkg()
                        if pkg is None:
-                               return True
+                               return (True, state_change)
+
+                       state_change += 1
 
                        if not pkg.installed:
                                self._pkg_count.curval += 1
@@ -9129,7 +9228,41 @@ class Scheduler(PollScheduler):
                                self._jobs += 1
                                task.addExitListener(self._build_exit)
                                task_queues.jobs.add(task)
-               return True
+               return (True, state_change)
+
+       def _load_avg_str(self, digits=1):
+               try:
+                       avg = os.getloadavg()
+               except OSError, e:
+                       return str(e)
+               return ", ".join(fpformat.fix(x, digits) for x in avg)
+
+       def _display_status(self):
+               if self._max_jobs < 2:
+                       return
+
+               # Don't use len(self._completed_tasks) here since that also
+               # can include uninstall tasks.
+               completed_str = str(self._completed_pkg_count)
+               maxval_str = str(self._pkg_count.maxval)
+               jobs_str = str(self._jobs)
+               merges_str = str(len(self._task_queues.merge))
+               load_avg_str = self._load_avg_str()
+
+               msg = ("Jobs: %s of %s complete, %s running, %s merges, " + \
+                       "load average: %s") % \
+                       (colorize("INFORM", completed_str), colorize("INFORM", maxval_str),
+                       colorize("INFORM", jobs_str), colorize("INFORM", merges_str),
+                       load_avg_str)
+               noiselevel = 0
+               if "--verbose" in self.myopts:
+                       noiselevel = -1
+               portage.writemsg_stdout(">>> %s\n" % msg, noiselevel=noiselevel)
+
+               short_msg = ("Jobs: %s of %s complete, %s running, %s merges, " + \
+                       "load average: %s") % \
+                       (completed_str, maxval_str, jobs_str, merges_str, load_avg_str)
+               xtermTitle(short_msg)
 
        def _task(self, pkg, background):
 
@@ -10955,7 +11088,7 @@ def action_info(settings, trees, myopts, myfiles):
        if mypkgs:
                # Get our global settings (we only print stuff if it varies from
                # the current config)
-               mydesiredvars = [ 'CHOST', 'CFLAGS', 'CXXFLAGS', 'EPREFIX' ]
+               mydesiredvars = [ 'CHOST', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'EPREFIX' ]
                auxkeys = mydesiredvars + [ "USE", "IUSE"]
                global_vals = {}
                pkgsettings = portage.config(clone=settings)
index 756f9156bf253ba04752f3384c7b828bfe3d0e33..bf6a1383462a84783410434d8c7ab507b3c19f67 100644 (file)
@@ -319,7 +319,14 @@ def help(myaction,myopts,havecolor=1):
                desc = "Specifies that no new builds should " + \
                        "be started if there are other builds " + \
                        "running and the load average is at " + \
-                       "least LOAD (a floating-point number)."
+                       "least LOAD (a floating-point number). " + \
+                       "This option is recommended for use " + \
+                       "in combination with --jobs in " + \
+                       "order to avoid excess load. See " + \
+                       "make(1) for information about " + \
+                       "analogous options that should be " + \
+                       "configured via MAKEOPTS in " + \
+                       "make.conf(5)."
                for line in wrap(desc, desc_width):
                        print desc_indent + line
                print
index fbd3d1950cf3066c4953eb9d86a8a794ce38d40f..6da2e7ffd0909769a815974a73a9d78819646880 100644 (file)
@@ -1041,7 +1041,6 @@ class config(object):
 
                self.already_in_regenerate = 0
 
-               self._filter_calling_env = False
                self.locked   = 0
                self.mycpv    = None
                self.puse     = []
@@ -1067,7 +1066,6 @@ class config(object):
                self.local_config = local_config
 
                if clone:
-                       self._filter_calling_env = copy.deepcopy(clone._filter_calling_env)
                        self.incrementals = copy.deepcopy(clone.incrementals)
                        self.profile_path = copy.deepcopy(clone.profile_path)
                        self.user_profile_dir = copy.deepcopy(clone.user_profile_dir)
@@ -2828,7 +2826,13 @@ class config(object):
                "return our locally-maintained environment"
                mydict={}
                environ_filter = self._environ_filter
-               filter_calling_env = self._filter_calling_env
+
+               filter_calling_env = False
+               temp_dir = self.get("T")
+               if temp_dir is not None and \
+                       os.path.exists(os.path.join(temp_dir, "environment")):
+                       filter_calling_env = True
+
                environ_whitelist = self._environ_whitelist
                env_d = self.configdict["env.d"]
                for x in self:
@@ -4250,16 +4254,14 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0,
        mysettings["EBUILD_PHASE"] = mydo
        _doebuild_exit_status_unlink(
                mysettings.get("EBUILD_EXIT_STATUS_FILE"))
-       filter_calling_env_state = mysettings._filter_calling_env
-       if os.path.exists(os.path.join(mysettings["T"], "environment")):
-               mysettings._filter_calling_env = True
+
        try:
                phase_retval = spawn(actionmap[mydo]["cmd"] % mydo,
                        mysettings, debug=debug, logfile=logfile,
                        fd_pipes=fd_pipes, returnpid=returnpid, **kwargs)
        finally:
                mysettings["EBUILD_PHASE"] = ""
-               mysettings._filter_calling_env = filter_calling_env_state
+
        if returnpid:
                return phase_retval
        msg = _doebuild_exit_status_check(mydo, mysettings)
@@ -4277,6 +4279,23 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0,
                        phase_retval = _post_src_install_checks(mysettings)
        return phase_retval
 
+_post_phase_cmds = {
+
+       "install" : [
+               "install_qa_check",
+               "install_symlink_html_docs"],
+
+       "preinst" : [
+               "preinst_bsdflags",
+               "preinst_sfperms",
+               "preinst_selinux_labels",
+               "preinst_suid_scan",
+               "preinst_mask"],
+
+       "postinst" : [
+               "postinst_bsdflags"]
+}
+
 def _post_phase_userpriv_perms(mysettings):
        if "userpriv" in mysettings.features and secpass >= 2:
                """ Privileged phases may have left files that need to be made
@@ -4287,8 +4306,8 @@ def _post_phase_userpriv_perms(mysettings):
 
 def _post_src_install_checks(mysettings):
        _post_src_install_uid_fix(mysettings)
-       retval = _spawn_misc_sh(mysettings, ["install_qa_check",
-               "install_symlink_html_docs"])
+       global _post_phase_cmds
+       retval = _spawn_misc_sh(mysettings, post_phase_cmds["install"])
        if retval != os.EX_OK:
                writemsg("!!! install_qa_check failed; exiting.\n",
                        noiselevel=-1)
@@ -4414,10 +4433,8 @@ def _post_pkg_preinst_cmd(mysettings):
                os.path.basename(MISC_SH_BINARY))
 
        mysettings["EBUILD_PHASE"] = ""
-       myargs = [_shell_quote(misc_sh_binary),
-               "preinst_bsdflags",
-               "preinst_sfperms", "preinst_selinux_labels",
-               "preinst_suid_scan", "preinst_mask"]
+       global _post_phase_cmds
+       myargs = [_shell_quote(misc_sh_binary)] + _post_phase_cmds["preinst"]
 
        return myargs
 
@@ -4432,7 +4449,8 @@ def _post_pkg_postinst_cmd(mysettings):
                os.path.basename(MISC_SH_BINARY))
 
        mysettings["EBUILD_PHASE"] = ""
-       myargs = [_shell_quote(misc_sh_binary), "postinst_bsdflags"]
+       global _post_phase_cmds
+       myargs = [_shell_quote(misc_sh_binary)] + _post_phase_cmds["postinst"]
 
        return myargs
 
@@ -4454,9 +4472,6 @@ def _spawn_misc_sh(mysettings, commands, **kwargs):
        mycommand = " ".join([_shell_quote(misc_sh_binary)] + commands)
        _doebuild_exit_status_unlink(
                mysettings.get("EBUILD_EXIT_STATUS_FILE"))
-       filter_calling_env_state = mysettings._filter_calling_env
-       if os.path.exists(os.path.join(mysettings["T"], "environment")):
-               mysettings._filter_calling_env = True
        debug = mysettings.get("PORTAGE_DEBUG") == "1"
        logfile = mysettings.get("PORTAGE_LOG_FILE")
        mydo = mysettings["EBUILD_PHASE"]
@@ -4464,7 +4479,7 @@ def _spawn_misc_sh(mysettings, commands, **kwargs):
                rval = spawn(mycommand, mysettings, debug=debug,
                        logfile=logfile, **kwargs)
        finally:
-               mysettings._filter_calling_env = filter_calling_env_state
+               pass
        msg = _doebuild_exit_status_check(mydo, mysettings)
        if msg:
                rval = 1
@@ -5111,7 +5126,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
        builddir_lock = None
        tmpdir = None
        tmpdir_orig = None
-       filter_calling_env_state = mysettings._filter_calling_env
+
        try:
                if mydo in ("digest", "manifest", "help"):
                        # Temporarily exempt the depend phase from manifest checks, in case
@@ -5327,7 +5342,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                                                del e
                                        env_stat = None
                        if env_stat:
-                               mysettings._filter_calling_env = True
+                               pass
                        else:
                                for var in ("ARCH", ):
                                        value = mysettings.get(var)
@@ -5636,7 +5651,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                return retval
 
        finally:
-               mysettings._filter_calling_env = filter_calling_env_state
+
                if tmpdir:
                        mysettings["PORTAGE_TMPDIR"] = tmpdir_orig
                        shutil.rmtree(tmpdir)
index 360d303adb58fce74a64a91a11b4d1543061d8c5..4bc7ab87b5cce97229079e25f46b724f9242c274 100644 (file)
@@ -28,6 +28,7 @@ from portage.elog.messages import ewarn
 from portage.elog.filtering import filter_mergephases, filter_unmergephases
 
 import os, re, sys, stat, errno, commands, copy, time, subprocess
+import logging
 from itertools import izip
 
 try:
@@ -1878,6 +1879,19 @@ class dblink(object):
                        contents=contents, env=self.settings.environ())
                return os.EX_OK
 
+       def _display_merge(self, msg, level=0):
+               if self._scheduler is not None:
+                       self._scheduler.dblinkDisplayMerge(self, msg, level=level)
+                       return
+
+               if level >= logging.WARNING:
+                       noiselevel = -1
+                       msg_func = writemsg
+               else:
+                       noiselevel = 0
+                       msg_func = writemsg_stdout
+               msg_func(msg, noiselevel=noiselevel)
+
        def _unmerge_pkgfiles(self, pkgfiles, others_in_slot):
                """
                
@@ -1891,8 +1905,10 @@ class dblink(object):
                @rtype: None
                """
 
+               showMessage = self._display_merge
+
                if not pkgfiles:
-                       writemsg_stdout("No package files given... Grabbing a set.\n")
+                       showMessage("No package files given... Grabbing a set.\n")
                        pkgfiles = self.getcontents()
 
                if others_in_slot is None:
@@ -1956,7 +1972,7 @@ class dblink(object):
                                                bsd_chflags.chflags(parent_name, pflags)
 
                        def show_unmerge(zing, desc, file_type, file_name):
-                                       writemsg_stdout("%s %s %s %s\n" % \
+                                       showMessage("%s %s %s %s\n" % \
                                                (zing, desc.ljust(8), file_type, file_name))
                        for objkey in mykeys:
                                obj = normalize_path(objkey)
@@ -2334,17 +2350,18 @@ class dblink(object):
                        collision_ignore = set([normalize_path(myignore) for myignore in \
                                self.settings.get("COLLISION_IGNORE", "").split()])
 
+                       showMessage = self._display_merge
                        stopmerge = False
                        i=0
                        collisions = []
                        destroot = normalize_path(destroot).rstrip(os.path.sep) + \
                                os.path.sep
-                       writemsg_stdout("%s checking %d files for package collisions\n" % \
+                       showMessage("%s checking %d files for package collisions\n" % \
                                (green("*"), len(mycontents)))
                        for f in mycontents:
                                i = i + 1
                                if i % 1000 == 0:
-                                       writemsg_stdout("%d files checked ...\n" % i)
+                                       showMessage("%d files checked ...\n" % i)
                                dest_path = normalize_path(
                                        os.path.join(destroot, f.lstrip(os.path.sep)))
                                try:
@@ -2402,6 +2419,9 @@ class dblink(object):
        def _security_check(self, installed_instances):
                if not installed_instances:
                        return 0
+
+               showMessage = self._display_merge
+
                file_paths = set()
                for dblnk in installed_instances:
                        file_paths.update(dblnk.getcontents())
@@ -2436,13 +2456,13 @@ class dblink(object):
                        return 0
                from portage.output import colorize
                prefix = colorize("SECURITY_WARN", "*") + " WARNING: "
-               writemsg(prefix + "suid/sgid file(s) " + \
-                       "with suspicious hardlink(s):\n", noiselevel=-1)
+               showMessage(prefix + "suid/sgid file(s) " + \
+                       "with suspicious hardlink(s):\n", level=logging.ERROR)
                for path_list in suspicious_hardlinks:
                        for path, s in path_list:
-                               writemsg(prefix + "  '%s'\n" % path, noiselevel=-1)
-               writemsg(prefix + "See the Gentoo Security Handbook " + \
-                       "guide for advice on how to proceed.\n", noiselevel=-1)
+                               showMessage(prefix + "  '%s'\n" % path, level=logging.ERROR)
+               showMessage(prefix + "See the Gentoo Security Handbook " + \
+                       "guide for advice on how to proceed.\n", level=logging.ERROR)
                return 1
 
        def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
@@ -2481,12 +2501,14 @@ class dblink(object):
                not existing; we will merge these symlinks at a later time.
                """
 
+               showMessage = self._display_merge
+
                srcroot = normalize_path(srcroot).rstrip(os.path.sep) + os.path.sep
                destroot = normalize_path(destroot).rstrip(os.path.sep) + os.path.sep
 
                if not os.path.isdir(srcroot):
-                       writemsg("!!! Directory Not Found: D='%s'\n" % srcroot,
-                               noiselevel=-1)
+                       showMessage("!!! Directory Not Found: D='%s'\n" % srcroot,
+                               level=logging.ERROR)
                        return 1
 
                inforoot_slot_file = os.path.join(inforoot, "SLOT")
@@ -2511,8 +2533,8 @@ class dblink(object):
                                _eerror(l, phase="preinst", key=self.settings.mycpv)
 
                if slot != self.settings["SLOT"]:
-                       writemsg("!!! WARNING: Expected SLOT='%s', got '%s'\n" % \
-                               (self.settings["SLOT"], slot))
+                       showMessage("!!! WARNING: Expected SLOT='%s', got '%s'\n" % \
+                               (self.settings["SLOT"], slot), level=logging.WARN)
 
                if not os.path.exists(self.dbcatdir):
                        os.makedirs(self.dbcatdir)
@@ -2729,7 +2751,7 @@ class dblink(object):
 
                # run preinst script
                if scheduler is None:
-                       writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot))
+                       showMessage(">>> Merging %s to %s\n" % (self.mycpv, destroot))
                        a = doebuild(myebuild, "preinst", destroot, self.settings,
                                use_cache=0, tree=self.treetype, mydbapi=mydbapi,
                                vartree=self.vartree)
@@ -2739,7 +2761,8 @@ class dblink(object):
 
                # XXX: Decide how to handle failures here.
                if a != os.EX_OK:
-                       writemsg("!!! FAILED preinst: "+str(a)+"\n", noiselevel=-1)
+                       showMessage("!!! FAILED preinst: "+str(a)+"\n",
+                               level=logging.ERROR)
                        return a
 
                # copy "info" files (like SLOT, CFLAGS, etc.) into the database
@@ -2849,19 +2872,20 @@ class dblink(object):
                                continue
                        if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self):
                                continue
-                       writemsg_stdout(">>> Safely unmerging already-installed instance...\n")
+                       showMessage(">>> Safely unmerging already-installed instance...\n")
                        others_in_slot.remove(dblnk) # dblnk will unmerge itself now
                        dblnk.unmerge(trimworld=0, ldpath_mtimes=prev_mtimes,
                                others_in_slot=others_in_slot)
                        # TODO: Check status and abort if necessary.
                        dblnk.delete()
-                       writemsg_stdout(">>> Original instance of package unmerged safely.\n")
+                       showMessage(">>> Original instance of package unmerged safely.\n")
 
                if len(others_in_slot) > 1:
                        from portage.output import colorize
-                       writemsg_stdout(colorize("WARN", "WARNING:")
+                       showMessage(colorize("WARN", "WARNING:")
                                + " AUTOCLEAN is disabled.  This can cause serious"
-                               + " problems due to overlapping packages.\n")
+                               + " problems due to overlapping packages.\n",
+                               level=logging.WARN)
 
                # We hold both directory locks.
                self.dbdir = self.dbpkgdir
@@ -2894,7 +2918,7 @@ class dblink(object):
                                        use_cache=0, tree=self.treetype, mydbapi=mydbapi,
                                        vartree=self.vartree)
                                if a == os.EX_OK:
-                                       writemsg_stdout(">>> %s %s\n" % (self.mycpv, "merged."))
+                                       showMessage(">>> %s %s\n" % (self.mycpv, "merged."))
                        else:
                                a = scheduler.dblinkEbuildPhase(
                                        self, mydbapi, myebuild, "postinst")
@@ -2903,7 +2927,8 @@ class dblink(object):
 
                # XXX: Decide how to handle failures here.
                if a != os.EX_OK:
-                       writemsg("!!! FAILED postinst: "+str(a)+"\n", noiselevel=-1)
+                       showMessage("!!! FAILED postinst: "+str(a)+"\n",
+                               level=logging.ERROR)
                        return a
 
                downgrade = False
@@ -2945,6 +2970,9 @@ class dblink(object):
                2. None otherwise
                
                """
+
+               showMessage = self._display_merge
+
                from os.path import sep, join
                srcroot = normalize_path(srcroot).rstrip(sep) + sep
                destroot = normalize_path(destroot).rstrip(sep) + sep
@@ -3045,7 +3073,7 @@ class dblink(object):
                                # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly
                                mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings)
                                if mymtime != None:
-                                       writemsg_stdout(">>> %s -> %s\n" % (mydest, myto))
+                                       showMessage(">>> %s -> %s\n" % (mydest, myto))
                                        outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n")
                                else:
                                        print "!!! Failed to move file."
@@ -3073,7 +3101,7 @@ class dblink(object):
 
                                        if stat.S_ISLNK(mydmode) or stat.S_ISDIR(mydmode):
                                                # a symlink to an existing directory will work for us; keep it:
-                                               writemsg_stdout("--- %s/\n" % mydest)
+                                               showMessage("--- %s/\n" % mydest)
                                                if bsd_chflags:
                                                        bsd_chflags.lchflags(mydest, dflags)
                                        else:
@@ -3092,7 +3120,7 @@ class dblink(object):
                                                        bsd_chflags.lchflags(mydest, dflags)
                                                os.chmod(mydest, mystat[0])
                                                os.chown(mydest, mystat[4], mystat[5])
-                                               writemsg_stdout(">>> %s/\n" % mydest)
+                                               showMessage(">>> %s/\n" % mydest)
                                else:
                                        #destination doesn't exist
                                        if self.settings.selinux_enabled():
@@ -3103,7 +3131,7 @@ class dblink(object):
                                                os.mkdir(mydest)
                                        os.chmod(mydest, mystat[0])
                                        os.chown(mydest, mystat[4], mystat[5])
-                                       writemsg_stdout(">>> %s/\n" % mydest)
+                                       showMessage(">>> %s/\n" % mydest)
                                outfile.write("dir "+myrealdest+"\n")
                                # recurse and merge this directory
                                if self.mergeme(srcroot, destroot, outfile, secondhand,
@@ -3122,7 +3150,7 @@ class dblink(object):
                                        if stat.S_ISDIR(mydmode):
                                                # install of destination is blocked by an existing directory with the same name
                                                moveme = 0
-                                               writemsg_stdout("!!! %s\n" % mydest)
+                                               showMessage("!!! %s\n" % mydest, level=logging.ERROR)
                                        elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])):
                                                cfgprot = 0
                                                # install of destination is blocked by an existing regular file,
@@ -3168,7 +3196,7 @@ class dblink(object):
 
                                if mymtime != None:
                                        outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n")
-                               writemsg_stdout("%s %s\n" % (zing,mydest))
+                               showMessage("%s %s\n" % (zing,mydest))
                        else:
                                # we are merging a fifo or device node
                                zing = "!!!"
@@ -3182,7 +3210,7 @@ class dblink(object):
                                        outfile.write("fif %s\n" % myrealdest)
                                else:
                                        outfile.write("dev %s\n" % myrealdest)
-                               writemsg_stdout(zing + " " + mydest + "\n")
+                               showMessage(zing + " " + mydest + "\n")
 
        def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0,
                mydbapi=None, prev_mtimes=None):