Merged from trunk 10843:10853
authorFabian Groffen <grobian@gentoo.org>
Tue, 1 Jul 2008 17:09:13 +0000 (17:09 -0000)
committerFabian Groffen <grobian@gentoo.org>
Tue, 1 Jul 2008 17:09:13 +0000 (17:09 -0000)
   | 10844   | Handle invalid atoms inside depgraph._add_pkg_deps(), to     |
   | zmedico | avoid an unhandled InvalidAtom exception from dep_expand()   |
   |         | when matching against the vardb.                             |

   | 10845   | Split out a _spawn_misc_sh() function from spawnebuild().    |
   | zmedico |                                                              |

   | 10846   | Split out a _post_src_install_uid_fix() function from        |
   | zmedico | spawnebuild().                                               |

   | 10847   | Split out a _check_build_log() function from spawnebuild().  |
   | zmedico |                                                              |

   | 10848   | In EbuildBuild.execute(), call doebuild() separately for     |
   | zmedico | each phase since it will be easier to add asynchronous       |
   |         | support to doebuild() if it's called this way.               |

   | 10849   | * Add "fd_pipes" and "returnpid" parameters to doebuild()    |
   | zmedico | and pass these into spawn calls, enabling ebuild processes   |
   |         | to execute asynchronously. * Add a EbuildPhase class that's  |
   |         | derived from the pty logging code inside portage.spawn(). *  |
   |         | Integrate post-phase code from spawnebuild() into            |
   |         | EbuildBuild.execute() so that it still gets called even      |
   |         | though doebuild() calls execute asynchronously.              |

   | 10850   | Add EMERGE_FROM to the blacklisted vars in the config        |
   | zmedico | constructor.                                                 |

   | 10851   | Centralize select.poll() event handling in                   |
   | zmedico | MergeTask._schedule(). This will allow the parent process to |
   |         | handle output of multiple child processes running in         |
   |         | parllel.                                                     |

   | 10852   | * Tweak conditionals in doebuild() to fix emerge --fetch     |
   | zmedico | breakage reported by Arfrever. * Change a couple EMERGE_FROM |
   |         | conditionals to use the "returnpid" parameter instead.       |

   | 10853   | Rename the MergeTask class to Scheduler.                     |
   | zmedico |                                                              |

svn path=/main/branches/prefix/; revision=10879

pym/_emerge/__init__.py
pym/portage/__init__.py

index 59c0c6d8b917d46f5139d0efdac83f46489bfd9c..90162d7e90bdfd9af0f95b3bad25e672a6244434 100644 (file)
@@ -20,6 +20,8 @@ try:
 except KeyboardInterrupt:
        sys.exit(1)
 
+import array
+import select
 import gc
 import os, stat
 import platform
@@ -1554,7 +1556,9 @@ class EbuildBuild(Task):
        """
        TODO: Support asynchronous execution, to implement parallel builds.
        """
-       __slots__ = ("pkg", "settings")
+       __slots__ = ("pkg", "register", "schedule", "settings", "unregister")
+
+       _phases = ("setup", "unpack", "compile", "test", "install")
 
        def _get_hash_key(self):
                hash_key = getattr(self, "_hash_key", None)
@@ -1566,18 +1570,170 @@ class EbuildBuild(Task):
                root_config = self.pkg.root_config
                portdb = root_config.trees["porttree"].dbapi
                ebuild_path = portdb.findname(self.pkg.cpv)
-               debug = self.settings.get("PORTAGE_DEBUG") == "1"
+               settings = self.settings
+               debug = settings.get("PORTAGE_DEBUG") == "1"
+               cleanup = 1
 
                retval = portage.doebuild(ebuild_path, "clean",
-                       root_config.root, self.settings, debug, cleanup=1,
+                       root_config.root, settings, debug, cleanup=cleanup,
                        mydbapi=portdb, tree="porttree")
                if retval != os.EX_OK:
                        return retval
 
-               retval = portage.doebuild(ebuild_path, "install",
-                       root_config.root, self.settings, debug,
-                       mydbapi=portdb, tree="porttree")
-               return retval
+               # This initializes PORTAGE_LOG_FILE.
+               portage.prepare_build_dirs(root_config.root, settings, cleanup)
+
+               fd_pipes = {
+                       0 : sys.stdin.fileno(),
+                       1 : sys.stdout.fileno(),
+                       2 : sys.stderr.fileno(),
+               }
+
+               for mydo in self._phases:
+                       ebuild_phase = EbuildPhase(fd_pipes=fd_pipes,
+                               pkg=self.pkg, phase=mydo, register=self.register,
+                               settings=settings, unregister=self.unregister)
+                       ebuild_phase.start()
+                       self.schedule()
+                       retval = ebuild_phase.wait()
+
+                       portage._post_phase_userpriv_perms(settings)
+                       if mydo == "install":
+                               portage._check_build_log(settings)
+                               if retval == os.EX_OK:
+                                       retval = portage._post_src_install_checks(settings)
+
+                       if retval != os.EX_OK:
+                               return retval
+
+               return os.EX_OK
+
+class EbuildPhase(SlotObject):
+
+       __slots__ = ("fd_pipes", "phase", "pkg",
+               "register", "settings", "unregister",
+               "pid", "returncode", "files")
+
+       _file_names = ("log", "stdout", "ebuild")
+       _files_dict = slot_dict_class(_file_names)
+       _bufsize = 4096
+
+       def start(self):
+               root_config = self.pkg.root_config
+               portdb = root_config.trees["porttree"].dbapi
+               ebuild_path = portdb.findname(self.pkg.cpv)
+               settings = self.settings
+               debug = settings.get("PORTAGE_DEBUG") == "1"
+               logfile = settings.get("PORTAGE_LOG_FILE")
+               master_fd = None
+               slave_fd = None
+               fd_pipes = self.fd_pipes.copy()
+
+               # flush any pending output
+               for fd in fd_pipes.itervalues():
+                       if fd == sys.stdout.fileno():
+                               sys.stdout.flush()
+                       if fd == sys.stderr.fileno():
+                               sys.stderr.flush()
+
+               fd_pipes_orig = None
+               self.files = self._files_dict()
+               files = self.files
+               got_pty = False
+
+               portage._doebuild_exit_status_unlink(
+                       settings.get("EBUILD_EXIT_STATUS_FILE"))
+
+               if logfile:
+                       if portage._disable_openpty:
+                               master_fd, slave_fd = os.pipe()
+                       else:
+                               from pty import openpty
+                               try:
+                                       master_fd, slave_fd = openpty()
+                                       got_pty = True
+                               except EnvironmentError, e:
+                                       portage._disable_openpty = True
+                                       portage.writemsg("openpty failed: '%s'\n" % str(e),
+                                               noiselevel=-1)
+                                       del e
+                                       master_fd, slave_fd = os.pipe()
+
+                       if got_pty:
+                               # Disable post-processing of output since otherwise weird
+                               # things like \n -> \r\n transformations may occur.
+                               import termios
+                               mode = termios.tcgetattr(slave_fd)
+                               mode[1] &= ~termios.OPOST
+                               termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
+
+                       import fcntl
+                       fcntl.fcntl(master_fd, fcntl.F_SETFL,
+                               fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
+
+                       fd_pipes.setdefault(0, sys.stdin.fileno())
+                       fd_pipes_orig = fd_pipes.copy()
+                       if got_pty and os.isatty(fd_pipes_orig[1]):
+                               from portage.output import get_term_size, set_term_size
+                               rows, columns = get_term_size()
+                               set_term_size(rows, columns, slave_fd)
+                       fd_pipes[0] = fd_pipes_orig[0]
+                       fd_pipes[1] = slave_fd
+                       fd_pipes[2] = slave_fd
+
+               retval = portage.doebuild(ebuild_path, self.phase,
+                       root_config.root, settings, debug,
+                       mydbapi=portdb, tree="porttree",
+                       fd_pipes=fd_pipes, returnpid=True)
+
+               self.pid = retval[0]
+
+               if logfile:
+                       os.close(slave_fd)
+                       files["log"] = open(logfile, 'a')
+                       files["stdout"] = os.fdopen(os.dup(fd_pipes_orig[1]), 'w')
+                       files["ebuild"] = os.fdopen(master_fd, 'r')
+                       self.register(files["ebuild"].fileno(),
+                               select.POLLIN, self._output_handler)
+
+       def _output_handler(self, fd, event):
+               files = self.files
+               buf = array.array('B')
+               try:
+                       buf.fromfile(files["ebuild"], self._bufsize)
+               except EOFError:
+                       pass
+               if buf:
+                       buf.tofile(files["stdout"])
+                       files["stdout"].flush()
+                       buf.tofile(files["log"])
+                       files["log"].flush()
+               else:
+                       self.unregister(files["ebuild"].fileno())
+                       for f in files.values():
+                               f.close()
+
+       def wait(self):
+               pid = self.pid
+               retval = os.waitpid(pid, 0)[1]
+               portage.process.spawned_pids.remove(pid)
+               if retval != os.EX_OK:
+                       if retval & 0xff:
+                               retval = (retval & 0xff) << 8
+                       else:
+                               retval = retval >> 8
+
+               msg = portage._doebuild_exit_status_check(
+                       self.phase, self.settings)
+               if msg:
+                       retval = 1
+                       from textwrap import wrap
+                       from portage.elog.messages import eerror
+                       for l in wrap(msg, 72):
+                               eerror(l, phase=self.phase, key=self.pkg.cpv)
+
+               self.returncode = retval
+               return self.returncode
 
 class EbuildBinpkg(Task):
        """
@@ -2776,6 +2932,13 @@ class depgraph(object):
                                if debug:
                                        print "Candidates:", selected_atoms
                                for atom in selected_atoms:
+                                       if isinstance(atom, basestring) \
+                                               and not portage.isvalidatom(atom):
+                                               show_invalid_depstring_notice(
+                                                       pkg, dep_string, str(atom))
+                                               if not pkg.installed:
+                                                       return 0
+                                               continue
                                        blocker = atom.startswith("!")
                                        if blocker:
                                                atom = atom[1:]
@@ -6198,7 +6361,7 @@ class PackageCounters(object):
                                        (self.blocks - self.blocks_satisfied))
                return "".join(myoutput)
 
-class MergeTask(object):
+class Scheduler(object):
 
        _opts_ignore_blockers = \
                frozenset(["--buildpkgonly",
@@ -6224,6 +6387,8 @@ class MergeTask(object):
                                clone=trees[root]["vartree"].settings)
                self.curval = 0
                self._spawned_pids = []
+               self._poll_event_handlers = {}
+               self._poll = select.poll()
 
        class _pkg_failure(portage.exception.PortageException):
                """
@@ -6388,6 +6553,19 @@ class MergeTask(object):
                                pass
                        spawned_pids.remove(pid)
 
+       def _register(self, f, eventmask, handler):
+               self._poll_event_handlers[f] = handler
+               self._poll.register(f, eventmask)
+
+       def _unregister(self, f):
+               self._poll.unregister(f)
+               del self._poll_event_handlers[f]
+
+       def _schedule(self):
+               while self._poll_event_handlers:
+                       for f, event in self._poll.poll():
+                               self._poll_event_handlers[f](f, event)
+
        def _merge(self):
                mylist = self._mergelist
                favorites = self._favorites
@@ -6626,7 +6804,9 @@ class MergeTask(object):
                                                        (mergecount, len(mymergelist), pkg_key)
                                                emergelog(xterm_titles, msg, short_msg=short_msg)
 
-                                               build = EbuildBuild(pkg=pkg, settings=pkgsettings)
+                                               build = EbuildBuild(pkg=pkg, register=self._register,
+                                                       schedule=self._schedule, settings=pkgsettings,
+                                                       unregister=self._unregister)
                                                retval = build.execute()
                                                if retval != os.EX_OK:
                                                        raise self._pkg_failure(retval)
@@ -6661,7 +6841,9 @@ class MergeTask(object):
                                                        (mergecount, len(mymergelist), pkg_key)
                                                emergelog(xterm_titles, msg, short_msg=short_msg)
 
-                                               build = EbuildBuild(pkg=pkg, settings=pkgsettings)
+                                               build = EbuildBuild(pkg=pkg, register=self._register,
+                                                       schedule=self._schedule, settings=pkgsettings,
+                                                       unregister=self._unregister)
                                                retval = build.execute()
                                                if retval != os.EX_OK:
                                                        raise self._pkg_failure(retval)
@@ -9297,7 +9479,7 @@ def action_build(settings, trees, mtimedb,
                        mymergelist = mydepgraph.altlist()
                        mydepgraph.break_refs(mymergelist)
                        mydepgraph.break_refs(mydepgraph.digraph.order)
-                       mergetask = MergeTask(settings, trees, mtimedb, myopts,
+                       mergetask = Scheduler(settings, trees, mtimedb, myopts,
                                spinner, mymergelist, favorites, mydepgraph.digraph)
                        del mydepgraph
                        clear_caches(trees)
@@ -9344,7 +9526,7 @@ def action_build(settings, trees, mtimedb,
                        mydepgraph.saveNomergeFavorites()
                        mydepgraph.break_refs(pkglist)
                        mydepgraph.break_refs(mydepgraph.digraph.order)
-                       mergetask = MergeTask(settings, trees, mtimedb, myopts,
+                       mergetask = Scheduler(settings, trees, mtimedb, myopts,
                                spinner, pkglist, favorites, mydepgraph.digraph)
                        del mydepgraph
                        clear_caches(trees)
index feb3d1332d66e472ef7715cb0baaa66120b34de3..101efdbdb51b9768d4ca43ccf741e802b45f517c 100644 (file)
@@ -1405,7 +1405,7 @@ class config(object):
 
                        # Blacklist vars that could interfere with portage internals.
                        for blacklisted in "CATEGORY", "EBUILD_PHASE", \
-                               "PKGUSE", "PORTAGE_CONFIGROOT", \
+                               "EMERGE_FROM", "PKGUSE", "PORTAGE_CONFIGROOT", \
                                "PORTAGE_IUSE", "PORTAGE_USE", "ROOT", \
                                "EPREFIX", "EROOT":
                                for cfg in self.lookuplist:
@@ -2960,6 +2960,10 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero
                env=mysettings.environ()
                keywords["opt_name"]="[%s]" % mysettings["PF"]
 
+       if keywords.get("returnpid"):
+               # emerge handles logging externally
+               keywords.pop("logfile", None)
+
        fd_pipes = keywords.get("fd_pipes")
        if fd_pipes is None:
                fd_pipes = {
@@ -4180,11 +4184,15 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
        return 1
 
 # parse actionmap to spawn ebuild with the appropriate args
-def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
-       if alwaysdep or "noauto" not in mysettings.features:
+def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0,
+       logfile=None, fd_pipes=None, returnpid=False):
+       if not returnpid and \
+               (alwaysdep or "noauto" not in mysettings.features):
                # process dependency first
                if "dep" in actionmap[mydo]:
-                       retval=spawnebuild(actionmap[mydo]["dep"],actionmap,mysettings,debug,alwaysdep=alwaysdep,logfile=logfile)
+                       retval = spawnebuild(actionmap[mydo]["dep"], actionmap,
+                               mysettings, debug, alwaysdep=alwaysdep, logfile=logfile,
+                               fd_pipes=fd_pipes, returnpid=returnpid)
                        if retval:
                                return retval
        kwargs = actionmap[mydo]["args"]
@@ -4196,10 +4204,13 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
                mysettings._filter_calling_env = True
        try:
                phase_retval = spawn(actionmap[mydo]["cmd"] % mydo,
-                       mysettings, debug=debug, logfile=logfile, **kwargs)
+                       mysettings, debug=debug, logfile=logfile,
+                       fd_pipes=fd_pipes, returnpid=returnpid, **kwargs)
        finally:
                mysettings["EBUILD_PHASE"] = ""
                mysettings._filter_calling_env = filter_calling_env_state
+       if returnpid:
+               return phase_retval
        msg = _doebuild_exit_status_check(mydo, mysettings)
        if msg:
                phase_retval = 1
@@ -4208,137 +4219,173 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
                for l in wrap(msg, 72):
                        eerror(l, phase=mydo, key=mysettings.mycpv)
 
-       if "userpriv" in mysettings.features and \
-               not kwargs["droppriv"] and secpass >= 2:
+       _post_phase_userpriv_perms(mysettings)
+       if mydo == "install":
+               _check_build_log(mysettings)
+               if phase_retval == os.EX_OK:
+                       phase_retval = _post_src_install_checks(mysettings)
+       return phase_retval
+
+def _post_phase_userpriv_perms(mysettings):
+       if "userpriv" in mysettings.features and secpass >= 2:
                """ Privileged phases may have left files that need to be made
                writable to a less privileged user."""
                apply_recursive_permissions(mysettings["T"],
                        uid=portage_uid, gid=portage_gid, dirmode=070, dirmask=0,
                        filemode=060, filemask=0)
 
-       if phase_retval == os.EX_OK:
-               if mydo == "install" and logfile:
-                       try:
-                               f = open(logfile, 'rb')
-                       except EnvironmentError:
-                               pass
-                       else:
-                               am_maintainer_mode = []
-
-                               bash_command_not_found = []
-                               bash_command_not_found_re = re.compile(
-                                       r'(.*): line (\d*): (.*): command not found$')
-
-                               configure_opts_warn = []
-                               configure_opts_warn_re = re.compile(
-                                       r'^configure: WARNING: Unrecognized options: .*')
-                               am_maintainer_mode_re = re.compile(r'.*/missing --run .*')
-                               am_maintainer_mode_exclude_re = \
-                                       re.compile(r'.*/missing --run (autoheader|makeinfo)')
-                               try:
-                                       for line in f:
-                                               if am_maintainer_mode_re.search(line) is not None and \
-                                                       am_maintainer_mode_exclude_re.search(line) is None:
-                                                       am_maintainer_mode.append(line.rstrip("\n"))
-
-                                               if bash_command_not_found_re.match(line) is not None:
-                                                       bash_command_not_found.append(line.rstrip("\n"))
-
-                                               if configure_opts_warn_re.match(line) is not None:
-                                                       configure_opts_warn.append(line.rstrip("\n"))
-                               finally:
-                                       f.close()
-
-                               from portage.elog.messages import eqawarn
-                               def _eqawarn(lines):
-                                       for line in lines:
-                                               eqawarn(line, phase=mydo, key=mysettings.mycpv)
-                               from textwrap import wrap
-                               wrap_width = 70
-
-                               if am_maintainer_mode:
-                                       msg = ["QA Notice: Automake \"maintainer mode\" detected:"]
-                                       msg.append("")
-                                       msg.extend("\t" + line for line in am_maintainer_mode)
-                                       msg.append("")
-                                       msg.extend(wrap(
-                                               "If you patch Makefile.am, " + \
-                                               "configure.in,  or configure.ac then you " + \
-                                               "should use autotools.eclass and " + \
-                                               "eautomake or eautoreconf. Exceptions " + \
-                                               "are limited to system packages " + \
-                                               "for which it is impossible to run " + \
-                                               "autotools during stage building. " + \
-                                               "See http://www.gentoo.org/p" + \
-                                               "roj/en/qa/autofailure.xml for more information.",
-                                               wrap_width))
-                                       _eqawarn(msg)
-
-                               if bash_command_not_found:
-                                       msg = ["QA Notice: command not found:"]
-                                       msg.append("")
-                                       msg.extend("\t" + line for line in bash_command_not_found)
-                                       _eqawarn(msg)
-
-                               if configure_opts_warn:
-                                       msg = ["QA Notice: Unrecognized configure options:"]
-                                       msg.append("")
-                                       msg.extend("\t" + line for line in configure_opts_warn)
-                                       _eqawarn(msg)
-
-               if mydo == "install":
-                       # User and group bits that match the "portage" user or group are
-                       # automatically mapped to PORTAGE_INST_UID and PORTAGE_INST_GID if
-                       # necessary.  The chown system call may clear S_ISUID and S_ISGID
-                       # bits, so those bits are restored if necessary.
-                       inst_uid = int(mysettings["PORTAGE_INST_UID"])
-                       inst_gid = int(mysettings["PORTAGE_INST_GID"])
-                       for parent, dirs, files in os.walk(mysettings["D"]):
-                               for fname in chain(dirs, files):
-                                       fpath = os.path.join(parent, fname)
-                                       mystat = os.lstat(fpath)
-                                       if mystat.st_uid != portage_uid and \
-                                               mystat.st_gid != portage_gid:
-                                               continue
-                                       myuid = -1
-                                       mygid = -1
-                                       if mystat.st_uid == portage_uid:
-                                               myuid = inst_uid
-                                       if mystat.st_gid == portage_gid:
-                                               mygid = inst_gid
-                                       apply_secpass_permissions(fpath, uid=myuid, gid=mygid,
-                                               mode=mystat.st_mode, stat_cached=mystat,
-                                               follow_links=False)
-                       # Note: PORTAGE_BIN_PATH may differ from the global
-                       # constant when portage is reinstalling itself.
-                       portage_bin_path = mysettings["PORTAGE_BIN_PATH"]
-                       misc_sh_binary = os.path.join(portage_bin_path,
-                               os.path.basename(MISC_SH_BINARY))
-                       mycommand = " ".join([_shell_quote(misc_sh_binary),
-                               "install_qa_check", "install_symlink_html_docs"])
-                       _doebuild_exit_status_unlink(
-                               mysettings.get("EBUILD_EXIT_STATUS_FILE"))
-                       filter_calling_env_state = mysettings._filter_calling_env
-                       if os.path.exists(os.path.join(mysettings["T"], "environment")):
-                               mysettings._filter_calling_env = True
-                       try:
-                               qa_retval = spawn(mycommand, mysettings, debug=debug,
-                                       logfile=logfile, **kwargs)
-                       finally:
-                               mysettings._filter_calling_env = filter_calling_env_state
-                       msg = _doebuild_exit_status_check(mydo, mysettings)
-                       if msg:
-                               qa_retval = 1
-                               from textwrap import wrap
-                               from portage.elog.messages import eerror
-                               for l in wrap(msg, 72):
-                                       eerror(l, phase=mydo, key=mysettings.mycpv)
-                       if qa_retval != os.EX_OK:
-                               writemsg("!!! install_qa_check failed; exiting.\n",
-                                       noiselevel=-1)
-                       return qa_retval
-       return phase_retval
+def _post_src_install_checks(mysettings):
+       _post_src_install_uid_fix(mysettings)
+       retval = _spawn_misc_sh(mysettings, ["install_qa_check",
+               "install_symlink_html_docs"])
+       if retval != os.EX_OK:
+               writemsg("!!! install_qa_check failed; exiting.\n",
+                       noiselevel=-1)
+       return retval
+
+def _check_build_log(mysettings):
+       """
+       Search the content of $PORTAGE_LOG_FILE if it exists
+       and generate the following QA Notices when appropriate:
+
+         * Automake "maintainer mode"
+         * command not found
+         * Unrecognized configure options
+       """
+       logfile = mysettings.get("PORTAGE_LOG_FILE")
+       if logfile is None:
+               return
+       try:
+               f = open(logfile, 'rb')
+       except EnvironmentError:
+               return
+
+       am_maintainer_mode = []
+       bash_command_not_found = []
+       bash_command_not_found_re = re.compile(
+               r'(.*): line (\d*): (.*): command not found$')
+
+       configure_opts_warn = []
+       configure_opts_warn_re = re.compile(
+               r'^configure: WARNING: Unrecognized options: .*')
+       am_maintainer_mode_re = re.compile(r'.*/missing --run .*')
+       am_maintainer_mode_exclude_re = \
+               re.compile(r'.*/missing --run (autoheader|makeinfo)')
+       try:
+               for line in f:
+                       if am_maintainer_mode_re.search(line) is not None and \
+                               am_maintainer_mode_exclude_re.search(line) is None:
+                               am_maintainer_mode.append(line.rstrip("\n"))
+
+                       if bash_command_not_found_re.match(line) is not None:
+                               bash_command_not_found.append(line.rstrip("\n"))
+
+                       if configure_opts_warn_re.match(line) is not None:
+                               configure_opts_warn.append(line.rstrip("\n"))
+       finally:
+               f.close()
 
+       from portage.elog.messages import eqawarn
+       def _eqawarn(lines):
+               for line in lines:
+                       eqawarn(line, phase="install", key=mysettings.mycpv)
+       from textwrap import wrap
+       wrap_width = 70
+
+       if am_maintainer_mode:
+               msg = ["QA Notice: Automake \"maintainer mode\" detected:"]
+               msg.append("")
+               msg.extend("\t" + line for line in am_maintainer_mode)
+               msg.append("")
+               msg.extend(wrap(
+                       "If you patch Makefile.am, " + \
+                       "configure.in,  or configure.ac then you " + \
+                       "should use autotools.eclass and " + \
+                       "eautomake or eautoreconf. Exceptions " + \
+                       "are limited to system packages " + \
+                       "for which it is impossible to run " + \
+                       "autotools during stage building. " + \
+                       "See http://www.gentoo.org/p" + \
+                       "roj/en/qa/autofailure.xml for more information.",
+                       wrap_width))
+               _eqawarn(msg)
+
+       if bash_command_not_found:
+               msg = ["QA Notice: command not found:"]
+               msg.append("")
+               msg.extend("\t" + line for line in bash_command_not_found)
+               _eqawarn(msg)
+
+       if configure_opts_warn:
+               msg = ["QA Notice: Unrecognized configure options:"]
+               msg.append("")
+               msg.extend("\t" + line for line in configure_opts_warn)
+               _eqawarn(msg)
+
+def _post_src_install_uid_fix(mysettings):
+       """
+       Files in $D with user and group bits that match the "portage"
+       user or group are automatically mapped to PORTAGE_INST_UID and
+       PORTAGE_INST_GID if necessary. The chown system call may clear
+       S_ISUID and S_ISGID bits, so those bits are restored if
+       necessary.
+       """
+       inst_uid = int(mysettings["PORTAGE_INST_UID"])
+       inst_gid = int(mysettings["PORTAGE_INST_GID"])
+       for parent, dirs, files in os.walk(mysettings["D"]):
+               for fname in chain(dirs, files):
+                       fpath = os.path.join(parent, fname)
+                       mystat = os.lstat(fpath)
+                       if mystat.st_uid != portage_uid and \
+                               mystat.st_gid != portage_gid:
+                               continue
+                       myuid = -1
+                       mygid = -1
+                       if mystat.st_uid == portage_uid:
+                               myuid = inst_uid
+                       if mystat.st_gid == portage_gid:
+                               mygid = inst_gid
+                       apply_secpass_permissions(fpath, uid=myuid, gid=mygid,
+                               mode=mystat.st_mode, stat_cached=mystat,
+                               follow_links=False)
+
+def _spawn_misc_sh(mysettings, commands, **kwargs):
+       """
+       @param mysettings: the ebuild config
+       @type mysettings: config
+       @param commands: a list of function names to call in misc-functions.sh
+       @type commands: list
+       @rtype: int
+       @returns: the return value from the spawn() call
+       """
+
+       # Note: PORTAGE_BIN_PATH may differ from the global
+       # constant when portage is reinstalling itself.
+       portage_bin_path = mysettings["PORTAGE_BIN_PATH"]
+       misc_sh_binary = os.path.join(portage_bin_path,
+               os.path.basename(MISC_SH_BINARY))
+       mycommand = " ".join([_shell_quote(misc_sh_binary)] + commands)
+       _doebuild_exit_status_unlink(
+               mysettings.get("EBUILD_EXIT_STATUS_FILE"))
+       filter_calling_env_state = mysettings._filter_calling_env
+       if os.path.exists(os.path.join(mysettings["T"], "environment")):
+               mysettings._filter_calling_env = True
+       debug = mysettings.get("PORTAGE_DEBUG") == "1"
+       logfile = mysettings.get("PORTAGE_LOG_FILE")
+       mydo = mysettings["EBUILD_PHASE"]
+       try:
+               rval = spawn(mycommand, mysettings, debug=debug,
+                       logfile=logfile, **kwargs)
+       finally:
+               mysettings._filter_calling_env = filter_calling_env_state
+       msg = _doebuild_exit_status_check(mydo, mysettings)
+       if msg:
+               rval = 1
+               from textwrap import wrap
+               from portage.elog.messages import eerror
+               for l in wrap(msg, 72):
+                       eerror(l, phase=mydo, key=mysettings.mycpv)
+       return rval
 
 _eapi_prefix_re = re.compile("(^|\s*)%s($|\s*)" % portage.const.EAPIPREFIX)
 _eapi_num_re    = re.compile("(^|\s*)[0-9]+($|\s*)")
@@ -4793,8 +4840,9 @@ _doebuild_broken_manifests = set()
 
 def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
        fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None,
-       mydbapi=None, vartree=None, prev_mtimes=None):
-       
+       mydbapi=None, vartree=None, prev_mtimes=None,
+       fd_pipes=None, returnpid=False):
+
        """
        Wrapper function that invokes specific ebuild phases through the spawning
        of ebuild.sh
@@ -5206,7 +5254,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                elif mydo == "setup":
                        retval = spawn(
                                _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings,
-                               debug=debug, free=1, logfile=logfile)
+                               debug=debug, free=1, logfile=logfile, fd_pipes=fd_pipes,
+                               returnpid=returnpid)
+                       if returnpid:
+                               return retval
                        retval = exit_status_check(retval)
                        if secpass >= 2:
                                """ Privileged phases may have left files that need to be made
@@ -5315,7 +5366,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                # unpack compile install`, we will try and fetch 4 times :/
                need_distfiles = (mydo in ("fetch", "unpack") or \
                        mydo not in ("digest", "manifest") and "noauto" not in features)
-               if need_distfiles and not fetch(
+               emerge_skip_distfiles = "EMERGE_FROM" in mysettings and \
+                       mydo not in ("fetch", "unpack")
+               if not emerge_skip_distfiles and \
+                       need_distfiles and not fetch(
                        fetchme, mysettings, listonly=listonly, fetchonly=fetchonly):
                        if have_build_dirs:
                                # Create an elog message for this fetch failure since the
@@ -5347,10 +5401,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                                return 1
 
                # See above comment about fetching only when needed
-               if not digestcheck(checkme, mysettings, ("strict" in features),
-                       (mydo not in ["digest","fetch","unpack"] and \
-                       mysettings.get("PORTAGE_CALLER", None) == "ebuild" and \
-                       "noauto" in features)):
+               if not emerge_skip_distfiles and \
+                       not digestcheck(checkme, mysettings, "strict" in features):
                        return 1
 
                if mydo == "fetch":
@@ -5435,7 +5487,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                                        raise portage.exception.PermissionDenied(
                                                "access('%s', os.W_OK)" % parent_dir)
                        retval = spawnebuild(mydo,
-                               actionmap, mysettings, debug, logfile=logfile)
+                               actionmap, mysettings, debug, logfile=logfile,
+                               fd_pipes=fd_pipes, returnpid=returnpid)
                elif mydo=="qmerge":
                        # check to ensure install was run.  this *only* pops up when users
                        # forget it and are using ebuild
@@ -5455,7 +5508,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                                mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes)
                elif mydo=="merge":
                        retval = spawnebuild("install", actionmap, mysettings, debug,
-                               alwaysdep=1, logfile=logfile)
+                               alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes,
+                               returnpid=returnpid)
                        retval = exit_status_check(retval)
                        if retval != os.EX_OK:
                                # The merge phase handles this already.  Callers don't know how