SpawnProcess: improve dummy pipe allocation logic
authorZac Medico <zmedico@gentoo.org>
Tue, 15 Jan 2013 20:09:21 +0000 (12:09 -0800)
committerZac Medico <zmedico@gentoo.org>
Tue, 15 Jan 2013 20:09:21 +0000 (12:09 -0800)
By using allocated file descriptors for keys in fd_pipes, we naturally
avoid interference with callers such as FileDigester and MergeProcess.
See the _setup_pipes docstring for more benefits of this allocation
approach.

bin/ebuild.sh
pym/_emerge/EbuildProcess.py
pym/_emerge/EbuildSpawnProcess.py
pym/_emerge/MiscFunctionsProcess.py
pym/_emerge/SpawnProcess.py
pym/_emerge/SubProcess.py
pym/portage/package/ebuild/_config/special_env_vars.py
pym/portage/process.py

index 80bdd800196cf9a873cbc731d4b5c0e2455de975..a4be7c0677cfc508dc1056bc1ac4d643b19831b8 100755 (executable)
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 1999-2012 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 PORTAGE_BIN_PATH="${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"
@@ -713,7 +713,10 @@ else
                (
                        # Don't allow subprocesses to inherit the pipe which
                        # emerge uses to monitor ebuild.sh.
-                       exec 9>&-
+                       if [[ -n ${PORTAGE_PIPE_FD} ]] ; then
+                               eval "exec ${PORTAGE_PIPE_FD}>&-"
+                               unset PORTAGE_PIPE_FD
+                       fi
                        __ebuild_main ${EBUILD_SH_ARGS}
                        exit 0
                )
index ce97aff0f3c813fb73bc4f515634ec4e63a67df3..333ad7bd0aaa07fcfef7d4a8a671df77460b3a95 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright 1999-2010 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess
@@ -17,5 +17,11 @@ class EbuildProcess(AbstractEbuildProcess):
                if actionmap is None:
                        actionmap = _spawn_actionmap(self.settings)
 
-               return _doebuild_spawn(self.phase, self.settings,
-                               actionmap=actionmap, **kwargs)
+               if self._dummy_pipe_fd is not None:
+                       self.settings["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd)
+
+               try:
+                       return _doebuild_spawn(self.phase, self.settings,
+                                       actionmap=actionmap, **kwargs)
+               finally:
+                       self.settings.pop("PORTAGE_PIPE_FD", None)
index e1f682a664c2071d03394eba6546a2c7f7615286..26d26fc7720ab777d04599f5f5011e78d7a74000 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright 2010 Gentoo Foundation
+# Copyright 2010-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess
@@ -13,4 +13,10 @@ class EbuildSpawnProcess(AbstractEbuildProcess):
        __slots__ = ('fakeroot_state', 'spawn_func')
 
        def _spawn(self, args, **kwargs):
-               return self.spawn_func(args, env=self.settings.environ(), **kwargs)
+
+               env = self.settings.environ()
+
+               if self._dummy_pipe_fd is not None:
+                       env["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd)
+
+               return self.spawn_func(args, env=env, **kwargs)
index afa44fb2ad0d98aa9e2caef50ff75cfae0ccca15..bada79d861e9346a138966861fe1588c5ec0c3a8 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess
@@ -29,6 +29,10 @@ class MiscFunctionsProcess(AbstractEbuildProcess):
                AbstractEbuildProcess._start(self)
 
        def _spawn(self, args, **kwargs):
+
+               if self._dummy_pipe_fd is not None:
+                       self.settings["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd)
+
                # Temporarily unset EBUILD_PHASE so that bashrc code doesn't
                # think this is a real phase.
                phase_backup = self.settings.pop("EBUILD_PHASE", None)
@@ -37,3 +41,4 @@ class MiscFunctionsProcess(AbstractEbuildProcess):
                finally:
                        if phase_backup is not None:
                                self.settings["EBUILD_PHASE"] = phase_backup
+                       self.settings.pop("PORTAGE_PIPE_FD", None)
index 9c754add54499a4484fa67f13645de1e7961e741..ebba7d3d4051586951006f1d7b39c7c0d98d1258 100644 (file)
@@ -70,23 +70,20 @@ class SpawnProcess(SubProcess):
 
                fd_pipes_orig = fd_pipes.copy()
 
-               if log_file_path is not None:
+               if log_file_path is not None or self.background:
                        fd_pipes[1] = slave_fd
                        fd_pipes[2] = slave_fd
 
                else:
-                       # Create a dummy pipe so the scheduler can monitor
-                       # the process from inside a poll() loop. Ensure that
-                       # it doesn't interfere with a random fd that's already
-                       # in fd_pipes though (as least FileDigester can pass
-                       # in a random fd returned from os.pipe()).
-                       unique_dummy_fd = self._dummy_pipe_fd
-                       while unique_dummy_fd in fd_pipes:
-                               unique_dummy_fd += 1
-                       fd_pipes[unique_dummy_fd] = slave_fd
-                       if self.background:
-                               fd_pipes[1] = slave_fd
-                               fd_pipes[2] = slave_fd
+                       # Create a dummy pipe that PipeLogger uses to efficiently
+                       # monitors for process exit by listening for the EOF event.
+                       # Re-use of the allocated fd number for the key in fd_pipes
+                       # guarantees that the keys will not collide for similarly
+                       # allocated pipes which are used by callers such as
+                       # FileDigester and MergeProcess. See the _setup_pipes
+                       # docstring for more benefits of this allocation approach.
+                       self._dummy_pipe_fd = slave_fd
+                       fd_pipes[slave_fd] = slave_fd
 
                kwargs = {}
                for k in self._spawn_kwarg_names:
index 92cbc27ab4f3b3abec5cc7a8cac5bb0b294da6c5..4ccf9164f8682eda96a7b8330e93cc391185b431 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright 1999-2012 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from portage import os
@@ -9,12 +9,7 @@ import errno
 class SubProcess(AbstractPollTask):
 
        __slots__ = ("pid",) + \
-               ("_files", "_reg_id")
-
-       # A file descriptor is required for the scheduler to monitor changes from
-       # inside a poll() loop. When logging is not enabled, create a pipe just to
-       # serve this purpose alone.
-       _dummy_pipe_fd = 9
+               ("_dummy_pipe_fd", "_files", "_reg_id")
 
        # This is how much time we allow for waitpid to succeed after
        # we've sent a kill signal to our subprocess.
index 578e417f2248e490f83b2536e1d7ffe7776880fb..93bb98a428e047a600bd61aa1a9de0cba4e1e112 100644 (file)
@@ -22,7 +22,7 @@ env_blacklist = frozenset((
        "PDEPEND", "PF", "PKGUSE", "PORTAGE_BACKGROUND",
        "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDDIR_LOCKED",
        "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", "PORTAGE_IUSE",
-       "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME",
+       "PORTAGE_NONFATAL", "PORTAGE_PIPE_FD", "PORTAGE_REPO_NAME",
        "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "REPOSITORY",
        "RESTRICT", "ROOT", "SLOT", "SRC_URI"
 ))
@@ -63,7 +63,7 @@ environ_whitelist += [
        "PORTAGE_GID", "PORTAGE_GRPNAME",
        "PORTAGE_INST_GID", "PORTAGE_INST_UID",
        "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE",
-       "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX",
+       "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", "PORTAGE_PIPE_FD",
        "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", "PORTAGE_QUIET",
        "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT",
        "PORTAGE_SIGPIPE_STATUS",
index 4cf1cec60c7aef9e9f45fe74ca113efc7c545827..23a9094669b76b7742a83c704dfeac4462c166c2 100644 (file)
@@ -429,6 +429,21 @@ def _setup_pipes(fd_pipes, close_fds=True):
        and also with CPython under some circumstances (as triggered
        by xmpppy in bug #374335). In order to close a safe subset of
        file descriptors, see portage.locks._close_fds().
+
+       NOTE: When not followed by exec, even when close_fds is False,
+       it's still possible for dup2() calls to cause interference in a
+       way that's similar to the way that close_fds interferes (since
+       dup2() has to close the target fd if it happens to be open).
+       It's possible to avoid such interference by using allocated
+       file descriptors as the keys in fd_pipes. For example:
+
+               pr, pw = os.pipe()
+               fd_pipes[pw] = pw
+
+       By using the allocated pw file descriptor as the key in fd_pipes,
+       it's not necessary for dup2() to close a file descriptor (it
+       actually does nothing in this case), which avoids possible
+       interference.
        """
        my_fds = {}
        # To protect from cases where direct assignment could