From 733bd2a458ab55ac2c465cf39c5ae0bc9e88abf4 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sat, 6 Jul 2013 19:41:37 -0700 Subject: [PATCH] doebuild: fix fd_pipes support, bug #475812 The fd_pipes was previously unsupported with returnpid=False. Now it's fixed to pass fd_pipes down to the appropriate places. --- pym/_emerge/EbuildPhase.py | 42 ++++--- pym/_emerge/SpawnProcess.py | 2 + pym/portage/dbapi/_MergeProcess.py | 4 +- pym/portage/dbapi/vartree.py | 5 +- pym/portage/package/ebuild/_spawn_nofetch.py | 4 +- pym/portage/package/ebuild/doebuild.py | 28 +++-- .../tests/ebuild/test_doebuild_fd_pipes.py | 118 ++++++++++++++++++ 7 files changed, 169 insertions(+), 34 deletions(-) create mode 100644 pym/portage/tests/ebuild/test_doebuild_fd_pipes.py diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py index 493134999..5c67f032b 100644 --- a/pym/_emerge/EbuildPhase.py +++ b/pym/_emerge/EbuildPhase.py @@ -39,7 +39,7 @@ from portage import _unicode_encode class EbuildPhase(CompositeTask): - __slots__ = ("actionmap", "phase", "settings") + \ + __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + \ ("_ebuild_lock",) # FEATURES displayed prior to setup phase @@ -157,8 +157,7 @@ class EbuildPhase(CompositeTask): return self._start_ebuild() - def _start_ebuild(self): - + def _get_log_path(self): # Don't open the log file during the clean phase since the # open file can result in an nfs lock on $T/build.log which # prevents the clean phase from removing $T. @@ -166,17 +165,21 @@ class EbuildPhase(CompositeTask): if self.phase not in ("clean", "cleanrm") and \ self.settings.get("PORTAGE_BACKGROUND") != "subprocess": logfile = self.settings.get("PORTAGE_LOG_FILE") + return logfile + + def _start_ebuild(self): - fd_pipes = None - if not self.background and self.phase == 'nofetch': - # All the pkg_nofetch output goes to stderr since - # it's considered to be an error message. - fd_pipes = {1 : sys.stderr.fileno()} + fd_pipes = self.fd_pipes + if fd_pipes is None: + if not self.background and self.phase == 'nofetch': + # All the pkg_nofetch output goes to stderr since + # it's considered to be an error message. + fd_pipes = {1 : sys.stderr.fileno()} ebuild_process = EbuildProcess(actionmap=self.actionmap, - background=self.background, fd_pipes=fd_pipes, logfile=logfile, - phase=self.phase, scheduler=self.scheduler, - settings=self.settings) + background=self.background, fd_pipes=fd_pipes, + logfile=self._get_log_path(), phase=self.phase, + scheduler=self.scheduler, settings=self.settings) self._start_task(ebuild_process, self._ebuild_exit) @@ -204,9 +207,7 @@ class EbuildPhase(CompositeTask): if not fail: self.returncode = None - logfile = None - if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": - logfile = self.settings.get("PORTAGE_LOG_FILE") + logfile = self._get_log_path() if self.phase == "install": out = io.StringIO() @@ -250,8 +251,9 @@ class EbuildPhase(CompositeTask): fd, logfile = tempfile.mkstemp() os.close(fd) post_phase = MiscFunctionsProcess(background=self.background, - commands=post_phase_cmds, logfile=logfile, phase=self.phase, - scheduler=self.scheduler, settings=settings) + commands=post_phase_cmds, fd_pipes=self.fd_pipes, + logfile=logfile, phase=self.phase, scheduler=self.scheduler, + settings=settings) self._start_task(post_phase, self._post_phase_exit) return @@ -326,8 +328,9 @@ class EbuildPhase(CompositeTask): self.returncode = None phase = 'die_hooks' die_hooks = MiscFunctionsProcess(background=self.background, - commands=[phase], phase=phase, - scheduler=self.scheduler, settings=self.settings) + commands=[phase], phase=phase, logfile=self._get_log_path(), + fd_pipes=self.fd_pipes, scheduler=self.scheduler, + settings=self.settings) self._start_task(die_hooks, self._die_hooks_exit) def _die_hooks_exit(self, die_hooks): @@ -346,7 +349,8 @@ class EbuildPhase(CompositeTask): portage.elog.elog_process(self.settings.mycpv, self.settings) phase = "clean" clean_phase = EbuildPhase(background=self.background, - phase=phase, scheduler=self.scheduler, settings=self.settings) + fd_pipes=fd_pipes, phase=phase, scheduler=self.scheduler, + settings=self.settings) self._start_task(clean_phase, self._fail_clean_exit) return diff --git a/pym/_emerge/SpawnProcess.py b/pym/_emerge/SpawnProcess.py index c57bd2e47..3b363a2a8 100644 --- a/pym/_emerge/SpawnProcess.py +++ b/pym/_emerge/SpawnProcess.py @@ -38,6 +38,8 @@ class SpawnProcess(SubProcess): if self.fd_pipes is None: self.fd_pipes = {} + else: + self.fd_pipes = self.fd_pipes.copy() fd_pipes = self.fd_pipes master_fd, slave_fd = self._pipe(fd_pipes) diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py index 76108c882..6c4104101 100644 --- a/pym/portage/dbapi/_MergeProcess.py +++ b/pym/portage/dbapi/_MergeProcess.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io @@ -53,6 +53,8 @@ class MergeProcess(ForkProcess): # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} + else: + self.fd_pipes = self.fd_pipes.copy() self.fd_pipes.setdefault(0, sys.stdin.fileno()) super(MergeProcess, self)._start() diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index e1a4753ce..bc17a59fc 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -4995,7 +4995,7 @@ class dblink(object): def merge(mycat, mypkg, pkgloc, infloc, myroot=None, settings=None, myebuild=None, mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None, - scheduler=None): + scheduler=None, fd_pipes=None): """ @param myroot: ignored, settings['EROOT'] is used instead """ @@ -5014,7 +5014,8 @@ def merge(mycat, mypkg, pkgloc, infloc, global_event_loop() or EventLoop(main=False)), background=background, blockers=blockers, pkgloc=pkgloc, infloc=infloc, myebuild=myebuild, mydbapi=mydbapi, - prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE')) + prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE'), + fd_pipes=fd_pipes) merge_task.start() retcode = merge_task.wait() return retcode diff --git a/pym/portage/package/ebuild/_spawn_nofetch.py b/pym/portage/package/ebuild/_spawn_nofetch.py index a8083824c..a8814a5bd 100644 --- a/pym/portage/package/ebuild/_spawn_nofetch.py +++ b/pym/portage/package/ebuild/_spawn_nofetch.py @@ -16,7 +16,7 @@ from portage.util._eventloop.EventLoop import EventLoop from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.EbuildPhase import EbuildPhase -def spawn_nofetch(portdb, ebuild_path, settings=None): +def spawn_nofetch(portdb, ebuild_path, settings=None, fd_pipes=None): """ This spawns pkg_nofetch if appropriate. The settings parameter is useful only if setcpv has already been called in order @@ -83,7 +83,7 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): phase='nofetch', scheduler=SchedulerInterface(portage._internal_caller and global_event_loop() or EventLoop(main=False)), - settings=settings) + fd_pipes=fd_pipes, settings=settings) ebuild_phase.start() ebuild_phase.wait() elog_process(settings.mycpv, settings) diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index 23ca2385f..1d44b93ab 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -139,14 +139,20 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): finally: settings.pop('EBUILD_PHASE', None) -def _spawn_phase(phase, settings, actionmap=None, **kwargs): - if kwargs.get('returnpid'): - return _doebuild_spawn(phase, settings, actionmap=actionmap, **kwargs) +def _spawn_phase(phase, settings, actionmap=None, returnpid=False, + logfile=None, **kwargs): + if returnpid: + return _doebuild_spawn(phase, settings, actionmap=actionmap, + returnpid=returnpid, logfile=logfile, **kwargs) + + # The logfile argument is unused here, since EbuildPhase uses + # the PORTAGE_LOG_FILE variable if set. ebuild_phase = EbuildPhase(actionmap=actionmap, background=False, phase=phase, scheduler=SchedulerInterface(portage._internal_caller and global_event_loop() or EventLoop(main=False)), - settings=settings) + settings=settings, **kwargs) + ebuild_phase.start() ebuild_phase.wait() return ebuild_phase.returncode @@ -499,9 +505,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, @param prev_mtimes: A dict of { filename:mtime } keys used by merge() to do config_protection @type prev_mtimes: dictionary @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } - for example. This is parameter only guaranteed to be respected when - returnpid is True (otherwise all subprocesses simply inherit file - descriptors from sys.__std* streams). + for example. @type fd_pipes: Dictionary @param returnpid: Return a list of process IDs for a successful spawn, or an integer value if spawn is unsuccessful. NOTE: This requires the @@ -957,7 +961,8 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, if not fetch(fetchme, mysettings, listonly=listonly, fetchonly=fetchonly, allow_missing_digests=True, digests=dist_digests): - spawn_nofetch(mydbapi, myebuild, settings=mysettings) + spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) if listonly: # The convention for listonly mode is to report # success in any case, even though fetch() may @@ -1089,7 +1094,8 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, - mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes) + mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes, + fd_pipes=fd_pipes) elif mydo=="merge": retval = spawnebuild("install", actionmap, mysettings, debug, alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes, @@ -1105,7 +1111,9 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, mydbapi=mydbapi, - vartree=vartree, prev_mtimes=prev_mtimes) + vartree=vartree, prev_mtimes=prev_mtimes, + fd_pipes=fd_pipes) + else: writemsg_stdout(_("!!! Unknown mydo: %s\n") % mydo, noiselevel=-1) return 1 diff --git a/pym/portage/tests/ebuild/test_doebuild_fd_pipes.py b/pym/portage/tests/ebuild/test_doebuild_fd_pipes.py new file mode 100644 index 000000000..1a5177003 --- /dev/null +++ b/pym/portage/tests/ebuild/test_doebuild_fd_pipes.py @@ -0,0 +1,118 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +import portage +from portage import os +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.package.ebuild._ipc.QueryCommand import QueryCommand +from portage.util._async.ForkProcess import ForkProcess +from portage.util._async.TaskScheduler import TaskScheduler +from portage.util._eventloop.global_event_loop import global_event_loop +from _emerge.Package import Package +from _emerge.PipeReader import PipeReader + +class DoebuildProcess(ForkProcess): + + __slots__ = ('doebuild_kwargs', 'doebuild_pargs') + + def _run(self): + return portage.doebuild(*self.doebuild_pargs, **self.doebuild_kwargs) + +class DoebuildFdPipesTestCase(TestCase): + """ + Invoke portage.package.ebuild.doebuild.spawn() with a + minimal environment. This gives coverage to some of + the ebuild execution internals, like ebuild.sh, + AbstractEbuildProcess, and EbuildIpcDaemon. + """ + + def testDoebuild(self): + + ebuild_body = textwrap.dedent(""" + S=${WORKDIR} + pkg_pretend() { echo pretend ; } + pkg_setup() { echo setup ; } + src_unpack() { echo unpack ; } + src_prepare() { echo prepare ; } + src_configure() { echo configure ; } + src_compile() { echo compile ; } + src_test() { echo test ; } + src_install() { echo install ; } + """) + + ebuilds = { + 'app-misct/foo-1': { + 'EAPI' : '5', + "MISC_CONTENT": ebuild_body, + } + } + + playground = ResolverPlayground(ebuilds=ebuilds) + try: + QueryCommand._db = playground.trees + root_config = playground.trees[playground.eroot]['root_config'] + portdb = root_config.trees["porttree"].dbapi + settings = portage.config(clone=playground.settings) + if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: + settings["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ + os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] + settings.backup_changes("__PORTAGE_TEST_HARDLINK_LOCKS") + + settings.features.add("noauto") + settings.features.add("test") + settings['PORTAGE_PYTHON'] = portage._python_interpreter + settings['PORTAGE_QUIET'] = "1" + + cpv = 'app-misct/foo-1' + metadata = dict(zip(Package.metadata_keys, + portdb.aux_get(cpv, Package.metadata_keys))) + + pkg = Package(built=False, cpv=cpv, installed=False, + metadata=metadata, root_config=root_config, + type_name='ebuild') + settings.setcpv(pkg) + ebuildpath = portdb.findname(cpv) + self.assertNotEqual(ebuildpath, None) + + for phase in ('pretend', 'setup', 'unpack', 'prepare', 'configure', + 'compile', 'test', 'install', 'clean', 'merge'): + + pr, pw = os.pipe() + + producer = DoebuildProcess(doebuild_pargs=(ebuildpath, phase), + doebuild_kwargs={"settings" : settings, + "mydbapi": portdb, "tree": "porttree", + "vartree": root_config.trees["vartree"], + "fd_pipes": {1: pw, 2: pw}, + "prev_mtimes": {}}) + + consumer = PipeReader( + input_files={"producer" : pr}) + + task_scheduler = TaskScheduler(iter([producer, consumer]), + max_jobs=2) + + try: + task_scheduler.start() + finally: + # PipeReader closes pr + os.close(pw) + + task_scheduler.wait() + output = portage._unicode_decode( + consumer.getvalue()).rstrip("\n") + + if task_scheduler.returncode != os.EX_OK: + portage.writemsg(output, noiselevel=-1) + + self.assertEqual(task_scheduler.returncode, os.EX_OK) + + if phase not in ('clean', 'merge'): + self.assertEqual(phase, output) + + finally: + playground.cleanup() + QueryCommand._db = None -- 2.26.2