From: Fabian Groffen Date: Mon, 14 Jul 2008 19:20:17 +0000 (-0000) Subject: Merged from trunk 11041:11047 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=d472130511b0d3799ba520a169108c713a5f7f1e;p=portage.git Merged from trunk 11041:11047 | 11042 | Ensure that that Scheduler._main_loop() doesn't return | | zmedico | before the merge queue is empty. | | 11043 | Remove redundant failed_packages check from | | zmedico | Scheduler._main_loop() since it's now handled inside the | | | _schedule() call. | | 11044 | Add async execution support for pkg_preinst and | | zmedico | pkg_postinst. This allows the scheduler's poll loop to run | | | so that other parallel tasks aren't starved for output | | | handling while pkg_preinst and pkg_postinst are executing. | | 11045 | * Fix doebuild so "returnpid" works for preinst and | | zmedico | postinst. * Add async support for pkg_prerm and pkg_postrm. | | | * Add missing "treetype" parameter to dblink constructor | | | calls. * Fix SubProcess._wait() to check self.returncode | | | _after_ calling the scheduler. | | 11046 | Call prepare_build_dirs() prior to pkg_preinst so that | | zmedico | logging works properly. | | 11047 | Unregister from poll() objects before closing file in order | | zmedico | to hopefully avoid stale events that can trigger a KeyError | | | in Scheduler._poll_loop() if the event's file descriptor no | | | longer has a registered handler. | svn path=/main/branches/prefix/; revision=11048 --- diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 3d4296fa3..736c6efe8 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -754,6 +754,17 @@ class search(object): class RootConfig(object): """This is used internally by depgraph to track information about a particular $ROOT.""" + + pkg_tree_map = { + "ebuild" : "porttree", + "binary" : "bintree", + "installed" : "vartree" + } + + tree_pkg_map = {} + for k, v in pkg_tree_map.iteritems(): + tree_pkg_map[v] = k + def __init__(self, settings, trees, setconfig): self.trees = trees self.settings = settings @@ -1742,11 +1753,13 @@ class PipeReader(AsynchronousTask): if buf: self._read_data.append(buf.tostring()) else: - for f in files.values(): - f.close() self.registered = False for reg_id in self._reg_ids: self.scheduler.unregister(reg_id) + + for f in files.values(): + f.close() + self.wait() return self.registered @@ -1947,10 +1960,10 @@ class SubProcess(AsynchronousTask): self.returncode is None def _wait(self): - if self.returncode is not None: - return self.returncode if self.registered: self.scheduler.schedule(self._reg_id) + if self.returncode is not None: + return self.returncode try: wait_retval = os.waitpid(self.pid, 0) except OSError, e: @@ -2085,12 +2098,12 @@ class SpawnProcess(SubProcess): buf.tofile(files.out) files.out.flush() else: - fd = files.process.fileno() - for f in files.values(): - f.flush() - f.close() self.registered = False self.scheduler.unregister(self._reg_id) + + for f in files.values(): + f.close() + self.wait() return self.registered @@ -2109,11 +2122,12 @@ class SpawnProcess(SubProcess): if buf: pass else: - fd = files.process.fileno() - for f in files.values(): - f.close() self.registered = False self.scheduler.unregister(self._reg_id) + + for f in files.values(): + f.close() + self.wait() return self.registered @@ -2436,6 +2450,7 @@ class EbuildBuild(CompositeTask): merge = EbuildMerge(find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, + scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ @@ -2577,10 +2592,12 @@ class EbuildMetadataPhase(SubProcess): files = self.files self._raw_metadata.append(files.ebuild.read()) if not self._raw_metadata[-1]: - for f in files.values(): - f.close() self.registered = False self.scheduler.unregister(self._reg_id) + + for f in files.values(): + f.close() + self.wait() if self.returncode == os.EX_OK: @@ -2702,11 +2719,12 @@ class EbuildPhase(SubProcess): buf.tofile(files.log) files.log.flush() else: - fd = files.ebuild.fileno() - for f in files.values(): - f.close() self.registered = False self.scheduler.unregister(self._reg_id) + + for f in files.values(): + f.close() + self.wait() return self.registered @@ -2725,33 +2743,62 @@ class EbuildPhase(SubProcess): if buf: pass else: - fd = files.ebuild.fileno() - for f in files.values(): - f.close() self.registered = False self.scheduler.unregister(self._reg_id) + + for f in files.values(): + f.close() + self.wait() return self.registered def _set_returncode(self, wait_retval): SubProcess._set_returncode(self, wait_retval) - if self.phase != "clean": - msg = portage._doebuild_exit_status_check( - self.phase, self.settings) - if msg: - self.returncode = 1 - from textwrap import wrap - from portage.elog.messages import eerror - for l in wrap(msg, 72): - eerror(l, phase=self.phase, key=self.pkg.cpv) settings = self.settings + debug = settings.get("PORTAGE_DEBUG") == "1" + log_path = settings.get("PORTAGE_LOG_FILE") + + if self.phase != "clean": + self.returncode = portage._doebuild_exit_status_check_and_log( + settings, self.phase, self.returncode) + portage._post_phase_userpriv_perms(settings) if self.phase == "install": portage._check_build_log(settings) if self.returncode == os.EX_OK: self.returncode = portage._post_src_install_checks(settings) + elif self.phase == "preinst": + + if self.returncode == os.EX_OK: + portage._doebuild_exit_status_unlink( + settings.get("EBUILD_EXIT_STATUS_FILE")) + phase_retval = portage.spawn( + " ".join(portage._post_pkg_preinst_cmd(settings)), + settings, debug=debug, free=1, logfile=log_path) + phase_retval = portage._doebuild_exit_status_check_and_log( + settings, self.phase, phase_retval) + if phase_retval != os.EX_OK: + writemsg("!!! post preinst failed; exiting.\n", + noiselevel=-1) + self.returncode = phase_retval + + elif self.phase == "postinst": + + if self.returncode == os.EX_OK: + portage._doebuild_exit_status_unlink( + settings.get("EBUILD_EXIT_STATUS_FILE")) + phase_retval = portage.spawn( + " ".join(portage._post_pkg_postinst_cmd(settings)), + settings, debug=debug, free=1, logfile=log_path) + phase_retval = portage._doebuild_exit_status_check_and_log( + settings, self.phase, phase_retval) + if phase_retval != os.EX_OK: + writemsg("!!! post postinst failed; exiting.\n", + noiselevel=-1) + self.returncode = phase_retval + class EbuildBinpkg(EbuildPhase): """ This assumes that src_install() has successfully completed. @@ -2794,7 +2841,7 @@ class EbuildMerge(SlotObject): __slots__ = ("find_blockers", "logger", "ldpath_mtimes", "pkg", "pkg_count", "pkg_path", "pretend", - "settings", "tree", "world_atom") + "scheduler", "settings", "tree", "world_atom") def execute(self): root_config = self.pkg.root_config @@ -2807,6 +2854,7 @@ class EbuildMerge(SlotObject): mytree=self.tree, mydbapi=root_config.trees[self.tree].dbapi, vartree=root_config.trees["vartree"], prev_mtimes=self.ldpath_mtimes, + scheduler=self.scheduler, blockers=self.find_blockers) if retval == os.EX_OK: @@ -3099,8 +3147,8 @@ class Binpkg(CompositeTask): merge = EbuildMerge(find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, - pkg_path=self._pkg_path, settings=settings, - tree=self._tree, world_atom=self.world_atom) + pkg_path=self._pkg_path, scheduler=self.scheduler, + settings=settings, tree=self._tree, world_atom=self.world_atom) try: retval = merge.execute() @@ -3870,10 +3918,7 @@ class PackageVirtualDbapi(portage.dbapi): class depgraph(object): - pkg_tree_map = { - "ebuild":"porttree", - "binary":"bintree", - "installed":"vartree"} + pkg_tree_map = RootConfig.pkg_tree_map _mydbapi_keys = Package.metadata_keys @@ -8285,7 +8330,8 @@ class Scheduler(PollScheduler): _fetch_log = EPREFIX + "/var/log/emerge-fetch.log" class _iface_class(SlotObject): - __slots__ = ("fetch", "register", "schedule", "unregister") + __slots__ = ("dblinkEbuildPhase", "fetch", + "register", "schedule", "unregister") class _fetch_iface_class(SlotObject): __slots__ = ("log_file", "schedule") @@ -8348,6 +8394,7 @@ class Scheduler(PollScheduler): fetch_iface = self._fetch_iface_class(log_file=self._fetch_log, schedule=self._schedule_fetch) self._sched_iface = self._iface_class( + dblinkEbuildPhase=self._dblink_ebuild_phase, fetch=fetch_iface, register=self._register, schedule=self._schedule_wait, unregister=self._unregister) @@ -8521,6 +8568,55 @@ class Scheduler(PollScheduler): return blocker_dblinks + def _dblink_pkg(self, pkg_dblink): + cpv = pkg_dblink.mycpv + type_name = RootConfig.tree_pkg_map[pkg_dblink.treetype] + root_config = self.trees[pkg_dblink.myroot]["root_config"] + installed = type_name == "installed" + return self._pkg(cpv, type_name, root_config, installed=installed) + + def _append_to_log_path(self, log_path, msg): + f = open(log_path, 'a') + try: + f.write(msg) + finally: + f.close() + + def _dblink_ebuild_phase(self, + pkg_dblink, pkg_dbapi, ebuild_path, phase): + """ + Using this callback for merge phases allows the scheduler + to run while these phases execute asynchronously, and allows + the scheduler control output handling. + """ + + scheduler = self._sched_iface + settings = pkg_dblink.settings + pkg = self._dblink_pkg(pkg_dblink) + background = self._max_jobs > 1 + log_path = settings.get("PORTAGE_LOG_FILE") + + if phase == "preinst": + msg = ">>> Merging %s to %s\n" % (pkg.cpv, pkg.root) + portage.writemsg_stdout(msg) + if log_path is not None: + self._append_to_log_path(log_path, msg) + + ebuild_phase = EbuildPhase(background=background, + pkg=pkg, phase=phase, scheduler=scheduler, + settings=settings, tree=pkg_dblink.treetype) + ebuild_phase.start() + ebuild_phase.wait() + + if phase == "postinst" and \ + ebuild_phase.returncode == os.EX_OK: + msg = ">>> %s %s\n" % (pkg.cpv, "merged.") + portage.writemsg_stdout(msg) + if log_path is not None: + self._append_to_log_path(log_path, msg) + + return ebuild_phase.returncode + def _check_manifests(self): # Verify all the manifests now so that the user is notified of failure # as soon as possible. @@ -8920,11 +9016,12 @@ class Scheduler(PollScheduler): if self._is_restart_scheduled(): self._set_max_jobs(1) - while not self._failed_pkgs and \ - self._schedule(): + merge_queue = self._task_queues.merge + + while self._schedule(): self._poll_loop() - while self._jobs: + while self._jobs or merge_queue: self._poll_loop() def _schedule_tasks(self): diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 18af5af4c..241e74f64 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4400,6 +4400,42 @@ def _post_src_install_uid_fix(mysettings): mode=mystat.st_mode, stat_cached=mystat, follow_links=False) +def _post_pkg_preinst_cmd(mysettings): + """ + Post phase logic and tasks that have been factored out of + ebuild.sh. Call preinst_mask last so that INSTALL_MASK can + can be used to wipe out any gmon.out files created during + previous functions (in case any tools were built with -pg + in CFLAGS). + """ + + portage_bin_path = mysettings["PORTAGE_BIN_PATH"] + misc_sh_binary = os.path.join(portage_bin_path, + os.path.basename(MISC_SH_BINARY)) + + mysettings["EBUILD_PHASE"] = "" + myargs = [_shell_quote(misc_sh_binary), + "preinst_bsdflags", + "preinst_sfperms", "preinst_selinux_labels", + "preinst_suid_scan", "preinst_mask"] + + return myargs + +def _post_pkg_postinst_cmd(mysettings): + """ + Post phase logic and tasks that have been factored out of + build.sh. + """ + + portage_bin_path = mysettings["PORTAGE_BIN_PATH"] + misc_sh_binary = os.path.join(portage_bin_path, + os.path.basename(MISC_SH_BINARY)) + + mysettings["EBUILD_PHASE"] = "" + myargs = [_shell_quote(misc_sh_binary), "postinst_bsdflags"] + + return myargs + def _spawn_misc_sh(mysettings, commands, **kwargs): """ @param mysettings: the ebuild config @@ -4871,6 +4907,18 @@ def _doebuild_exit_status_check(mydo, settings): "errors (bug #200313)." return msg +def _doebuild_exit_status_check_and_log(settings, mydo, retval): + if retval != os.EX_OK: + return retval + msg = _doebuild_exit_status_check(mydo, settings) + if msg: + retval = 1 + from textwrap import wrap + from portage.elog.messages import eerror + for l in wrap(msg, 72): + eerror(l, phase=mydo, key=settings.mycpv) + return retval + def _doebuild_exit_status_unlink(exit_status_file): """ Double check to make sure it really doesn't exist @@ -5326,22 +5374,18 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, elif mydo == "preinst": phase_retval = spawn( _shell_quote(ebuild_sh_binary) + " " + mydo, - mysettings, debug=debug, free=1, logfile=logfile) + mysettings, debug=debug, free=1, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) + + if returnpid: + return phase_retval + phase_retval = exit_status_check(phase_retval) if phase_retval == os.EX_OK: - # Post phase logic and tasks that have been factored out of - # ebuild.sh. Call preinst_mask last so that INSTALL_MASK can - # can be used to wipe out any gmon.out files created during - # previous functions (in case any tools were built with -pg - # in CFLAGS). - myargs = [_shell_quote(misc_sh_binary), - "preinst_bsdflags", - "preinst_sfperms", "preinst_selinux_labels", - "preinst_suid_scan", "preinst_mask"] _doebuild_exit_status_unlink( mysettings.get("EBUILD_EXIT_STATUS_FILE")) - mysettings["EBUILD_PHASE"] = "" - phase_retval = spawn(" ".join(myargs), + phase_retval = spawn( + " ".join(_post_pkg_preinst_cmd(mysettings)), mysettings, debug=debug, free=1, logfile=logfile) phase_retval = exit_status_check(phase_retval) if phase_retval != os.EX_OK: @@ -5351,16 +5395,17 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, elif mydo == "postinst": phase_retval = spawn( _shell_quote(ebuild_sh_binary) + " " + mydo, - mysettings, debug=debug, free=1, logfile=logfile) + mysettings, debug=debug, free=1, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) + + if returnpid: + return phase_retval + phase_retval = exit_status_check(phase_retval) if phase_retval == os.EX_OK: - # Post phase logic and tasks that have been factored out of - # ebuild.sh. - myargs = [_shell_quote(misc_sh_binary), "postinst_bsdflags"] _doebuild_exit_status_unlink( mysettings.get("EBUILD_EXIT_STATUS_FILE")) - mysettings["EBUILD_PHASE"] = "" - phase_retval = spawn(" ".join(myargs), + phase_retval = spawn(" ".join(_post_pkg_postinst_cmd(mysettings)), mysettings, debug=debug, free=1, logfile=logfile) phase_retval = exit_status_check(phase_retval) if phase_retval != os.EX_OK: @@ -5370,7 +5415,12 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, elif mydo in ("prerm", "postrm", "config", "info"): retval = spawn( _shell_quote(ebuild_sh_binary) + " " + mydo, - mysettings, debug=debug, free=1, logfile=logfile) + mysettings, debug=debug, free=1, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) + + if returnpid: + return retval + retval = exit_status_check(retval) return retval @@ -5776,13 +5826,14 @@ def movefile(src,dest,newmtime=None,sstat=None,mysettings=None): return newmtime def merge(mycat, mypkg, pkgloc, infloc, myroot, mysettings, myebuild=None, - mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None): + mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None, + scheduler=None): if not os.access(myroot + EPREFIX_LSTRIP, os.W_OK): writemsg("Permission denied: access('%s', W_OK)\n" % (myroot + EPREFIX_LSTRIP), noiselevel=-1) return errno.EACCES mylink = dblink(mycat, mypkg, myroot, mysettings, treetype=mytree, - vartree=vartree, blockers=blockers) + vartree=vartree, blockers=blockers, scheduler=scheduler) return mylink.merge(pkgloc, infloc, myroot, myebuild, mydbapi=mydbapi, prev_mtimes=prev_mtimes) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 79d49705e..360d303ad 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1035,7 +1035,7 @@ class vardbapi(dbapi): def _dblink(self, cpv): category, pf = catsplit(cpv) return dblink(category, pf, self.root, - self.settings, vartree=self.vartree) + self.settings, vartree=self.vartree, treetype="vartree") def removeFromContents(self, pkg, paths, relative_paths=True): """ @@ -1422,7 +1422,7 @@ class dblink(object): } def __init__(self, cat, pkg, myroot, mysettings, treetype=None, - vartree=None, blockers=None): + vartree=None, blockers=None, scheduler=None): """ Creates a DBlink object for a given CPV. The given CPV may not be present in the database already. @@ -1452,6 +1452,7 @@ class dblink(object): vartree = db[myroot]["vartree"] self.vartree = vartree self._blockers = blockers + self._scheduler = scheduler self.dbroot = normalize_path(os.path.join(myroot, VDB_PATH)) self.dbcatdir = self.dbroot+"/"+cat @@ -1662,7 +1663,9 @@ class dblink(object): if cur_cpv == self.mycpv: continue others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], - self.vartree.root, self.settings, vartree=self.vartree)) + self.vartree.root, self.settings, vartree=self.vartree, + treetype="vartree")) + retval = self._security_check([self] + others_in_slot) if retval: return retval @@ -1699,6 +1702,7 @@ class dblink(object): uid=portage_uid, gid=portage_gid, mode=070, mask=0) builddir_lock = None catdir_lock = None + scheduler = self._scheduler retval = -1 try: if myebuildpath: @@ -1712,11 +1716,18 @@ class dblink(object): unlockdir(catdir_lock) finally: catdir_lock = None - # Eventually, we'd like to pass in the saved ebuild env here... - retval = doebuild(myebuildpath, "prerm", self.myroot, - self.settings, cleanup=cleanup, use_cache=0, - mydbapi=self.vartree.dbapi, tree="vartree", - vartree=self.vartree) + + prepare_build_dirs(self.myroot, self.settings, 1) + + if scheduler is None: + retval = doebuild(myebuildpath, ebuild_phase, self.myroot, + self.settings, cleanup=cleanup, use_cache=0, + mydbapi=self.vartree.dbapi, tree=self.treetype, + vartree=self.vartree) + else: + retval = scheduler.dblinkEbuildPhase( + self, self.vartree.dbapi, myebuildpath, ebuild_phase) + # XXX: Decide how to handle failures here. if retval != os.EX_OK: writemsg("!!! FAILED prerm: %s\n" % retval, noiselevel=-1) @@ -1731,9 +1742,13 @@ class dblink(object): if myebuildpath: ebuild_phase = "postrm" - retval = doebuild(myebuildpath, "postrm", self.myroot, - self.settings, use_cache=0, tree="vartree", - mydbapi=self.vartree.dbapi, vartree=self.vartree) + if scheduler is None: + retval = doebuild(myebuildpath, ebuild_phase, self.myroot, + self.settings, use_cache=0, tree=self.treetype, + mydbapi=self.vartree.dbapi, vartree=self.vartree) + else: + retval = scheduler.dblinkEbuildPhase( + self, self.vartree.dbapi, myebuildpath, ebuild_phase) # XXX: Decide how to handle failures here. if retval != os.EX_OK: @@ -1890,7 +1905,8 @@ class dblink(object): continue others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], self.vartree.root, self.settings, - vartree=self.vartree)) + vartree=self.vartree, treetype="vartree")) + dest_root = normalize_path(self.vartree.root).rstrip(os.path.sep) + \ os.path.sep dest_root_len = len(dest_root) - 1 @@ -2522,7 +2538,9 @@ class dblink(object): # we need it to have private ${T} etc... for things like elog. others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], self.vartree.root, config(clone=self.settings), - vartree=self.vartree)) + vartree=self.vartree, treetype="vartree", + scheduler=self._scheduler)) + retval = self._security_check(others_in_slot) if retval: return retval @@ -2693,8 +2711,6 @@ class dblink(object): if collision_protect: return 1 - writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot)) - # The merge process may move files out of the image directory, # which causes invalidation of the .installed flag. try: @@ -2709,10 +2725,17 @@ class dblink(object): self.delete() ensure_dirs(self.dbtmpdir) + scheduler = self._scheduler + # run preinst script - a = doebuild(myebuild, "preinst", destroot, self.settings, - use_cache=0, tree=self.treetype, mydbapi=mydbapi, - vartree=self.vartree) + if scheduler is None: + writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot)) + a = doebuild(myebuild, "preinst", destroot, self.settings, + use_cache=0, tree=self.treetype, mydbapi=mydbapi, + vartree=self.vartree) + else: + a = scheduler.dblinkEbuildPhase( + self, mydbapi, myebuild, "preinst") # XXX: Decide how to handle failures here. if a != os.EX_OK: @@ -2865,9 +2888,18 @@ class dblink(object): self.settings["PORTAGE_UPDATE_ENV"] = \ os.path.join(self.dbpkgdir, "environment.bz2") self.settings.backup_changes("PORTAGE_UPDATE_ENV") - a = doebuild(myebuild, "postinst", destroot, self.settings, use_cache=0, - tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) - self.settings.pop("PORTAGE_UPDATE_ENV", None) + try: + if scheduler is None: + a = doebuild(myebuild, "postinst", destroot, self.settings, + use_cache=0, tree=self.treetype, mydbapi=mydbapi, + vartree=self.vartree) + if a == os.EX_OK: + writemsg_stdout(">>> %s %s\n" % (self.mycpv, "merged.")) + else: + a = scheduler.dblinkEbuildPhase( + self, mydbapi, myebuild, "postinst") + finally: + self.settings.pop("PORTAGE_UPDATE_ENV", None) # XXX: Decide how to handle failures here. if a != os.EX_OK: @@ -2884,7 +2916,6 @@ class dblink(object): target_root=self.settings["ROOT"], prev_mtimes=prev_mtimes, contents=contents, env=self.settings.environ()) - writemsg_stdout(">>> %s %s\n" % (self.mycpv,"merged.")) return os.EX_OK def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime):