From 24639aad995591c061b9571a8f305aa7e477b23c Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 2 Jul 2008 10:49:22 +0000 Subject: [PATCH] Merged from trunk 10884:10892 | 10885 | * Rename existing EbuildBuild class to EbuildExecuter. * | | zmedico | Split more code out of Scheduler._execute_task() and use it | | | to make a more comprehensive EbuildBuild class. | | 10886 | Remove debugging code from the previous commit. | | zmedico | | | 10887 | Split out a Binpkg class from code inside | | zmedico | Scheduler._execute_task(). | | 10888 | Fix inverted buildpkgonly logic. | | zmedico | | | 10889 | Fix EbuildBuild.execute() to return os.EX_OK on success. | | zmedico | | | 10890 | Py3k compatibility patch by Ali Polatel . Don't | | zmedico | use the format raise Exception, "string" | | 10891 | Split out a SubProcess base class for EbuildFetcherAsync, | | zmedico | EbuildPhase, and BinpkgFetcherAsync. | | 10892 | Remove $PORTAGE_BUILDDIR locking from portage.pkgmerge() and | | zmedico | use an instance of EbuildBuildDir instead. | svn path=/main/branches/prefix/; revision=10893 --- DEVELOPING | 12 + pym/_emerge/__init__.py | 629 +++++++++++++++++++--------------- pym/portage/__init__.py | 30 +- pym/portage/checksum.py | 5 +- pym/portage/dbapi/porttree.py | 12 +- pym/portage/dep.py | 2 +- pym/portage/getbinpkg.py | 11 +- pym/portage/gpg.py | 46 ++- pym/portage/locks.py | 13 +- 9 files changed, 423 insertions(+), 337 deletions(-) diff --git a/DEVELOPING b/DEVELOPING index cd9f78f94..8286a8bcf 100644 --- a/DEVELOPING +++ b/DEVELOPING @@ -94,6 +94,18 @@ except KeyError: The get call is nicer (compact) and faster (try,except are slow). +Exceptions +---------- + +Don't use the format raise Exception, "string" +It will be removed in py3k. + +YES: + raise KeyError("No key") + +NO: + raise KeyError, "No key" + Imports ------- diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 4a9b9c572..64940f630 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -66,6 +66,7 @@ bad = create_color_func("BAD") # white looks bad on terminals with white background from portage.output import bold as white +import portage.elog import portage.dep portage.dep._dep_check_strict = True import portage.util @@ -1466,7 +1467,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): class EbuildFetcher(SlotObject): - __slots__ = ("fetch_all", "pkg", "pretend", "settings") + __slots__ = ("cancelled", "fetch_all", "pkg", "pretend", "settings") def execute(self): portdb = self.pkg.root_config.trees["porttree"].dbapi @@ -1479,17 +1480,63 @@ class EbuildFetcher(SlotObject): mydbapi=portdb, tree="porttree") return retval -class EbuildFetcherAsync(SlotObject): +class SubProcess(SlotObject): + __slots__ = ("cancelled", "pid", "returncode") + + def poll(self): + if self.returncode is not None: + return self.returncode + retval = os.waitpid(self.pid, os.WNOHANG) + if retval == (0, 0): + return None + self._set_returncode(retval) + return self.returncode + + def cancel(self): + if self.isAlive(): + os.kill(self.pid, signal.SIGTERM) + self.cancelled = True + if self.pid is not None: + self.wait() + return self.returncode + + def isAlive(self): + return self.pid is not None and \ + self.returncode is None + + def wait(self): + if self.returncode is not None: + return self.returncode + self._set_returncode(os.waitpid(self.pid, 0)) + return self.returncode + + def _set_returncode(self, wait_retval): + + retval = wait_retval[1] + portage.process.spawned_pids.remove(self.pid) + if retval != os.EX_OK: + if retval & 0xff: + retval = (retval & 0xff) << 8 + else: + retval = retval >> 8 + + self.returncode = retval + +class EbuildFetcherAsync(SubProcess): __slots__ = ("log_file", "fd_pipes", "pkg", "register", "unregister", - "pid", "returncode", "files") + "files") _file_names = ("fetcher", "out") _files_dict = slot_dict_class(_file_names, prefix="") _bufsize = 4096 def start(self): + + if self.cancelled: + return + # flush any pending output fd_pipes = self.fd_pipes if fd_pipes is None: @@ -1572,33 +1619,6 @@ class EbuildFetcherAsync(SlotObject): for f in files.values(): f.close() - def poll(self): - if self.returncode is not None: - return self.returncode - retval = os.waitpid(self.pid, os.WNOHANG) - if retval == (0, 0): - return None - self._set_returncode(retval) - return self.returncode - - def wait(self): - if self.returncode is not None: - return self.returncode - self._set_returncode(os.waitpid(self.pid, 0)) - return self.returncode - - def _set_returncode(self, wait_retval): - - retval = wait_retval[1] - portage.process.spawned_pids.remove(self.pid) - if retval != os.EX_OK: - if retval & 0xff: - retval = (retval & 0xff) << 8 - else: - retval = retval >> 8 - - self.returncode = retval - class EbuildBuildDir(SlotObject): __slots__ = ("pkg", "settings", @@ -1671,20 +1691,138 @@ class EbuildBuildDir(SlotObject): class AlreadyLocked(portage.exception.PortageException): pass -class EbuildBuild(Task): - """ - TODO: Support asynchronous execution, to implement parallel builds. - """ +class EbuildBuild(SlotObject): + + __slots__ = ("args_set", "find_blockers", + "ldpath_mtimes", "logger", "opts", + "pkg", "pkg_count", "scheduler", + "settings") + + def execute(self): + + args_set = self.args_set + find_blockers = self.find_blockers + ldpath_mtimes = self.ldpath_mtimes + logger = self.logger + opts = self.opts + pkg = self.pkg + pkg_count = self.pkg_count + scheduler = self.scheduler + settings = self.settings + root_config = pkg.root_config + root = root_config.root + system_set = root_config.sets["system"] + world_set = root_config.sets["world"] + vartree = root_config.trees["vartree"] + portdb = root_config.trees["porttree"].dbapi + debug = settings.get("PORTAGE_DEBUG") == "1" + features = self.settings.features + settings["EMERGE_FROM"] = pkg.type_name + settings.backup_changes("EMERGE_FROM") + settings.reset() + ebuild_path = portdb.findname(self.pkg.cpv) + + #buildsyspkg: Check if we need to _force_ binary package creation + issyspkg = "buildsyspkg" in features and \ + system_set.findAtomForPackage(pkg) and \ + not opts.buildpkg + + if opts.fetchonly: + fetcher = EbuildFetcher(fetch_all=opts.fetch_all_uri, + pkg=pkg, pretend=opts.pretend, settings=settings) + retval = fetcher.execute() + if retval != os.EX_OK: + from portage.elog.messages import eerror + eerror("!!! Fetch for %s failed, continuing..." % pkg.cpv, + phase="unpack", key=pkg.cpv) + return retval + + build_dir = EbuildBuildDir(pkg=pkg, settings=settings) + try: + build_dir.lock() + # Cleaning is triggered before the setup + # phase, in portage.doebuild(). + msg = " === (%s of %s) Cleaning (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) + short_msg = "emerge: (%s of %s) %s Clean" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + if opts.buildpkg or issyspkg: + if issyspkg: + portage.writemsg(">>> This is a system package, " + \ + "let's pack a rescue tarball.\n", noiselevel=-1) + msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) + short_msg = "emerge: (%s of %s) %s Compile" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + build = EbuildExecuter(pkg=pkg, register=scheduler.register, + schedule=scheduler.schedule, settings=settings, + unregister=scheduler.unregister) + retval = build.execute() + if retval != os.EX_OK: + return retval + + build = EbuildBinpkg(pkg=pkg, settings=settings) + retval = build.execute() + if retval != os.EX_OK: + return retval + + if not opts.buildpkgonly: + msg = " === (%s of %s) Merging (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, + pkg.cpv, ebuild_path) + short_msg = "emerge: (%s of %s) %s Merge" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + merge = EbuildMerge( + find_blockers=find_blockers, + ldpath_mtimes=ldpath_mtimes, + pkg=pkg, settings=settings) + retval = merge.execute() + if retval != os.EX_OK: + return retval + elif "noclean" not in settings.features: + portage.doebuild(ebuild_path, "clean", root, + settings, debug=debug, mydbapi=portdb, + tree="porttree") + else: + msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) + short_msg = "emerge: (%s of %s) %s Compile" % \ + (pkg_count.curval, pkg_count.curval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + build = EbuildExecuter(pkg=pkg, register=scheduler.register, + schedule=scheduler.schedule, settings=settings, + unregister=scheduler.unregister) + retval = build.execute() + if retval != os.EX_OK: + return retval + + merge = EbuildMerge( + find_blockers=self.find_blockers, + ldpath_mtimes=ldpath_mtimes, + pkg=pkg, settings=settings) + retval = merge.execute() + + if retval != os.EX_OK: + return retval + finally: + if build_dir.locked: + portage.elog.elog_process(pkg.cpv, settings) + build_dir.unlock() + return os.EX_OK + +class EbuildExecuter(SlotObject): + __slots__ = ("pkg", "register", "schedule", "settings", "unregister") _phases = ("setup", "unpack", "compile", "test", "install") - def _get_hash_key(self): - hash_key = getattr(self, "_hash_key", None) - if hash_key is None: - self._hash_key = ("EbuildBuild", self.pkg._get_hash_key()) - return self._hash_key - def execute(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi @@ -1730,11 +1868,11 @@ class EbuildBuild(Task): return os.EX_OK -class EbuildPhase(SlotObject): +class EbuildPhase(SubProcess): __slots__ = ("fd_pipes", "phase", "pkg", "register", "settings", "unregister", - "pid", "returncode", "files") + "files") _file_names = ("log", "stdout", "ebuild") _files_dict = slot_dict_class(_file_names, prefix="") @@ -1835,42 +1973,17 @@ class EbuildPhase(SlotObject): for f in files.values(): f.close() - def poll(self): - if self.returncode is not None: - return self.returncode - retval = os.waitpid(self.pid, os.WNOHANG) - if retval == (0, 0): - return None - self._set_returncode(retval) - return self.returncode - - def wait(self): - if self.returncode is not None: - return self.returncode - self._set_returncode(os.waitpid(self.pid, 0)) - return self.returncode - def _set_returncode(self, wait_retval): - - retval = wait_retval[1] - portage.process.spawned_pids.remove(self.pid) - if retval != os.EX_OK: - if retval & 0xff: - retval = (retval & 0xff) << 8 - else: - retval = retval >> 8 - + SubProcess._set_returncode(self, wait_retval) msg = portage._doebuild_exit_status_check( self.phase, self.settings) if msg: - retval = 1 + self.returncode = 1 from textwrap import wrap from portage.elog.messages import eerror for l in wrap(msg, 72): eerror(l, phase=self.phase, key=self.pkg.cpv) - self.returncode = retval - class EbuildBinpkg(Task): """ This assumes that src_install() has successfully completed. @@ -1963,6 +2076,95 @@ class PackageUninstall(Task): return e.status return os.EX_OK +class Binpkg(SlotObject): + + __slots__ = ("find_blockers", + "ldpath_mtimes", "logger", "opts", + "pkg", "pkg_count", "prefetcher", "scheduler", + "settings") + + def execute(self): + + find_blockers = self.find_blockers + ldpath_mtimes = self.ldpath_mtimes + logger = self.logger + opts = self.opts + pkg = self.pkg + pkg_count = self.pkg_count + scheduler = self.scheduler + settings = self.settings + + # The prefetcher has already completed or it + # could be running now. If it's running now, + # wait for it to complete since it holds + # a lock on the file being fetched. The + # portage.locks functions are only designed + # to work between separate processes. Since + # the lock is held by the current process, + # use the scheduler and fetcher methods to + # synchronize with the fetcher. + prefetcher = self.prefetcher + if prefetcher is not None: + if not prefetcher.isAlive(): + prefetcher.cancel() + else: + retval = prefetcher.poll() + + if retval is None: + waiting_msg = ("Fetching '%s' " + \ + "in the background. " + \ + "To view fetch progress, run `tail -f " + \ + "/var/log/emerge-fetch.log` in another " + \ + "terminal.") % prefetcher.pkg_path + msg_prefix = colorize("GOOD", " * ") + from textwrap import wrap + waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ + for line in wrap(waiting_msg, 65)) + writemsg(waiting_msg, noiselevel=-1) + + while retval is None: + scheduler.schedule() + retval = prefetcher.poll() + del prefetcher + + fetcher = BinpkgFetcher(pkg=pkg, pretend=opts.pretend, + use_locks=("distlocks" in settings.features)) + pkg_path = fetcher.pkg_path + + if opts.getbinpkg: + retval = fetcher.execute() + if fetcher.remote: + msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ + (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) + short_msg = "emerge: (%s of %s) %s Fetch" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + if retval != os.EX_OK: + return retval + + if opts.fetchonly: + return os.EX_OK + + msg = " === (%s of %s) Merging Binary (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) + short_msg = "emerge: (%s of %s) %s Merge Binary" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + build_dir = EbuildBuildDir(pkg=pkg, settings=settings) + try: + build_dir.lock() + merge = BinpkgMerge(find_blockers=find_blockers, + ldpath_mtimes=ldpath_mtimes, pkg=pkg, pretend=opts.pretend, + pkg_path=pkg_path, settings=settings) + retval = merge.execute() + if retval != os.EX_OK: + return retval + finally: + build_dir.unlock() + return os.EX_OK + class BinpkgFetcher(Task): __slots__ = ("use_locks", "pkg", "pretend", @@ -2022,25 +2224,24 @@ class BinpkgFetcher(Task): rval = 1 return rval -class BinpkgFetcherAsync(SlotObject): +class BinpkgFetcherAsync(SubProcess): - __slots__ = ("cancelled", "log_file", "fd_pipes", "pkg", + __slots__ = ("log_file", "fd_pipes", "pkg", "register", "unregister", - "locked", "files", "pid", "pkg_path", "returncode", "_lock_obj") + "locked", "files", "pkg_path", "_lock_obj") _file_names = ("fetcher", "out") _files_dict = slot_dict_class(_file_names, prefix="") _bufsize = 4096 def __init__(self, **kwargs): - SlotObject.__init__(self, **kwargs) + SubProcess.__init__(self, **kwargs) pkg = self.pkg self.pkg_path = pkg.root_config.trees["bintree"].getname(pkg.cpv) def start(self): if self.cancelled: - self.pid = -1 return fd_pipes = self.fd_pipes @@ -2172,45 +2373,6 @@ class BinpkgFetcherAsync(SlotObject): self._lock_obj = None self.locked = False - def poll(self): - if self.returncode is not None: - return self.returncode - retval = os.waitpid(self.pid, os.WNOHANG) - if retval == (0, 0): - return None - self._set_returncode(retval) - return self.returncode - - def cancel(self): - if self.isAlive(): - os.kill(self.pid, signal.SIGTERM) - self.cancelled = True - if self.pid is not None: - self.wait() - return self.returncode - - def isAlive(self): - return self.pid is not None and \ - self.returncode is None - - def wait(self): - if self.returncode is not None: - return self.returncode - self._set_returncode(os.waitpid(self.pid, 0)) - return self.returncode - - def _set_returncode(self, wait_retval): - - retval = wait_retval[1] - portage.process.spawned_pids.remove(self.pid) - if retval != os.EX_OK: - if retval & 0xff: - retval = (retval & 0xff) << 8 - else: - retval = retval >> 8 - - self.returncode = retval - class BinpkgMerge(Task): __slots__ = ("find_blockers", "ldpath_mtimes", @@ -2223,6 +2385,12 @@ class BinpkgMerge(Task): return self._hash_key def execute(self): + + settings = self.settings + settings["EMERGE_FROM"] = self.pkg.type_name + settings.backup_changes("EMERGE_FROM") + settings.reset() + root_config = self.pkg.root_config retval = portage.pkgmerge(self.pkg_path, root_config.root, self.settings, @@ -6695,6 +6863,25 @@ class Scheduler(object): _fetch_log = EPREFIX + "/var/log/emerge-fetch.log" + class _iface_class(SlotObject): + __slots__ = ("register", "schedule", "unregister") + + class _build_opts_class(SlotObject): + __slots__ = ("buildpkg", "buildpkgonly", + "fetch_all_uri", "fetchonly", "pretend") + + class _binpkg_opts_class(SlotObject): + __slots__ = ("fetchonly", "getbinpkg", "pretend") + + class _pkg_count_class(SlotObject): + __slots__ = ("curval", "maxval") + + class _emerge_log_class(SlotObject): + __slots__ = ("xterm_titles",) + + def log(self, *pargs, **kwargs): + emergelog(self.xterm_titles, *pargs, **kwargs) + def __init__(self, settings, trees, mtimedb, myopts, spinner, mergelist, favorites, digraph): self.settings = settings @@ -6705,6 +6892,14 @@ class Scheduler(object): self._mtimedb = mtimedb self._mergelist = mergelist self._favorites = favorites + self._args_set = InternalPackageSet(favorites) + self._build_opts = self._build_opts_class() + for k in self._build_opts.__slots__: + setattr(self._build_opts, k, "--" + k.replace("_", "-") in myopts) + self._binpkg_opts = self._binpkg_opts_class() + for k in self._binpkg_opts.__slots__: + setattr(self._binpkg_opts, k, "--" + k.replace("_", "-") in myopts) + self.edebug = 0 if settings.get("PORTAGE_DEBUG", "") == "1": self.edebug = 1 @@ -6713,6 +6908,11 @@ class Scheduler(object): self.pkgsettings[root] = portage.config( clone=trees[root]["vartree"].settings) self.curval = 0 + self._logger = self._emerge_log_class( + xterm_titles=("notitles" not in settings.features)) + self._sched_iface = self._iface_class( + register=self._register, schedule=self._schedule, + unregister=self._unregister) self._poll_event_handlers = {} self._poll = select.poll() from collections import deque @@ -6946,12 +7146,12 @@ class Scheduler(object): "--onlydeps" in self.myopts pretend = "--pretend" in self.myopts ldpath_mtimes = mtimedb["ldpath"] - xterm_titles = "notitles" not in self.settings.features + logger = self._logger if "--resume" in self.myopts: # We're resuming. print colorize("GOOD", "*** Resuming merge...") - emergelog(xterm_titles, " *** Resuming merge...") + self._logger.log(" *** Resuming merge...") # Do this before verifying the ebuild Manifests since it might # be possible for the user to use --resume --skipfirst get past @@ -6964,6 +7164,7 @@ class Scheduler(object): getbinpkg = "--getbinpkg" in self.myopts if self._parallel_fetch: + portage.writemsg(">>> starting parallel fetch\n") for pkg in mylist: if not isinstance(pkg, Package): continue @@ -7022,7 +7223,8 @@ class Scheduler(object): # Filter mymergelist so that all the len(mymergelist) calls # below (for display) do not count Uninstall instances. mymergelist = [x for x in mymergelist if x[-1] == "merge"] - mergecount=0 + pkg_count = self._pkg_count_class( + curval=0, maxval=len(mymergelist)) for x in task_list: if x[0] == "blocks": continue @@ -7046,24 +7248,24 @@ class Scheduler(object): else: raise AssertionError("Package type: '%s'" % pkg_type) if not x.installed: - mergecount += 1 + pkg_count.curval += 1 try: self._execute_task(bad_resume_opts, failed_fetches, - mydbapi, mergecount, + mydbapi, pkg_count, myfeat, mymergelist, x, - prefetchers, xterm_titles) + prefetchers) except self._pkg_failure, e: return e.status - return self._post_merge(mtimedb, xterm_titles, failed_fetches) + return self._post_merge(mtimedb, + self._logger.xterm_titles, failed_fetches) def _execute_task(self, bad_resume_opts, - failed_fetches, mydbapi, mergecount, myfeat, - mymergelist, pkg, prefetchers, xterm_titles): + failed_fetches, mydbapi, pkg_count, myfeat, + mymergelist, pkg, prefetchers): favorites = self._favorites mtimedb = self._mtimedb - from portage.elog import elog_process - from portage.elog.filtering import filter_mergephases + mergecount = pkg_count.curval pkgsettings = self.pkgsettings[pkg.root] buildpkgonly = "--buildpkgonly" in self.myopts fetch_all = "--fetch-all-uri" in self.myopts @@ -7076,6 +7278,7 @@ class Scheduler(object): xterm_titles = "notitles" not in self.settings.features x = pkg + y = None root_config = pkg.root_config system_set = root_config.sets["system"] args_set = InternalPackageSet(favorites) @@ -7097,7 +7300,7 @@ class Scheduler(object): if x[0]=="blocks": pkgindex=3 - y = portdb.findname(pkg_key) + if "--pretend" not in self.myopts: print "\n>>> Emerging (" + \ colorize("MERGE_LIST_PROGRESS", str(mergecount)) + " of " + \ @@ -7107,163 +7310,35 @@ class Scheduler(object): str(mergecount)+" of "+str(len(mymergelist))+\ ") "+x[pkgindex]+" to "+x[1]) - pkgsettings["EMERGE_FROM"] = x[0] - pkgsettings.backup_changes("EMERGE_FROM") - pkgsettings.reset() - - #buildsyspkg: Check if we need to _force_ binary package creation - issyspkg = ("buildsyspkg" in myfeat) \ - and x[0] != "blocks" \ - and system_set.findAtomForPackage(pkg) \ - and "--buildpkg" not in self.myopts - if x[0] in ["ebuild","blocks"]: - if x[0] == "blocks" and "--fetchonly" not in self.myopts: - raise Exception, "Merging a blocker" - elif fetchonly: - fetcher = EbuildFetcher(fetch_all=fetch_all, - pkg=pkg, pretend=pretend, settings=pkgsettings) - retval = fetcher.execute() - if (retval is None) or retval: - print - print "!!! Fetch for",y,"failed, continuing..." - print - failed_fetches.append(pkg_key) - self.curval += 1 - return + self._schedule() - build_dir = EbuildBuildDir(pkg=pkg, settings=pkgsettings) - try: - build_dir.lock() - # Cleaning is triggered before the setup - # phase, in portage.doebuild(). - msg = " === (%s of %s) Cleaning (%s::%s)" % \ - (mergecount, len(mymergelist), pkg_key, y) - short_msg = "emerge: (%s of %s) %s Clean" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) - - if "--buildpkg" in self.myopts or issyspkg: - if issyspkg: - print ">>> This is a system package, " + \ - "let's pack a rescue tarball." - msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ - (mergecount, len(mymergelist), pkg_key, y) - short_msg = "emerge: (%s of %s) %s Compile" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) - - build = EbuildBuild(pkg=pkg, register=self._register, - schedule=self._schedule, settings=pkgsettings, - unregister=self._unregister) - retval = build.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - - build = EbuildBinpkg(pkg=pkg, settings=pkgsettings) - retval = build.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - - if "--buildpkgonly" not in self.myopts: - msg = " === (%s of %s) Merging (%s::%s)" % \ - (mergecount, len(mymergelist), pkg_key, y) - short_msg = "emerge: (%s of %s) %s Merge" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) - - merge = EbuildMerge( - find_blockers=self._find_blockers(pkg), - ldpath_mtimes=ldpath_mtimes, - pkg=pkg, pretend=pretend, settings=pkgsettings) - retval = merge.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - elif "noclean" not in pkgsettings.features: - portage.doebuild(y, "clean", myroot, - pkgsettings, self.edebug, mydbapi=portdb, - tree="porttree") + if x.type_name == "ebuild": + y = portdb.findname(pkg.cpv) + build = EbuildBuild(args_set=self._args_set, + find_blockers=self._find_blockers(pkg), + ldpath_mtimes=ldpath_mtimes, logger=self._logger, + opts=self._build_opts, pkg=pkg, pkg_count=pkg_count, + settings=pkgsettings, scheduler=self._sched_iface) + retval = build.execute() + if retval != os.EX_OK: + if fetchonly: + failed_fetches.append(pkg.cpv) else: - msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ - (mergecount, len(mymergelist), pkg_key, y) - short_msg = "emerge: (%s of %s) %s Compile" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) - - build = EbuildBuild(pkg=pkg, register=self._register, - schedule=self._schedule, settings=pkgsettings, - unregister=self._unregister) - retval = build.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - - merge = EbuildMerge( - find_blockers=self._find_blockers(pkg), - ldpath_mtimes=ldpath_mtimes, - pkg=pkg, pretend=pretend, settings=pkgsettings) - retval = merge.execute() - - if retval != os.EX_OK: - raise self._pkg_failure(retval) - finally: - if build_dir.locked: - elog_process(pkg.cpv, pkgsettings, - phasefilter=filter_mergephases) - build_dir.unlock() + raise self._pkg_failure(retval) elif x.type_name == "binary": - # The prefetcher have already completed or it - # could be running now. If it's running now, - # wait for it to complete since it holds - # a lock on the file being fetched. The - # portage.locks functions are only designed - # to work between separate processes. Since - # the lock is held by the current process, - # use the scheduler and fetcher methods to - # synchronize with the fetcher. - prefetcher = prefetchers.get(pkg) - if prefetcher is not None: - if not prefetcher.isAlive(): - prefetcher.cancel() - else: - retval = None - while retval is None: - self._schedule() - retval = prefetcher.poll() - del prefetcher - - fetcher = BinpkgFetcher(pkg=pkg, pretend=pretend, - use_locks=("distlocks" in pkgsettings.features)) - mytbz2 = fetcher.pkg_path - if "--getbinpkg" in self.myopts: - retval = fetcher.execute() - if fetcher.remote: - msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ - (mergecount, len(mymergelist), pkg_key, mytbz2) - short_msg = "emerge: (%s of %s) %s Fetch" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) - - if retval != os.EX_OK: + binpkg = Binpkg(find_blockers=self._find_blockers(pkg), + ldpath_mtimes=ldpath_mtimes, logger=self._logger, + opts=self._binpkg_opts, pkg=pkg, pkg_count=pkg_count, + prefetcher=prefetchers.get(pkg), settings=pkgsettings, + scheduler=self._sched_iface) + retval = binpkg.execute() + if retval != os.EX_OK: + if fetchonly: failed_fetches.append(pkg.cpv) - if not fetchonly: - raise self._pkg_failure() - - if "--fetchonly" in self.myopts or \ - "--fetch-all-uri" in self.myopts: - self.curval += 1 - return + else: + raise self._pkg_failure(retval) - short_msg = "emerge: ("+str(mergecount)+" of "+str(len(mymergelist))+") "+x[pkgindex]+" Merge Binary" - emergelog(xterm_titles, " === ("+str(mergecount)+\ - " of "+str(len(mymergelist))+") Merging Binary ("+\ - x[pkgindex]+"::"+mytbz2+")", short_msg=short_msg) - merge = BinpkgMerge(find_blockers=self._find_blockers(pkg), - ldpath_mtimes=ldpath_mtimes, pkg=pkg, pretend=pretend, - pkg_path=fetcher.pkg_path, settings=pkgsettings) - retval = merge.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - #need to check for errors if not buildpkgonly: if not (fetchonly or oneshot or pretend) and \ args_set.findAtomForPackage(pkg): @@ -7321,7 +7396,7 @@ class Scheduler(object): (mergecount, len(mymergelist), x[pkgindex]) emergelog(xterm_titles, (" === (%s of %s) " + \ "Post-Build Cleaning (%s::%s)") % \ - (mergecount, len(mymergelist), x[pkgindex], y), + (mergecount, len(mymergelist), pkg.cpv, y), short_msg=short_msg) emergelog(xterm_titles, " ::: completed emerge ("+\ str(mergecount)+" of "+str(len(mymergelist))+") "+\ diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 98342ebc1..1790b1592 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -170,7 +170,7 @@ def best_from_dict(key, top_dict, key_order, EmptyOnError=1, FullCopy=1, AllowEm if EmptyOnError: return "" else: - raise KeyError, "Key not found in list; '%s'" % key + raise KeyError("Key not found in list; '%s'" % key) def getcwd(): "this fixes situations where the current directory doesn't exist" @@ -1820,14 +1820,14 @@ class config(object): def modifying(self): if self.locked: - raise Exception, "Configuration is locked." + raise Exception("Configuration is locked.") def backup_changes(self,key=None): self.modifying() if key and key in self.configdict["env"]: self.backupenv[key] = copy.deepcopy(self.configdict["env"][key]) else: - raise KeyError, "No such key defined in environment: %s" % key + raise KeyError("No such key defined in environment: %s" % key) def reset(self,keeping_pkg=0,use_cache=1): """ @@ -6638,8 +6638,6 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, return 1 tbz2_lock = None - builddir_lock = None - catdir_lock = None mycat = None mypkg = None did_merge_phase = False @@ -6682,11 +6680,6 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, catdir_lock = portage.locks.lockdir(catdir) portage.util.ensure_dirs(catdir, uid=portage_uid, gid=portage_gid, mode=070, mask=0) - builddir_lock = portage.locks.lockdir(builddir) - try: - portage.locks.unlockdir(catdir_lock) - finally: - catdir_lock = None try: shutil.rmtree(builddir) except (IOError, OSError), e: @@ -6764,7 +6757,7 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, mysettings.pop("PORTAGE_BINPKG_FILE", None) if tbz2_lock: portage.locks.unlockfile(tbz2_lock) - if builddir_lock: + if True: if not did_merge_phase: # The merge phase handles this already. Callers don't know how # far this function got, so we have to call elog_process() here @@ -6778,21 +6771,6 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, if e.errno != errno.ENOENT: raise del e - portage.locks.unlockdir(builddir_lock) - try: - if not catdir_lock: - # Lock catdir for removal if empty. - catdir_lock = portage.locks.lockdir(catdir) - finally: - if catdir_lock: - try: - os.rmdir(catdir) - except OSError, e: - if e.errno not in (errno.ENOENT, - errno.ENOTEMPTY, errno.EEXIST): - raise - del e - portage.locks.unlockdir(catdir_lock) def deprecated_profile_check(): if not os.access(DEPRECATED_PROFILE_FILE, os.R_OK): diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 52ce59148..8364d26ed 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -186,7 +186,10 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0): myhash = perform_checksum(filename, x, calc_prelink=calc_prelink)[0] if mydict[x] != myhash: if strict: - raise portage.exception.DigestException, "Failed to verify '$(file)s' on checksum type '%(type)s'" % {"file":filename, "type":x} + raise portage.exception.DigestException( + ("Failed to verify '$(file)s' on " + \ + "checksum type '%(type)s'") % \ + {"file" : filename, "type" : x}) else: file_is_ok = False reason = (("Failed on %s verification" % x), myhash,mydict[x]) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index e2a53aac4..dd8015cc1 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -263,11 +263,15 @@ class portdbapi(dbapi): elif self.manifestVerifier: if not self.manifestVerifier.verify(myManifestPath): # Verification failed the desired level. - raise UntrustedSignature, "Untrusted Manifest: %(manifest)s" % {"manifest":myManifestPath} + raise UntrustedSignature( + "Untrusted Manifest: %(manifest)s" % \ + {"manifest" : myManifestPath}) if ("severe" in self.mysettings.features) and \ (mys != portage.gpg.fileStats(myManifestPath)): - raise SecurityViolation, "Manifest changed: %(manifest)s" % {"manifest":myManifestPath} + raise SecurityViolation( + "Manifest changed: %(manifest)s" % \ + {"manifest":myManifestPath}) except InvalidSignature, e: if ("strict" in self.mysettings.features) or \ @@ -284,7 +288,9 @@ class portdbapi(dbapi): except (OSError, FileNotFound), e: if ("strict" in self.mysettings.features) or \ ("severe" in self.mysettings.features): - raise SecurityViolation, "Error in verification of signatures: %(errormsg)s" % {"errormsg":str(e)} + raise SecurityViolation( + "Error in verification of signatures: " + \ + "%(errormsg)s" % {"errormsg" : str(e)}) writemsg("!!! Manifest is missing or inaccessable: %(manifest)s\n" % {"manifest":myManifestPath}, noiselevel=-1) diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 41d6b128b..c2f506d4b 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -246,7 +246,7 @@ def use_reduce(deparray, uselist=[], masklist=[], matchall=0, excludeall=[]): if mydeparray: newdeparray.append(mydeparray.pop(0)) else: - raise ValueError, "Conditional with no target." + raise ValueError("Conditional with no target.") # Deprecation checks warned = 0 diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index b8e44fdae..6801d8527 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -83,7 +83,8 @@ def create_conn(baseurl,conn=None): parts = baseurl.split("://",1) if len(parts) != 2: - raise ValueError, "Provided URL does not contain protocol identifier. '%s'" % baseurl + raise ValueError("Provided URL does not " + \ + "contain protocol identifier. '%s'" % baseurl) protocol,url_parts = parts del parts @@ -105,7 +106,7 @@ def create_conn(baseurl,conn=None): del userpass_host if len(userpass) > 2: - raise ValueError, "Unable to interpret username/password provided." + raise ValueError("Unable to interpret username/password provided.") elif len(userpass) == 2: username = userpass[0] password = userpass[1] @@ -324,7 +325,7 @@ def dir_get_list(baseurl,conn=None): elif protocol == "sftp": listing = conn.listdir(address) else: - raise TypeError, "Unknown protocol. '%s'" % protocol + raise TypeError("Unknown protocol. '%s'" % protocol) if not keepconnection: conn.close() @@ -356,7 +357,7 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000): finally: f.close() else: - raise TypeError, "Unknown protocol. '%s'" % protocol + raise TypeError("Unknown protocol. '%s'" % protocol) if data: xpaksize = portage.xpak.decodeint(data[-8:-4]) @@ -448,7 +449,7 @@ def file_get_lib(baseurl,dest,conn=None): finally: f.close() else: - raise TypeError, "Unknown protocol. '%s'" % protocol + raise TypeError("Unknown protocol. '%s'" % protocol) if not keepconnection: conn.close() diff --git a/pym/portage/gpg.py b/pym/portage/gpg.py index 1fdac6252..1ddb99d3e 100644 --- a/pym/portage/gpg.py +++ b/pym/portage/gpg.py @@ -10,6 +10,10 @@ import types import commands import portage.exception import portage.checksum +from portage.exception import CommandNotFound, \ + DirectoryNotFound, FileNotFound, \ + InvalidData, InvalidDataType, InvalidSignature, MissingParameter, \ + MissingSignature, PortageException, SecurityViolation GPG_BINARY = "/usr/bin/gpg" GPG_OPTIONS = " --lock-never --no-random-seed-file --no-greeting --no-sig-cache " @@ -42,34 +46,38 @@ class FileChecker(object): if (keydir != None): # Verify that the keydir is valid. if type(keydir) != types.StringType: - raise portage.exception.InvalidDataType, "keydir argument: %s" % keydir + raise InvalidDataType( + "keydir argument: %s" % keydir) if not os.path.isdir(keydir): - raise portage.exception.DirectoryNotFound, "keydir: %s" % keydir + raise DirectoryNotFound("keydir: %s" % keydir) self.keydir = copy.deepcopy(keydir) if (keyring != None): # Verify that the keyring is a valid filename and exists. if type(keyring) != types.StringType: - raise portage.exception.InvalidDataType, "keyring argument: %s" % keyring + raise InvalidDataType("keyring argument: %s" % keyring) if keyring.find("/") != -1: - raise portage.exception.InvalidData, "keyring: %s" % keyring + raise InvalidData("keyring: %s" % keyring) pathname = "" if keydir: pathname = keydir + "/" + keyring if not os.path.isfile(pathname): - raise portage.exception.FileNotFound, "keyring missing: %s (dev.gentoo.org/~carpaski/gpg/)" % pathname + raise FileNotFound( + "keyring missing: %s (dev.gentoo.org/~carpaski/gpg/)" % \ + pathname) keyringPath = keydir+"/"+keyring if not keyring or not keyringPath and requireSignedRing: - raise portage.exception.MissingParameter + raise MissingParameter((keyring, keyringPath)) self.keyringStats = fileStats(keyringPath) self.minimumTrust = TRUSTED if not self.verify(keyringPath, keyringPath+".asc"): self.keyringIsTrusted = False if requireSignedRing: - raise portage.exception.InvalidSignature, "Required keyring verification: "+keyringPath + raise InvalidSignature( + "Required keyring verification: " + keyringPath) else: self.keyringIsTrusted = True @@ -81,27 +89,27 @@ class FileChecker(object): if self.keyringStats and self.keyringPath: new_stats = fileStats(self.keyringPath) if new_stats != self.keyringStats: - raise portage.exception.SecurityViolation, "GPG keyring changed!" + raise SecurityViolation("GPG keyring changed!") def verify(self, filename, sigfile=None): """Uses minimumTrust to determine if it is Valid/True or Invalid/False""" self._verifyKeyring() if not os.path.isfile(filename): - raise portage.exception.FileNotFound, filename + raise FileNotFound, filename if sigfile and not os.path.isfile(sigfile): - raise portage.exception.FileNotFound, sigfile + raise FileNotFound, sigfile if self.keydir and not os.path.isdir(self.keydir): - raise portage.exception.DirectoryNotFound, filename + raise DirectoryNotFound, filename if self.keyringPath: if not os.path.isfile(self.keyringPath): - raise portage.exception.FileNotFound, self.keyringPath + raise FileNotFound, self.keyringPath if not os.path.isfile(filename): - raise portage.exception.CommandNotFound, filename + raise CommandNotFound(filename) command = GPG_BINARY + GPG_VERIFY_FLAGS + GPG_OPTIONS if self.keydir: @@ -119,7 +127,7 @@ class FileChecker(object): result = (result >> 8) if signal: - raise SignalCaught, "Signal: %d" % (signal) + raise PortageException("Signal: %d" % (signal)) trustLevel = UNTRUSTED if result == 0: @@ -127,22 +135,22 @@ class FileChecker(object): #if portage.output.find("WARNING") != -1: # trustLevel = MARGINAL if portage.output.find("BAD") != -1: - raise portage.exception.InvalidSignature, filename + raise InvalidSignature(filename) elif result == 1: trustLevel = EXISTS if portage.output.find("BAD") != -1: - raise portage.exception.InvalidSignature, filename + raise InvalidSignature(filename) elif result == 2: trustLevel = UNTRUSTED if portage.output.find("could not be verified") != -1: - raise portage.exception.MissingSignature, filename + raise MissingSignature(filename) if portage.output.find("public key not found") != -1: if self.keyringIsTrusted: # We trust the ring, but not the key specifically. trustLevel = MARGINAL else: - raise portage.exception.InvalidSignature, filename+" (Unknown Signature)" + raise InvalidSignature(filename+"(Unknown Signature)") else: - raise portage.exception.UnknownCondition, "GPG returned unknown result: %d" % (result) + raise PortageException("GPG returned unknown result: %d" % (result)) if trustLevel >= self.minimumTrust: return True diff --git a/pym/portage/locks.py b/pym/portage/locks.py index dd3df1025..37fae639e 100644 --- a/pym/portage/locks.py +++ b/pym/portage/locks.py @@ -24,7 +24,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None): import fcntl if not mypath: - raise InvalidData, "Empty path given" + raise InvalidData("Empty path given") if type(mypath) == types.StringType and mypath[-1] == '/': mypath = mypath[:-1] @@ -45,7 +45,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None): if type(mypath) == types.StringType: if not os.path.exists(os.path.dirname(mypath)): - raise DirectoryNotFound, os.path.dirname(mypath) + raise DirectoryNotFound(os.path.dirname(mypath)) if not os.path.exists(lockfilename): old_mask=os.umask(000) myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660) @@ -66,7 +66,8 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None): myfd = mypath else: - raise ValueError, "Unknown type passed in '%s': '%s'" % (type(mypath),mypath) + raise ValueError("Unknown type passed in '%s': '%s'" % \ + (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. @@ -166,7 +167,7 @@ def unlockfile(mytuple): except OSError: if type(lockfilename) == types.StringType: os.close(myfd) - raise IOError, "Failed to unlock file '%s'\n" % lockfilename + raise IOError("Failed to unlock file '%s'\n" % lockfilename) try: # This sleep call was added to allow other processes that are @@ -231,7 +232,9 @@ def hardlink_lockfile(lockfilename, max_wait=14400): os.close(myfd) if not os.path.exists(myhardlock): - raise FileNotFound, _("Created lockfile is missing: %(filename)s") % {"filename":myhardlock} + raise FileNotFound( + _("Created lockfile is missing: %(filename)s") % \ + {"filename" : myhardlock}) try: res = os.link(myhardlock, lockfilename) -- 2.26.2