From 0b3cf8d3e1a1e4f7c91129caf493974548c2c233 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 11 Oct 2008 15:44:10 +0000 Subject: [PATCH] Merged from trunk -r11635:11654 | 11636 | Bug #240022 - Avoid duplicate output for the same library | | zmedico | (due to symlinks) in display_preserved_libs() by using | | | os.path.realpath() to group duplicate references together. | | | Thanks to Fabian Groffen for the initial | | | patch. | | 11637 | Fix KeyError raised from display_preserved_libs() when the | | zmedico | realpath is used as a key in consumer_map instead of the | | | orignal path that was returned from the | | | PreservedLibsRegistry. | | 11642 | In order to avoid "waiting for lock" messages at the | | zmedico | beginning, which annoy users, never spawn a prefetcher for | | | the first package. | | 11643 | In doebuild(), don't create build dirs when "listonly" fetch | | zmedico | mode. | | 11644 | In fetch(), never call the pkg_nofetch phase when in | | zmedico | "listonly" mode. | | 11645 | Bug #234301 - Add support to EbuildFetcher to collect elog | | zmedico | messages that might be generated during the pkg_nofetch | | | phase. This involves maintaining a lock on PORTAGE_BUILDDIR | | | where the elog messages are collected, and cleaning up | | | PORTAGE_BUILDDIR before releasing the lock. | | 11646 | Fix EbuildFetcher._wait_hook() so it can be called more than | | zmedico | once without throwing an AttributeError. | | 11647 | Make EbuildFetcher direct fetch output to the normal build | | zmedico | log. By keeping the output in the build log instead of the | | | global emerge-fetch.log file, output from a possible fetch | | | failure can be recovered and displayed later. TODO: Convert | | | failed fetch to elog message as suggested in bug #234301. | | 11648 | Make EbuildFetcher clean up the build dir after failures in | | zmedico | fetchonly mode since the log is in emerge-fetch.log anyway. | | 11649 | When EbuildBuild encounters a failed fetch, propagee the log | | zmedico | path so that it can be properly displayed in the status | | | display. | | 11650 | Make HOMEPAGE.missing fatal again since, in cases when | | zmedico | homepages are no longer available, we can use a link to some | | | site like packages.gentoo.org or | | | www.unmaintained-free-software.org. | | 11651 | Change the "qawarnings" variable from a list to a set. | | zmedico | | | 11652 | Make "qawarnings" a normal set since the missingvars code | | zmedico | might modify it. | | 11653 | Instead of using inheritance, reference the ConfigParser | | zmedico | part of SetConfig as an attribute in case we want to swap | | | out the ConfigParser implemention and no longer want to | | | implement the whole interface. | | 11654 | Generate elog messages for failed fetches. This restores | | zmedico | some functionality that was removed in r11645. | svn path=/main/branches/prefix/; revision=11674 --- bin/repoman | 7 +-- pym/_emerge/__init__.py | 109 +++++++++++++++++++++++++++++------ pym/portage/__init__.py | 17 +----- pym/portage/sets/__init__.py | 54 +++++++++-------- 4 files changed, 126 insertions(+), 61 deletions(-) diff --git a/bin/repoman b/bin/repoman index 2459412f1..eb968a804 100755 --- a/bin/repoman +++ b/bin/repoman @@ -320,7 +320,7 @@ qahelp={ qacats = qahelp.keys() qacats.sort() -qawarnings=[ +qawarnings = set(( "changelog.ebuildadded", "changelog.missing", "changelog.notadded", @@ -337,7 +337,6 @@ qawarnings=[ "KEYWORDS.dropped", "KEYWORDS.stupid", "KEYWORDS.missing", -"HOMEPAGE.missing", "IUSE.invalid", "IUSE.undefined", "RDEPEND.suspect", @@ -356,7 +355,7 @@ qawarnings=[ "virtual.unavailable", "usage.obsolete", "LIVEVCS.stable" -] +)) missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"] allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_")) @@ -369,7 +368,7 @@ for x in missingvars: if x not in qacats: logging.warn('* missingvars values need to be added to qahelp ("%s")' % x) qacats.append(x) - qawarnings.append(x) + qawarnings.add(x) valid_restrict = frozenset(["binchecks", "bindist", "fetch", "installsources", "mirror", diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index a5700298d..77b21bc41 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2265,14 +2265,20 @@ class MiscFunctionsProcess(SpawnProcess): class EbuildFetcher(SpawnProcess): - __slots__ = ("fetchonly", "fetchall", "pkg",) + __slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \ + ("_build_dir",) def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) - settings = root_config.settings + settings = self.config_pool.allocate() + self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings) + self._build_dir.lock() + if self.logfile is None: + self.logfile = settings.get("PORTAGE_LOG_FILE") + phase = "fetch" if self.fetchall: phase = "fetchall" @@ -2284,7 +2290,7 @@ class EbuildFetcher(SpawnProcess): fetch_env = os.environ.copy() fetch_env["PORTAGE_NICENESS"] = "0" - if self.fetchonly: + if self.prefetch: fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1" ebuild_binary = os.path.join( @@ -2299,6 +2305,34 @@ class EbuildFetcher(SpawnProcess): self.env = fetch_env SpawnProcess._start(self) + def _set_returncode(self, wait_retval): + SpawnProcess._set_returncode(self, wait_retval) + # Collect elog messages that might have been + # created by the pkg_nofetch phase. + if self._build_dir is not None: + # Skip elog messages for prefetch, in order to avoid duplicates. + if not self.prefetch and self.returncode != os.EX_OK: + elog_out = None + if self.logfile is not None: + if self.background: + elog_out = open(self.logfile, 'a') + eerror("Fetch failed for '%s'" % self.pkg.cpv, + phase="unpack", key=self.pkg.cpv, out=elog_out) + if elog_out is not None: + elog_out.close() + if not self.prefetch: + portage.elog.elog_process(self.pkg.cpv, self._build_dir.settings) + if self.fetchonly or self.returncode == os.EX_OK: + try: + shutil.rmtree(self._build_dir.settings["PORTAGE_BUILDDIR"]) + except EnvironmentError, e: + if e.errno != errno.ENOENT: + raise + del e + self._build_dir.unlock() + self.config_pool.deallocate(self._build_dir.settings) + self._build_dir = None + class EbuildBuildDir(SlotObject): __slots__ = ("dir_path", "pkg", "settings", @@ -2324,11 +2358,13 @@ class EbuildBuildDir(SlotObject): portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) settings = self.settings + settings.setcpv(self.pkg) debug = settings.get("PORTAGE_DEBUG") == "1" use_cache = 1 # always true portage.doebuild_environment(ebuild_path, "setup", root_config.root, self.settings, debug, use_cache, portdb) dir_path = self.settings["PORTAGE_BUILDDIR"] + portage.prepare_build_dirs(self.pkg.root, self.settings, 0) catdir = os.path.dirname(dir_path) self._catdir = catdir @@ -2376,7 +2412,7 @@ class EbuildBuildDir(SlotObject): class EbuildBuild(CompositeTask): - __slots__ = ("args_set", "background", "find_blockers", + __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") @@ -2439,10 +2475,9 @@ class EbuildBuild(CompositeTask): return fetch_log = None - if self.background: - fetch_log = self.scheduler.fetch.log_file - fetcher = EbuildFetcher(fetchall=opts.fetch_all_uri, + fetcher = EbuildFetcher(config_pool=self.config_pool, + fetchall=opts.fetch_all_uri, fetchonly=opts.fetchonly, background=self.background, logfile=fetch_log, pkg=pkg, scheduler=self.scheduler) @@ -2455,19 +2490,20 @@ class EbuildBuild(CompositeTask): self._start_task(fetcher, self._fetch_exit) def _fetch_exit(self, fetcher): - opts = self.opts pkg = self.pkg + fetch_failed = False if opts.fetchonly: - if self._final_exit(fetcher) != os.EX_OK: - if not self.background: - eerror("Fetch for %s failed, continuing..." % pkg.cpv, - phase="unpack", key=pkg.cpv) - self.wait() - return + fetch_failed = self._final_exit(fetcher) != os.EX_OK + else: + fetch_failed = self._default_exit(fetcher) != os.EX_OK + + if fetch_failed and fetcher.logfile is not None and \ + os.path.exists(fetcher.logfile): + self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile - if self._default_exit(fetcher) != os.EX_OK: + if fetch_failed or opts.fetchonly: self.wait() return @@ -3466,7 +3502,7 @@ class MergeListItem(CompositeTask): """ __slots__ = ("args_set", - "binpkg_opts", "build_opts", "emerge_opts", + "binpkg_opts", "build_opts", "config_pool", "emerge_opts", "failed_fetches", "find_blockers", "logger", "mtimedb", "pkg", "pkg_count", "pkg_to_replace", "prefetcher", "settings", "statusMessage", "world_atom") + \ @@ -3519,6 +3555,7 @@ class MergeListItem(CompositeTask): build = EbuildBuild(args_set=args_set, background=self.background, + config_pool=self.config_pool, find_blockers=find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, opts=build_opts, pkg=pkg, pkg_count=pkg_count, @@ -8990,6 +9027,21 @@ class Scheduler(PollScheduler): __slots__ = ("build_dir", "build_log", "fetch_log", "pkg", "returncode") + class _ConfigPool(object): + """Interface for a task to temporarily allocate a config + instance from a pool. This allows a task to be constructed + long before the config instance actually becomes needed, like + when prefetchers are constructed for the whole merge list.""" + __slots__ = ("_root", "_allocate", "_deallocate") + def __init__(self, root, allocate, deallocate): + self._root = root + self._allocate = allocate + self._deallocate = deallocate + def allocate(self): + return self._allocate(self._root) + def deallocate(self, settings): + self._deallocate(settings) + def __init__(self, settings, trees, mtimedb, myopts, spinner, mergelist, favorites, digraph): PollScheduler.__init__(self) @@ -9389,7 +9441,10 @@ class Scheduler(PollScheduler): prefetchers = self._prefetchers getbinpkg = "--getbinpkg" in self.myopts - for pkg in self._mergelist: + # In order to avoid "waiting for lock" messages + # at the beginning, which annoy users, never + # spawn a prefetcher for the first package. + for pkg in self._mergelist[1:]: prefetcher = self._create_prefetcher(pkg) if prefetcher is not None: self._task_queues.fetch.add(prefetcher) @@ -9407,8 +9462,10 @@ class Scheduler(PollScheduler): elif pkg.type_name == "ebuild": prefetcher = EbuildFetcher(background=True, + config_pool=self._ConfigPool(pkg.root, + self._allocate_config, self._deallocate_config), fetchonly=1, logfile=self._fetch_log, - pkg=pkg, scheduler=self._sched_iface) + pkg=pkg, prefetch=True, scheduler=self._sched_iface) elif pkg.type_name == "binary" and \ "--getbinpkg" in self.myopts and \ @@ -10044,6 +10101,8 @@ class Scheduler(PollScheduler): task = MergeListItem(args_set=self._args_set, background=self._background, binpkg_opts=self._binpkg_opts, build_opts=self._build_opts, + config_pool=self._ConfigPool(pkg.root, + self._allocate_config, self._deallocate_config), emerge_opts=self.myopts, failed_fetches=self._failed_fetches, find_blockers=self._find_blockers(pkg), logger=self._logger, @@ -11000,8 +11059,20 @@ def display_preserved_libs(vardbapi): for cpv in plibdata: print colorize("WARN", ">>>") + " package: %s" % cpv + samefile_map = {} for f in plibdata[cpv]: - print colorize("WARN", " * ") + " - %s" % f + real_path = os.path.realpath(f) + alt_paths = samefile_map.get(real_path) + if alt_paths is None: + alt_paths = set() + samefile_map[real_path] = alt_paths + alt_paths.add(f) + + for alt_paths in samefile_map.itervalues(): + alt_paths = sorted(alt_paths) + for p in alt_paths: + print colorize("WARN", " * ") + " - %s" % (p,) + f = alt_paths[0] consumers = consumer_map[f] for c in consumers[:MAX_DISPLAY]: print colorize("WARN", " * ") + " used by %s (%s)" % \ diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 950f5353c..1c34f44ac 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4077,7 +4077,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if listonly: writemsg_stdout("\n", noiselevel=-1) - if fetched != 2: + elif fetched != 2: if restrict_fetch: print "\n!!!", mysettings["CATEGORY"] + "/" + \ mysettings["PF"], "has fetch restriction turned on." @@ -4095,8 +4095,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", mysettings.pop("EBUILD_PHASE", None) else: mysettings["EBUILD_PHASE"] = ebuild_phase - elif listonly: - continue elif not filedict[myfile]: writemsg("Warning: No mirrors available for file" + \ " '%s'\n" % (myfile), noiselevel=-1) @@ -5478,7 +5476,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Build directory creation isn't required for any of these. have_build_dirs = False - if mydo not in ("digest", "help", "manifest"): + if not (mydo in ("digest", "help", "manifest") or \ + (mydo == "fetch" and listonly)): mystatus = prepare_build_dirs(myroot, mysettings, cleanup) if mystatus: return mystatus @@ -5671,16 +5670,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if not emerge_skip_distfiles and \ need_distfiles and not fetch( fetchme, mysettings, listonly=listonly, fetchonly=fetchonly): - if have_build_dirs: - # Create an elog message for this fetch failure since the - # mod_echo module might push the original message off of the - # top of the terminal and prevent the user from being able to - # see it. - from portage.elog.messages import eerror - eerror("Fetch failed for '%s'" % mycpv, - phase="unpack", key=mycpv) - from portage.elog import elog_process - elog_process(mysettings.mycpv, mysettings) return 1 if mydo == "fetch" and listonly: diff --git a/pym/portage/sets/__init__.py b/pym/portage/sets/__init__.py index 7ac6502b7..b892bb308 100644 --- a/pym/portage/sets/__init__.py +++ b/pym/portage/sets/__init__.py @@ -22,10 +22,10 @@ def get_boolean(options, name, default): class SetConfigError(Exception): pass -class SetConfig(SafeConfigParser): +class SetConfig(object): def __init__(self, paths, settings, trees): - SafeConfigParser.__init__(self) - self.read(paths) + self._parser = SafeConfigParser() + self._parser.read(paths) self.errors = [] self.psets = {} self.trees = trees @@ -34,6 +34,7 @@ class SetConfig(SafeConfigParser): self.active = [] def update(self, setname, options): + parser = self._parser self.errors = [] if not setname in self.psets: options["name"] = setname @@ -41,30 +42,32 @@ class SetConfig(SafeConfigParser): # for the unlikely case that there is already a section with the requested setname import random - while setname in self.sections(): + while setname in parser.sections(): setname = "%08d" % random.randint(0, 10**10) - self.add_section(setname) + parser.add_section(setname) for k, v in options.items(): - self.set(setname, k, v) + parser.set(setname, k, v) else: section = self.psets[setname].creator - if self.has_option(section, "multiset") and self.getboolean(section, "multiset"): + if parser.has_option(section, "multiset") and \ + parser.getboolean(section, "multiset"): self.errors.append("Invalid request to reconfigure set '%s' generated by multiset section '%s'" % (setname, section)) return for k, v in options.items(): - self.set(section, k, v) + parser.set(section, k, v) self._parse(update=True) def _parse(self, update=False): if self._parsed and not update: return - for sname in self.sections(): + parser = self._parser + for sname in parser.sections(): # find classname for current section, default to file based sets - if not self.has_option(sname, "class"): + if not parser.has_option(sname, "class"): classname = "portage.sets.files.StaticFileSet" else: - classname = self.get(sname, "class") + classname = parser.get(sname, "class") # try to import the specified class try: @@ -77,11 +80,12 @@ class SetConfig(SafeConfigParser): continue # prepare option dict for the current section optdict = {} - for oname in self.options(sname): - optdict[oname] = self.get(sname, oname) + for oname in parser.options(sname): + optdict[oname] = parser.get(sname, oname) # create single or multiple instances of the given class depending on configuration - if self.has_option(sname, "multiset") and self.getboolean(sname, "multiset"): + if parser.has_option(sname, "multiset") and \ + parser.getboolean(sname, "multiset"): if hasattr(setclass, "multiBuilder"): newsets = {} try: @@ -93,7 +97,8 @@ class SetConfig(SafeConfigParser): if x in self.psets and not update: self.errors.append("Redefinition of set '%s' (sections: '%s', '%s')" % (x, self.psets[x].creator, sname)) newsets[x].creator = sname - if self.has_option(sname, "world-candidate") and not self.getboolean(sname, "world-candidate"): + if parser.has_option(sname, "world-candidate") and \ + not parser.getboolean(sname, "world-candidate"): newsets[x].world_candidate = False self.psets.update(newsets) else: @@ -101,7 +106,7 @@ class SetConfig(SafeConfigParser): continue else: try: - setname = self.get(sname, "name") + setname = parser.get(sname, "name") except NoOptionError: setname = sname if setname in self.psets and not update: @@ -110,7 +115,8 @@ class SetConfig(SafeConfigParser): try: self.psets[setname] = setclass.singleBuilder(optdict, self.settings, self.trees) self.psets[setname].creator = sname - if self.has_option(sname, "world-candidate") and not self.getboolean(sname, "world-candidate"): + if parser.has_option(sname, "world-candidate") and \ + not parser.getboolean(sname, "world-candidate"): self.psets[setname].world_candidate = False except SetConfigError, e: self.errors.append("Configuration error in section '%s': %s" % (sname, str(e))) @@ -127,7 +133,7 @@ class SetConfig(SafeConfigParser): def getSetAtoms(self, setname, ignorelist=None): myset = self.getSets()[setname] myatoms = myset.getAtoms() - + parser = self._parser extend = set() remove = set() intersect = set() @@ -135,12 +141,12 @@ class SetConfig(SafeConfigParser): if ignorelist is None: ignorelist = set() if not setname in ignorelist: - if self.has_option(myset.creator, "extend"): - extend.update(self.get(myset.creator, "extend").split()) - if self.has_option(myset.creator, "remove"): - remove.update(self.get(myset.creator, "remove").split()) - if self.has_option(myset.creator, "intersect"): - intersect.update(self.get(myset.creator, "intersect").split()) + if parser.has_option(myset.creator, "extend"): + extend.update(parser.get(myset.creator, "extend").split()) + if parser.has_option(myset.creator, "remove"): + remove.update(parser.get(myset.creator, "remove").split()) + if parser.has_option(myset.creator, "intersect"): + intersect.update(parser.get(myset.creator, "intersect").split()) ignorelist.add(setname) for n in myset.getNonAtoms(): -- 2.26.2