From f1512693882d72e4ecee3e2fe00b13438ea8559c Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 5 Nov 2008 18:08:52 +0000 Subject: [PATCH] Merged from trunk -r11799:11809 | 11800 | Remove locking from EbuildFetchPretend since it uses a | | zmedico | private temp directory now (since bug #245231). | | 11801 | Remove duplicate code inside EbuildFetchPretend._execute(). | | zmedico | | | 11802 | Import shutil globally. | | zmedico | | | 11803 | Make metadata.missing and metadata.bad fatal by removing | | zmedico | them from the qawarnings set. Thanks to Robin H. Johnson | | | for the suggestion. | | 11804 | Bug #245356 - Filter POSIXLY_CORRECT from the ebuild | | zmedico | environment since it breaks stuff. | | 11805 | Bug #245358 - For unsatisfied dependencies, display the | | zmedico | parent nodes and the argument that pulled them in. | | 11806 | Bug #245362 - Use tuples of (device, inode) for all path | | zmedico | comparisons inside LinkageMap, so that they work regardless | | | of path differences due to symlinked directories. TODO: Fix | | | other preserve-libs code, such as dblink._preserve_libs(), | | | to use this approach for path comparisons. | | 11807 | Free some memory by clearing the LinkageMap cache when it's | | zmedico | not needed. | | 11808 | Synchronize portage update/restart logic in dblink.merge() | | zmedico | and depgraph.display() so it's consistent with | | | Scheduler._is_restart_necessary(). Thanks to Jeremy Olexa | | | for reporting. | | 11809 | Fix broken reference to self inside unmerge(). | | zmedico | | svn path=/main/branches/prefix/; revision=11810 --- bin/ebuild.sh | 2 +- bin/repoman | 2 -- pym/_emerge/__init__.py | 55 ++++++++++++++++++------------- pym/portage/__init__.py | 7 ++++ pym/portage/dbapi/vartree.py | 63 +++++++++++++++++++++--------------- pym/portage/util.py | 6 +--- 6 files changed, 78 insertions(+), 57 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index fff81dc4d..0deb938c4 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1608,7 +1608,7 @@ filter_readonly_variables() { SANDBOX_DEBUG_LOG SANDBOX_DISABLED SANDBOX_LIB SANDBOX_LOG SANDBOX_ON" filtered_vars="${readonly_bash_vars} ${READONLY_PORTAGE_VARS} - BASH_.* PATH" + BASH_.* PATH POSIXLY_CORRECT" if hasq --filter-sandbox $* ; then filtered_vars="${filtered_vars} SANDBOX_.*" else diff --git a/bin/repoman b/bin/repoman index b78443076..bf2b6a7b0 100755 --- a/bin/repoman +++ b/bin/repoman @@ -349,8 +349,6 @@ qawarnings = set(( "file.size", "inherit.autotools", "java.eclassesnotused", -"metadata.missing", -"metadata.bad", "virtual.versioned", "virtual.exists", "virtual.unavailable", diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 64a394d22..ab129754a 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1634,13 +1634,17 @@ class EbuildFetchPretend(SlotObject): return retval def _execute(self): - build_dir = EbuildBuildDir(pkg=self.pkg, settings=self.settings) - build_dir.lock() - build_dir.clean() + settings = self.settings + pkg = self.pkg + root_config = pkg.root_config + portdb = root_config.trees["porttree"].dbapi + ebuild_path = portdb.findname(pkg.cpv) + settings.setcpv(pkg) + debug = settings.get("PORTAGE_DEBUG") == "1" + use_cache = 1 # always true + portage.doebuild_environment(ebuild_path, "fetch", + root_config.root, settings, debug, use_cache, portdb) portage.prepare_build_dirs(self.pkg.root, self.settings, 0) - portdb = self.pkg.root_config.trees["porttree"].dbapi - ebuild_path = portdb.findname(self.pkg.cpv) - debug = self.settings.get("PORTAGE_DEBUG") == "1" retval = portage.doebuild(ebuild_path, "fetch", self.settings["ROOT"], self.settings, debug=debug, @@ -1648,8 +1652,6 @@ class EbuildFetchPretend(SlotObject): mydbapi=portdb, tree="porttree") portage.elog.elog_process(self.pkg.cpv, self.settings) - build_dir.clean() - build_dir.unlock() return retval class AsynchronousTask(SlotObject): @@ -5313,10 +5315,6 @@ class depgraph(object): xinfo='"%s"' % arg # Discard null/ from failed cpv_expand category expansion. xinfo = xinfo.replace("null/", "") - if myparent: - xfrom = '(dependency required by '+ \ - green('"%s"' % myparent[2]) + \ - red(' [%s]' % myparent[0]) + ')' masked_packages = [] missing_use = [] missing_licenses = [] @@ -5415,8 +5413,24 @@ class depgraph(object): show_mask_docs() else: print "\nemerge: there are no ebuilds to satisfy "+green(xinfo)+"." - if myparent: - print xfrom + + # Show parent nodes and the argument that pulled them in. + node = myparent + msg = [] + while node is not None: + msg.append('(dependency required by "%s" [%s])' % \ + (colorize('INFORM', str(node.cpv)), node.type_name)) + parent = None + for parent in self.digraph.parent_nodes(node): + if isinstance(parent, DependencyArg): + msg.append('(dependency required by "%s" [argument])' % \ + (colorize('INFORM', str(parent)))) + parent = None + break + node = parent + for line in msg: + print line + print def _select_pkg_highest_available(self, root, atom, onlydeps=False): @@ -7553,15 +7567,9 @@ class depgraph(object): not self._opts_no_restart.intersection(self.myopts) and \ pkg.root == self._running_root.root and \ portage.match_from_list( - portage.const.PORTAGE_PACKAGE_ATOM, [pkg]): - - pn, ver, rev = pkg.pv_split - if rev == "r0": - myversion = ver - else: - myversion = "%s-%s" % (ver, rev) - - if myversion != portage.VERSION and "--quiet" not in self.myopts: + portage.const.PORTAGE_PACKAGE_ATOM, [pkg]) and \ + not vardb.cpv_exists(pkg.cpv) and \ + "--quiet" not in self.myopts: if mylist_index < len(mylist) - 1: p.append(colorize("WARN", "*** Portage will stop merging at this point and reload itself,")) p.append(colorize("WARN", " then resume the merge.")) @@ -13540,6 +13548,7 @@ def clear_caches(trees): d["porttree"].dbapi._aux_cache.clear() d["bintree"].dbapi._aux_cache.clear() d["bintree"].dbapi._clear_cache() + d["vartree"].dbapi.linkmap._clear_cache() portage.dircache.clear() gc.collect() diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 5f13e4217..7c44b20de 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -994,6 +994,11 @@ class config(object): "VENDOR", "__CF_USER_TEXT_ENCODING", ] + # variables that break bash + _environ_filter += [ + "POSIXLY_CORRECT", + ] + # portage config variables and variables set directly by portage _environ_filter += [ "ACCEPT_KEYWORDS", "AUTOCLEAN", @@ -6122,6 +6127,7 @@ def unmerge(cat, pkg, myroot, mysettings, mytrimworld=1, vartree=None, ldpath_mtimes=None, scheduler=None): mylink = dblink(cat, pkg, myroot, mysettings, treetype="vartree", vartree=vartree, scheduler=scheduler) + vartree = mylink.vartree try: mylink.lockdb() if mylink.exists(): @@ -6134,6 +6140,7 @@ def unmerge(cat, pkg, myroot, mysettings, mytrimworld=1, vartree=None, return retval return os.EX_OK finally: + vartree.dbapi.linkmap._clear_cache() mylink.unlockdb() def getCPFromCPV(mycpv): diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index ea26c1135..cabf8de41 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -30,7 +30,7 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ from portage.elog import elog_process from portage.elog.filtering import filter_mergephases, filter_unmergephases -import os, re, stat, errno, copy, subprocess +import os, re, shutil, stat, errno, copy, subprocess import logging import shlex from itertools import izip @@ -149,8 +149,23 @@ class LinkageMap(object): self._root = self._dbapi.root self._libs = {} self._obj_properties = {} - self._defpath = set(getlibpaths(self._root)) self._obj_key_cache = {} + self._defpath = set() + self._path_key_cache = {} + + def _clear_cache(self): + self._libs.clear() + self._obj_properties.clear() + self._obj_key_cache.clear() + self._defpath.clear() + self._path_key_cache.clear() + + def _path_key(self, path): + key = self._path_key_cache.get(path) + if key is None: + key = self._ObjectKey(path, self._root) + self._path_key_cache[path] = key + return key class _ObjectKey(object): @@ -222,10 +237,12 @@ class LinkageMap(object): def rebuild(self, exclude_pkgs=None, include_file=None): root = self._root - self._defpath = set(getlibpaths(root)) - libs = {} - obj_key_cache = {} - obj_properties = {} + self._clear_cache() + self._defpath.update(getlibpaths(self._root)) + libs = self._libs + obj_key_cache = self._obj_key_cache + obj_properties = self._obj_properties + lines = [] for cpv in self._dbapi.cpv_all(): if exclude_pkgs is not None and cpv in exclude_pkgs: @@ -259,8 +276,7 @@ class LinkageMap(object): obj = fields[1] obj_key = self._ObjectKey(obj, root) soname = fields[2] - path = set([ - normalize_path(os.path.join(self._root, x.lstrip(os.path.sep))) + path = set([normalize_path(x) \ for x in filter(None, fields[3].replace( "${ORIGIN}", os.path.dirname(obj)).replace( "$ORIGIN", os.path.dirname(obj)).split(":"))]) @@ -281,10 +297,6 @@ class LinkageMap(object): obj_properties.setdefault(obj_key, \ (arch, needed, path, soname, set()))[4].add(obj) - self._libs = libs - self._obj_properties = obj_properties - self._obj_key_cache = obj_key_cache - def listBrokenBinaries(self, debug=False): """ Find binaries and their needed sonames, which have no providers. @@ -530,7 +542,7 @@ class LinkageMap(object): raise KeyError("%s (%s) not in object list" % (obj_key, obj)) arch, needed, path, _, _ = self._obj_properties[obj_key] - path = path.union(self._defpath) + path_keys = set(self._path_key(x) for x in path.union(self._defpath)) for soname in needed: rValue[soname] = set() if soname not in self._libs or arch not in self._libs[soname]: @@ -540,8 +552,7 @@ class LinkageMap(object): for provider_key in self._libs[soname][arch]["providers"]: providers = self._obj_properties[provider_key][4] for provider in providers: - if os.path.join(self._root, - os.path.dirname(provider).lstrip(os.path.sep)) in path: + if self._path_key(os.path.dirname(provider)) in path_keys: rValue[soname].add(provider) return rValue @@ -584,10 +595,6 @@ class LinkageMap(object): if obj_key not in self._obj_properties: raise KeyError("%s (%s) not in object list" % (obj_key, obj)) - # Determine the directory(ies) from the set of objects. - objs_dirs = set(os.path.join(self._root, - os.path.dirname(x).lstrip(os.sep)) for x in objs) - # If there is another version of this lib with the # same soname and the master link points to that # other version, this lib will be shadowed and won't @@ -606,6 +613,10 @@ class LinkageMap(object): (master_st.st_dev, master_st.st_ino): return set() + # Determine the directory(ies) from the set of objects. + objs_dir_keys = set(self._path_key(os.path.dirname(x)) for x in objs) + defpath_keys = set(self._path_key(x) for x in self._defpath) + arch, _, _, soname, _ = self._obj_properties[obj_key] if soname in self._libs and arch in self._libs[soname]: # For each potential consumer, add it to rValue if an object from the @@ -613,8 +624,8 @@ class LinkageMap(object): for consumer_key in self._libs[soname][arch]["consumers"]: _, _, path, _, consumer_objs = \ self._obj_properties[consumer_key] - path = path.union(self._defpath) - if objs_dirs.intersection(path): + path_keys = defpath_keys.union(self._path_key(x) for x in path) + if objs_dir_keys.intersection(path_keys): rValue.update(consumer_objs) return rValue @@ -2158,7 +2169,7 @@ class dblink(object): writemsg("portage.dblink.delete(): invalid dbdir: %s\n" % \ self.dbdir, noiselevel=-1) return - import shutil + shutil.rmtree(self.dbdir) self.vartree.dbapi._remove(self) @@ -2966,7 +2977,6 @@ class dblink(object): del mylibs, mycontents, old_contents, liblist # inject files that should be preserved into our image dir - import shutil preserve_paths = [] candidates_stack = list(candidates) while candidates_stack: @@ -4173,11 +4183,12 @@ class dblink(object): if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None if self.myroot == "/" and \ - match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]): + match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]) and \ + not self.vartree.dbapi.cpv_exists(self.mycpv): settings = self.settings base_path_orig = os.path.dirname(settings["PORTAGE_BIN_PATH"]) from tempfile import mkdtemp - import shutil + # Make the temp directory inside PORTAGE_TMPDIR since, unlike # /tmp, it can't be mounted with the "noexec" option. base_path_tmp = mkdtemp("", "._portage_reinstall_.", @@ -4228,6 +4239,7 @@ class dblink(object): self._scheduler.dblinkEbuildPhase( self, mydbapi, myebuild, "clean") finally: + self.vartree.dbapi.linkmap._clear_cache() self.unlockdb() return retval @@ -4241,7 +4253,6 @@ class dblink(object): return " ".join(mydata) def copyfile(self,fname): - import shutil shutil.copyfile(fname,self.dbdir+"/"+os.path.basename(fname)) def getfile(self,fname): diff --git a/pym/portage/util.py b/pym/portage/util.py index 650b4ad45..eef9cf12f 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -1220,8 +1220,4 @@ def getlibpaths(root): rval.append("/usr/lib") rval.append("/lib") - rval = [normalize_path(os.path.join(root, x.lstrip(os.path.sep))) \ - for x in rval if x] - - return rval - + return [normalize_path(x) for x in rval if x] -- 2.26.2