From 66b22b5931bf9f8ca77e825d92916ae967e805cc Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 15 Nov 2008 15:23:03 +0000 Subject: [PATCH] Merged from trunk -r11816:11833 | 11817 | Inside LinkageMap._libs, use a SlotDict to store consumers | | zmedico | and providers, to use less memory than a normal dict. | | 11818 | Swap the locations of the arch and soname keys inside | | zmedico | LinkageMap._libs in order to conserve memory. This new | | | branching layout uses fewer dict instances since the number | | | of sonames is much larger than the number of archs. | | 11819 | Combine redundant arch_map code inside LinkageMap.rebuild(). | | zmedico | | | 11820 | When using the `read` builtin to split newlines in e* | | zmedico | function arguments, use $REPLY in order to ensure that | | | whitespace in each line is correctly preserved. Thanks to | | | Joe Peterson for reporting. | | 11821 | Fix LinkageMap.rebuild() so that the data from include_file | | zmedico | overrides the data from any previously installed files. This | | | prevent possible corruption of the data since only one set | | | of data can be correct for a given file. | | 11822 | When calling scanelf inside LinkageMap.rebuild(), join paths | | zmedico | with $ROOT when generating the arguments and then strip | | | $ROOT from the paths in the output. | | 11823 | Improve the error message that's generated in | | zmedico | LinkageMap.rebuild() for corrupt NEEDED.ELF.2 entries. | | 11824 | Replace NEEDED.ELF.2 strings with references to | | zmedico | LinkageMap._needed_aux_key. | | 11825 | Handle a potential OSError that occurs if the scanelf binary | | zmedico | is missing when LinkageMap.rebuild() is called. | | 11826 | In LinkageMap.rebuild(), immediately raise a CommandNotFound | | zmedico | exception if scanelf is missing since otherwise it will lead | | | to a KeyError later on from findConsumers or findProviders. | | | This will allow the caller to handle the CommandNotFound | | | exception if necessary, and skip any findConsumers or | | | findProviders since they won't be able to return valid | | | results. | | 11827 | Handle CommandNotFound exceptions if the scanelf binary | | zmedico | happens to be missing, and disable preserve-libs code in | | | that case. | | 11828 | With python-2.6, importing the Crypto.Hash.MD5 and | | zmedico | Crypto.Hash.SHA modules from pycrypto triggers warnings | | | since those modules are implemented using the deprecated md5 | | | and sha modules from python's stdlib. So, in order to avoid | | | the warning and the inferior hash implementations that come | | | with them, never use these particular modules from pycrypto. | | | Instead, use hashlib or directly use stdlib's md5 and sha | | | modules if necessary. Thanks to Markus Peloquin for | | | reporting. | | 11829 | Fix $ROOT handlink inside display_preserved_libs(). | | zmedico | | | 11830 | Inside LinkageMap, use self._obj_key() whenever possible. | | zmedico | | | 11831 | Bug #245362 - Rewrite preserve-libs preservation code so | | zmedico | that it always relies on inode comparisons rather than | | | string comparisons. Instead of injecting libraries into $D | | | before the files are merged, the preservation code now | | | executes after the files are merged but before the old | | | version is unmerged. After determining which libs to | | | preserve, the CONTENTS are updated to include those libs. | | | The PreservedLibsRegistry.register() call is now done just | | | after the temporary vdb entry has been moved into place, | | | guaranteeing that a valid vdb entry is in place so that the | | | unregistration code from bug #210501 is no longer needed. | | 11832 | Bug #243030 - In PreservedLibraryConsumerSet.load(), avoid | | zmedico | rebuilding packages just because they contain preserved libs | | | that happen to be consumers of other preserved libs. | | 11833 | Inside vardbapi.removeFromContents(), automatically clear | | zmedico | the contents cache of the dblink instance in case an | | | existing one was passed in. | svn path=/main/branches/prefix/; revision=11938 --- bin/isolated-functions.sh | 24 +- pym/_emerge/__init__.py | 40 +-- pym/portage/checksum.py | 5 +- pym/portage/dbapi/vartree.py | 468 ++++++++++++++++++----------------- pym/portage/sets/libs.py | 7 +- 5 files changed, 291 insertions(+), 253 deletions(-) diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 61bc6b698..d984a11f8 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -177,8 +177,8 @@ elog_base() { return 1 ;; esac - echo -e "$@" | while read line ; do - echo "${messagetype} ${line}" >> \ + echo -e "$@" | while read ; do + echo "$messagetype $REPLY" >> \ "${T}/logging/${EBUILD_PHASE:-other}" done return 0 @@ -187,8 +187,8 @@ elog_base() { eqawarn() { elog_base QA "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - echo -e "$@" | while read line ; do - vecho -e " ${WARN}*${NORMAL} ${line}" >&2 + echo -e "$@" | while read ; do + vecho -e " $WARN*$NORMAL $REPLY" >&2 done LAST_E_CMD="eqawarn" return 0 @@ -197,8 +197,8 @@ eqawarn() { elog() { elog_base LOG "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - echo -e "$@" | while read line ; do - echo -e " ${GOOD}*${NORMAL} ${line}" + echo -e "$@" | while read ; do + echo -e " $GOOD*$NORMAL $REPLY" done LAST_E_CMD="elog" return 0 @@ -227,8 +227,8 @@ esyslog() { einfo() { elog_base INFO "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - echo -e "$@" | while read line ; do - echo -e " ${GOOD}*${NORMAL} ${line}" + echo -e "$@" | while read ; do + echo -e " $GOOD*$NORMAL $REPLY" done LAST_E_CMD="einfo" return 0 @@ -245,8 +245,8 @@ einfon() { ewarn() { elog_base WARN "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - echo -e "$@" | while read line ; do - echo -e " ${WARN}*${NORMAL} ${RC_INDENTATION}${line}" >&2 + echo -e "$@" | while read ; do + echo -e " $WARN*$NORMAL $RC_INDENTATION$REPLY" >&2 done LAST_E_CMD="ewarn" return 0 @@ -255,8 +255,8 @@ ewarn() { eerror() { elog_base ERROR "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - echo -e "$@" | while read line ; do - echo -e " ${BAD}*${NORMAL} ${RC_INDENTATION}${line}" >&2 + echo -e "$@" | while read ; do + echo -e " $BAD*$NORMAL $RC_INDENTATION$REPLY" >&2 done LAST_E_CMD="eerror" return 0 diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index da1876860..0dcf2f193 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -11126,29 +11126,39 @@ def display_preserved_libs(vardbapi): print colorize("WARN", "!!!") + " existing preserved libs:" plibdata = vardbapi.plib_registry.getPreservedLibs() linkmap = vardbapi.linkmap - consumer_map = {} - search_for_owners = set() - for cpv in plibdata: - for f in plibdata[cpv]: - if f in consumer_map: - continue - consumers = list(linkmap.findConsumers(f)) - consumers.sort() - consumer_map[f] = consumers - search_for_owners.update(consumers[:MAX_DISPLAY+1]) + owners = {} + linkmap_broken = False + + try: + linkmap.rebuild() + except portage.exception.CommandNotFound, e: + writemsg_level("!!! Command Not Found: %s\n" % (e,), + level=logging.ERROR, noiselevel=-1) + del e + linkmap_broken = True + else: + search_for_owners = set() + for cpv in plibdata: + for f in plibdata[cpv]: + if f in consumer_map: + continue + consumers = list(linkmap.findConsumers(f)) + consumers.sort() + consumer_map[f] = consumers + search_for_owners.update(consumers[:MAX_DISPLAY+1]) - owners = vardbapi._owners.getFileOwnerMap(search_for_owners) + owners = vardbapi._owners.getFileOwnerMap(search_for_owners) for cpv in plibdata: print colorize("WARN", ">>>") + " package: %s" % cpv samefile_map = {} for f in plibdata[cpv]: - real_path = os.path.realpath(f) - alt_paths = samefile_map.get(real_path) + obj_key = linkmap._obj_key(f) + alt_paths = samefile_map.get(obj_key) if alt_paths is None: alt_paths = set() - samefile_map[real_path] = alt_paths + samefile_map[obj_key] = alt_paths alt_paths.add(f) for alt_paths in samefile_map.itervalues(): @@ -11156,7 +11166,7 @@ def display_preserved_libs(vardbapi): for p in alt_paths: print colorize("WARN", " * ") + " - %s" % (p,) f = alt_paths[0] - consumers = consumer_map[f] + consumers = consumer_map.get(f, []) for c in consumers[:MAX_DISPLAY]: print colorize("WARN", " * ") + " used by %s (%s)" % \ (c, ", ".join(x.mycpv for x in owners.get(c, []))) diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 8364d26ed..56719a1df 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -61,10 +61,7 @@ sha1hash = _generate_hash_function("SHA1", _new_sha1, origin="internal") # Use pycrypto when available, prefer it over the internal fallbacks try: - from Crypto.Hash import MD5, SHA, SHA256, RIPEMD - - md5hash = _generate_hash_function("MD5", MD5.new, origin="pycrypto") - sha1hash = _generate_hash_function("SHA1", SHA.new, origin="pycrypto") + from Crypto.Hash import SHA256, RIPEMD sha256hash = _generate_hash_function("SHA256", SHA256.new, origin="pycrypto") rmd160hash = _generate_hash_function("RMD160", RIPEMD.new, origin="pycrypto") except ImportError, e: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index dee1b0669..e37997f16 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -13,7 +13,8 @@ from portage.data import portage_gid, portage_uid, secpass, ostype from portage.dbapi import dbapi from portage.dep import use_reduce, paren_reduce, isvalidatom, \ isjustname, dep_getkey, match_from_list -from portage.exception import InvalidData, InvalidPackageName, \ +from portage.exception import CommandNotFound, \ + InvalidData, InvalidPackageName, \ FileNotFound, PermissionDenied, UnsupportedAPIException from portage.locks import lockdir, unlockdir from portage.output import bold, red, green @@ -29,6 +30,7 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ from portage.elog import elog_process from portage.elog.filtering import filter_mergephases, filter_unmergephases +from portage.cache.mappings import slot_dict_class import os, re, shutil, stat, errno, copy, subprocess import logging @@ -144,6 +146,10 @@ class LinkageMap(object): """Models dynamic linker dependencies.""" + _needed_aux_key = "NEEDED.ELF.2" + _soname_map_class = slot_dict_class( + ("consumers", "providers"), prefix="") + def __init__(self, vardbapi): self._dbapi = vardbapi self._root = self._dbapi.root @@ -167,6 +173,13 @@ class LinkageMap(object): self._path_key_cache[path] = key return key + def _obj_key(self, path): + key = self._obj_key_cache.get(path) + if key is None: + key = self._ObjectKey(path, self._root) + self._obj_key_cache[path] = key + return key + class _ObjectKey(object): """Helper class used as _obj_properties keys for objects.""" @@ -236,7 +249,12 @@ class LinkageMap(object): return str(sorted(self.alt_paths)) def rebuild(self, exclude_pkgs=None, include_file=None): + """ + Raises CommandNotFound if there are preserved libs + and the scanelf binary is not available. + """ root = self._root + root_len = len(root) - 1 self._clear_cache() self._defpath.update(getlibpaths(self._root)) libs = self._libs @@ -244,58 +262,102 @@ class LinkageMap(object): obj_properties = self._obj_properties lines = [] + + # Data from include_file is processed first so that it + # overrides any data from previously installed files. + if include_file is not None: + lines += grabfile(include_file) + + aux_keys = [self._needed_aux_key] for cpv in self._dbapi.cpv_all(): if exclude_pkgs is not None and cpv in exclude_pkgs: continue - lines += self._dbapi.aux_get(cpv, ["NEEDED.ELF.2"])[0].split('\n') + lines += self._dbapi.aux_get(cpv, aux_keys)[0].split('\n') # Cache NEEDED.* files avoid doing excessive IO for every rebuild. self._dbapi.flush_cache() - - if include_file: - lines += grabfile(include_file) - + # have to call scanelf for preserved libs here as they aren't # registered in NEEDED.ELF.2 files if self._dbapi.plib_registry and self._dbapi.plib_registry.getPreservedLibs(): args = [EPREFIX+"/usr/bin/scanelf", "-qF", "%a;%F;%S;%r;%n"] for items in self._dbapi.plib_registry.getPreservedLibs().values(): - args += [x.lstrip(".") for x in items] - proc = subprocess.Popen(args, stdout=subprocess.PIPE) - output = [l[3:] for l in proc.communicate()[0].split("\n")] - lines += output + args.extend(os.path.join(root, x.lstrip("." + os.sep)) \ + for x in items) + try: + proc = subprocess.Popen(args, stdout=subprocess.PIPE) + except EnvironmentError, e: + if e.errno != errno.ENOENT: + raise + raise CommandNotFound(args[0]) + else: + for l in proc.stdout: + l = l[3:].rstrip("\n") + if not l: + continue + fields = l.split(";") + if len(fields) < 5: + writemsg_level("\nWrong number of fields " + \ + "returned from scanelf: %s\n\n" % (l,), + level=logging.ERROR, noiselevel=-1) + continue + fields[1] = fields[1][root_len:] + lines.append(";".join(fields)) + proc.wait() for l in lines: - if l.strip() == "": + l = l.rstrip("\n") + if not l: continue - fields = l.strip("\n").split(";") + fields = l.split(";") if len(fields) < 5: - print "Error", fields - # insufficient field length + writemsg_level("\nWrong number of fields " + \ + "in %s: %s\n\n" % (self._needed_aux_key, l), + level=logging.ERROR, noiselevel=-1) continue arch = fields[0] obj = fields[1] - obj_key = self._ObjectKey(obj, root) soname = fields[2] path = set([normalize_path(x) \ for x in filter(None, fields[3].replace( "${ORIGIN}", os.path.dirname(obj)).replace( "$ORIGIN", os.path.dirname(obj)).split(":"))]) needed = filter(None, fields[4].split(",")) + + obj_key = self._obj_key(obj) + indexed = True + myprops = obj_properties.get(obj_key) + if myprops is None: + indexed = False + myprops = (arch, needed, path, soname, set()) + obj_properties[obj_key] = myprops + # All object paths are added into the obj_properties tuple. + myprops[4].add(obj) + + # Don't index the same file more that once since only one + # set of data can be correct and therefore mixing data + # may corrupt the index (include_file overrides previously + # installed). + if indexed: + continue + + arch_map = libs.get(arch) + if arch_map is None: + arch_map = {} + libs[arch] = arch_map if soname: - libs.setdefault(soname, \ - {arch: {"providers": set(), "consumers": set()}}) - libs[soname].setdefault(arch, \ - {"providers": set(), "consumers": set()}) - libs[soname][arch]["providers"].add(obj_key) - for x in needed: - libs.setdefault(x, \ - {arch: {"providers": set(), "consumers": set()}}) - libs[x].setdefault(arch, {"providers": set(), "consumers": set()}) - libs[x][arch]["consumers"].add(obj_key) - obj_key_cache.setdefault(obj, obj_key) - # All object paths are added into the obj_properties tuple - obj_properties.setdefault(obj_key, \ - (arch, needed, path, soname, set()))[4].add(obj) + soname_map = arch_map.get(soname) + if soname_map is None: + soname_map = self._soname_map_class( + providers=set(), consumers=set()) + arch_map[soname] = soname_map + soname_map.providers.add(obj_key) + for needed_soname in needed: + soname_map = arch_map.get(needed_soname) + if soname_map is None: + soname_map = self._soname_map_class( + providers=set(), consumers=set()) + arch_map[needed_soname] = soname_map + soname_map.consumers.add(obj_key) def listBrokenBinaries(self, debug=False): """ @@ -341,10 +403,7 @@ class LinkageMap(object): if obj in cache_self.cache: return cache_self.cache[obj] else: - if obj in self._obj_key_cache: - obj_key = self._obj_key_cache.get(obj) - else: - obj_key = self._ObjectKey(obj, self._root) + obj_key = self._obj_key(obj) # Check that the library exists on the filesystem. if obj_key.file_exists(): # Get the arch and soname from LinkageMap._obj_properties if @@ -458,7 +517,7 @@ class LinkageMap(object): """ basename = os.path.basename(obj) - obj_key = self._ObjectKey(obj, self._root) + obj_key = self._obj_key(obj) if obj_key not in self._obj_properties: raise KeyError("%s (%s) not in object list" % (obj_key, obj)) soname = self._obj_properties[obj_key][3] @@ -477,9 +536,9 @@ class LinkageMap(object): rValue = [] if not self._libs: self.rebuild() - for soname in self._libs: - for arch in self._libs[soname]: - for obj_key in self._libs[soname][arch]["providers"]: + for arch_map in self._libs.itervalues(): + for soname_map in arch_map.itervalues(): + for obj_key in soname_map.providers: rValue.extend(self._obj_properties[obj_key][4]) return rValue @@ -535,21 +594,19 @@ class LinkageMap(object): if obj_key not in self._obj_properties: raise KeyError("%s not in object list" % obj_key) else: - obj_key = self._obj_key_cache.get(obj) + obj_key = self._obj_key(obj) if obj_key not in self._obj_properties: - obj_key = self._ObjectKey(obj, self._root) - if obj_key not in self._obj_properties: - raise KeyError("%s (%s) not in object list" % (obj_key, obj)) + raise KeyError("%s (%s) not in object list" % (obj_key, obj)) arch, needed, path, _, _ = self._obj_properties[obj_key] path_keys = set(self._path_key(x) for x in path.union(self._defpath)) for soname in needed: rValue[soname] = set() - if soname not in self._libs or arch not in self._libs[soname]: + if arch not in self._libs or soname not in self._libs[arch]: continue # For each potential provider of the soname, add it to rValue if it # resides in the obj's runpath. - for provider_key in self._libs[soname][arch]["providers"]: + for provider_key in self._libs[arch][soname].providers: providers = self._obj_properties[provider_key][4] for provider in providers: if self._path_key(os.path.dirname(provider)) in path_keys: @@ -589,11 +646,9 @@ class LinkageMap(object): objs = self._obj_properties[obj_key][4] else: objs = set([obj]) - obj_key = self._obj_key_cache.get(obj) + obj_key = self._obj_key(obj) if obj_key not in self._obj_properties: - obj_key = self._ObjectKey(obj, self._root) - if obj_key not in self._obj_properties: - raise KeyError("%s (%s) not in object list" % (obj_key, obj)) + raise KeyError("%s (%s) not in object list" % (obj_key, obj)) # If there is another version of this lib with the # same soname and the master link points to that @@ -618,10 +673,10 @@ class LinkageMap(object): defpath_keys = set(self._path_key(x) for x in self._defpath) arch, _, _, soname, _ = self._obj_properties[obj_key] - if soname in self._libs and arch in self._libs[soname]: + if arch in self._libs and soname in self._libs[arch]: # For each potential consumer, add it to rValue if an object from the # arguments resides in the consumer's runpath. - for consumer_key in self._libs[soname][arch]["consumers"]: + for consumer_key in self._libs[arch][soname].consumers: _, _, path, _, consumer_objs = \ self._obj_properties[consumer_key] path_keys = defpath_keys.union(self._path_key(x) for x in path) @@ -1722,6 +1777,7 @@ class vardbapi(dbapi): f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) write_contents(new_contents, root, f) f.close() + pkg._clear_contents_cache() class _owners_cache(object): """ @@ -2148,6 +2204,7 @@ class dblink(object): self.contentscache = None self._contents_inodes = None self._contents_basenames = None + self._linkmap_broken = False def lockdb(self): if self._lock_vdb: @@ -2436,7 +2493,7 @@ class dblink(object): # already called LinkageMap.rebuild() and passed it's NEEDED file # in as an argument. if not others_in_slot: - self.vartree.dbapi.linkmap.rebuild(exclude_pkgs=(self.mycpv,)) + self._linkmap_rebuild(exclude_pkgs=(self.mycpv,)) # remove preserved libraries that don't have any consumers left cpv_lib_map = self._find_unused_preserved_libs() @@ -2881,158 +2938,142 @@ class dblink(object): return False - def _preserve_libs(self, srcroot, destroot, mycontents, counter, inforoot): - showMessage = self._display_merge - # read global reverse NEEDED map + def _linkmap_rebuild(self, **kwargs): + if self._linkmap_broken: + return + try: + self.vartree.dbapi.linkmap.rebuild(**kwargs) + except CommandNotFound, e: + self._linkmap_broken = True + self._display_merge("!!! Disabling preserve-libs " + \ + "due to error: Command Not Found: %s\n" % (e,), + level=logging.ERROR, noiselevel=-1) + + def _find_libs_to_preserve(self): + """ + Get CONTENTS entries for libraries to be preserved. Returns a + dict instance like that returned from getcontents(). + """ + if self._linkmap_broken or not \ + (self._installed_instance is not None and \ + "preserve-libs" in self.settings.features): + return None + linkmap = self.vartree.dbapi.linkmap - if ostype == "Darwin": - neededfile = "NEEDED.MACHO.3" - else: - neededfile = "NEEDED.ELF.2" - linkmap.rebuild(include_file=os.path.join(inforoot, neededfile)) - liblist = linkmap.listLibraryObjects() - - # get list of libraries from old package instance - root_len = len(self.myroot) - 1 - old_contents = set(p[root_len:] \ - for p in self._installed_instance.getcontents()) - old_libs = old_contents.intersection(liblist) - - # get list of libraries from new package instance - mycontents = set(os.path.join(os.path.sep, x) for x in mycontents) - mylibs = mycontents.intersection(liblist) - - # check which libs are present in the old, but not the new package instance - candidates = old_libs.difference(mylibs) - candidates_inodes = set() - for x in candidates: - x_destroot = os.path.join(destroot, x.lstrip(os.path.sep)) - try: - st = os.stat(x_destroot) - except OSError: - continue - candidates_inodes.add((st.st_dev, st.st_ino)) + installed_instance = self._installed_instance + old_contents = installed_instance.getcontents() + root = self.myroot + root_len = len(root) - 1 + lib_graph = digraph() + path_node_map = {} + + def path_to_node(path): + node = path_node_map.get(path) + if node is None: + node = LinkageMap._LibGraphNode(path, root) + alt_path_node = lib_graph.get(node) + if alt_path_node is not None: + node = alt_path_node + node.alt_paths.add(path) + path_node_map[path] = node + return node - for x in old_contents: - x_destroot = os.path.join(destroot, x.lstrip(os.path.sep)) - if not os.path.islink(x_destroot): + consumer_map = {} + provider_nodes = set() + # Create provider nodes and add them to the graph. + for f_abs in old_contents: + f = f_abs[root_len:] + if self.isowner(f, root): continue try: - st = os.stat(x_destroot) - except OSError: + consumers = linkmap.findConsumers(f) + except KeyError: continue - if (st.st_dev, st.st_ino) in candidates_inodes and \ - x not in mycontents: - candidates.add(x) - - provider_cache = {} - consumer_cache = {} - - # ignore any libs that are only internally used by the package - def has_external_consumers(lib, contents, otherlibs): - consumers = consumer_cache.get(lib) - if consumers is None: - consumers = linkmap.findConsumers(lib) - consumer_cache[lib] = consumers - contents_without_libs = [x for x in contents if x not in otherlibs] - - # just used by objects that will be autocleaned - if len(consumers.difference(contents_without_libs)) == 0: - return False - # used by objects that are referenced as well, need to check those - # recursively to break any reference cycles - elif len(consumers.difference(contents)) == 0: - otherlibs = set(otherlibs) - for ol in otherlibs.intersection(consumers): - if has_external_consumers(ol, contents, otherlibs.difference([lib])): - return True - return False - # used by external objects directly - else: - return True - - for lib in list(candidates): - if not has_external_consumers(lib, old_contents, candidates): - candidates.remove(lib) + if not consumers: continue - if linkmap.isMasterLink(lib): - candidates.remove(lib) + provider_node = path_to_node(f) + lib_graph.add(provider_node, None) + provider_nodes.add(provider_node) + consumer_map[provider_node] = consumers + + # Create consumer nodes and add them to the graph. + # Note that consumers can also be providers. + for provider_node, consumers in consumer_map.iteritems(): + for c in consumers: + if self.isowner(c, root): + continue + consumer_node = path_to_node(c) + if installed_instance.isowner(c, root) and \ + consumer_node not in provider_nodes: + # This is not a provider, so it will be uninstalled. + continue + lib_graph.add(provider_node, consumer_node) + + # Locate nodes which should be preserved. They consist of all + # providers that are reachable from consumers that are not + # providers themselves. + preserve_nodes = set() + for consumer_node in lib_graph.root_nodes(): + if consumer_node in provider_nodes: continue - # only preserve the lib if there is no other copy to use for each consumer - keep = False + # Preserve all providers that are reachable from this consumer. + node_stack = lib_graph.child_nodes(consumer_node) + while node_stack: + provider_node = node_stack.pop() + if provider_node in preserve_nodes: + continue + preserve_nodes.add(provider_node) + node_stack.extend(lib_graph.child_nodes(provider_node)) - lib_consumers = consumer_cache.get(lib) - if lib_consumers is None: - lib_consumers = linkmap.findConsumers(lib) - consumer_cache[lib] = lib_consumers + preserve_paths = set() + for preserve_node in preserve_nodes: + preserve_paths.update(preserve_node.alt_paths) - for c in lib_consumers: - localkeep = True - providers = provider_cache.get(c) - if providers is None: - providers = linkmap.findProviders(c) - provider_cache[c] = providers - - for soname in providers: - if lib in providers[soname]: - for p in providers[soname]: - if p not in candidates or os.path.exists(os.path.join(srcroot, p.lstrip(os.sep))): - localkeep = False - break - break - if localkeep: - keep = True - if not keep: - candidates.remove(lib) - continue - - del mylibs, mycontents, old_contents, liblist - - # inject files that should be preserved into our image dir - preserve_paths = [] - candidates_stack = list(candidates) - while candidates_stack: - x = candidates_stack.pop() - x_srcroot = os.path.join(srcroot, x.lstrip(os.path.sep)) - x_destroot = os.path.join(destroot, x.lstrip(os.path.sep)) - # skip existing files so the 'new' libs aren't overwritten - if os.path.exists(os.path.join(srcroot, x.lstrip(os.sep))): - continue - showMessage("injecting %s into %s\n" % (x, srcroot), - noiselevel=-1) - if not os.path.exists(x_destroot): - showMessage("%s does not exist so can't be preserved\n" % x, - noiselevel=-1) - continue - mydir = os.path.join(srcroot, os.path.dirname(x).lstrip(os.sep)) - if not os.path.exists(mydir): - os.makedirs(mydir) - - # resolve symlinks and extend preserve list - # NOTE: we're extending the list in the loop to emulate recursion to - # also get indirect symlinks - if os.path.islink(x_destroot): - linktarget = os.readlink(x_destroot) - os.symlink(linktarget, os.path.join(srcroot, x.lstrip(os.sep))) - if linktarget[0] != os.sep: - linktarget = os.path.join(os.path.dirname(x), linktarget) - if linktarget not in candidates: - candidates.add(linktarget) - candidates_stack.append(linktarget) - else: - shutil.copy2(x_destroot, x_srcroot) - preserve_paths.append(x) - - del candidates + return preserve_paths - # keep track of the libs we preserved - self.vartree.dbapi.plib_registry.register(self.mycpv, self.settings["SLOT"], counter, preserve_paths) + def _add_preserve_libs_to_contents(self, preserve_paths): + """ + Preserve libs returned from _find_libs_to_preserve(). + """ + + if not preserve_paths: + return + + showMessage = self._display_merge + root = self.myroot + + # Copy contents entries from the old package to the new one. + new_contents = self.getcontents().copy() + old_contents = self._installed_instance.getcontents() + for f in list(preserve_paths): + f_abs = os.path.join(root, f.lstrip(os.sep)) + new_contents[f_abs] = old_contents[f_abs] + if os.path.islink(f_abs): + obj_type = "sym" + else: + obj_type = "obj" + showMessage(">>> needed %s %s\n" % (obj_type, f_abs)) + # Add parent directories to contents if necessary. + parent_dir = os.path.dirname(f_abs) + while parent_dir != root: + new_contents[parent_dir] = ["dir"] + prev = parent_dir + parent_dir = os.path.dirname(parent_dir) + if prev == parent_dir: + break + outfile = atomic_ofstream(os.path.join(self.dbtmpdir, "CONTENTS")) + write_contents(new_contents, root, outfile) + outfile.close() + self._clear_contents_cache() def _find_unused_preserved_libs(self): """ Find preserved libraries that don't have any consumers left. """ + if self._linkmap_broken: + return {} + # Since preserved libraries can be consumers of other preserved # libraries, use a graph to track consumer relationships. plib_dict = self.vartree.dbapi.plib_registry.getPreservedLibs() @@ -3479,18 +3520,6 @@ class dblink(object): max_dblnk = dblnk self._installed_instance = max_dblnk - # get current counter value (counter_tick also takes care of incrementing it) - # XXX Need to make this destroot, but it needs to be initialized first. XXX - # XXX bis: leads to some invalidentry() call through cp_all(). - # Note: The counter is generated here but written later because preserve_libs - # needs the counter value but has to be before dbtmpdir is made (which - # has to be before the counter is written) - genone - counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) - - # Save this for unregistering preserved-libs if the merge fails. - self.settings["COUNTER"] = str(counter) - self.settings.backup_changes("COUNTER") - myfilelist = [] mylinklist = [] def onerror(e): @@ -3543,10 +3572,6 @@ class dblink(object): if installed_files: return 1 - # Preserve old libs if they are still in use - if slot_matches and "preserve-libs" in self.settings.features: - self._preserve_libs(srcroot, destroot, myfilelist+mylinklist, counter, inforoot) - # check for package collisions blockers = None if self._blockers is not None: @@ -3693,6 +3718,7 @@ class dblink(object): self.copyfile(inforoot+"/"+x) # write local package counter for recording + counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) lcfile = open(os.path.join(self.dbtmpdir, "COUNTER"),"w") lcfile.write(str(counter)) lcfile.close() @@ -3775,25 +3801,23 @@ class dblink(object): gid=portage_gid, mode=02750, mask=02) writedict(cfgfiledict, conf_mem_file) - # TODO: In case some elf files collide with blocked packages, - # ensure that NEEDED data from include_file overrides the stale - # NEEDED data from the colliding files in the blocked packages. - exclude_pkgs = set(dblnk.mycpv for dblnk in others_in_slot) - if ostype == "Darwin": - neededfile = "NEEDED.MACHO.3" - else: - neededfile = "NEEDED.ELF.2" - self.vartree.dbapi.linkmap.rebuild(exclude_pkgs=exclude_pkgs, - include_file=os.path.join(inforoot, neededfile)) - # These caches are populated during collision-protect and the data # they contain is now invalid. It's very important to invalidate # the contents_inodes cache so that FEATURES=unmerge-orphans # doesn't unmerge anything that belongs to this package that has # just been merged. - others_in_slot.append(self) # self has just been merged for dblnk in others_in_slot: dblnk._clear_contents_cache() + self._clear_contents_cache() + + linkmap = self.vartree.dbapi.linkmap + self._linkmap_rebuild(include_file=os.path.join(inforoot, + linkmap._needed_aux_key)) + + # Preserve old libs if they are still in use + preserve_paths = self._find_libs_to_preserve() + if preserve_paths: + self._add_preserve_libs_to_contents(preserve_paths) # If portage is reinstalling itself, remove the old # version now since we want to use the temporary @@ -3804,6 +3828,7 @@ class dblink(object): reinstall_self = True autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" + others_in_slot.append(self) # self has just been merged for dblnk in list(others_in_slot): if dblnk is self: continue @@ -3811,6 +3836,7 @@ class dblink(object): continue showMessage(">>> Safely unmerging already-installed instance...\n") others_in_slot.remove(dblnk) # dblnk will unmerge itself now + dblnk._linkmap_broken = self._linkmap_broken dblnk.unmerge(trimworld=0, ldpath_mtimes=prev_mtimes, others_in_slot=others_in_slot) # TODO: Check status and abort if necessary. @@ -3829,6 +3855,11 @@ class dblink(object): self.delete() _movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) + # keep track of the libs we preserved + if preserve_paths: + self.vartree.dbapi.plib_registry.register(self.mycpv, + slot, counter, sorted(preserve_paths)) + # Check for file collisions with blocking packages # and remove any colliding files from their CONTENTS # since they now belong to this package. @@ -3860,9 +3891,6 @@ class dblink(object): self.vartree.dbapi._add(self) contents = self.getcontents() - # regenerate reverse NEEDED map - self.vartree.dbapi.linkmap.rebuild() - #do postinst script self.settings["PORTAGE_UPDATE_ENV"] = \ os.path.join(self.dbpkgdir, "environment.bz2") @@ -3899,7 +3927,7 @@ class dblink(object): # For gcc upgrades, preserved libs have to be removed after the # the library path has been updated. - self.vartree.dbapi.linkmap.rebuild() + self._linkmap_rebuild() cpv_lib_map = self._find_unused_preserved_libs() if cpv_lib_map: self._remove_preserved_libs(cpv_lib_map) @@ -4234,9 +4262,7 @@ class dblink(object): self.vartree.dbapi.plib_registry.pruneNonExisting() retval = self.treewalk(mergeroot, myroot, inforoot, myebuild, cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) - # undo registrations of preserved libraries, bug #210501 - if retval != os.EX_OK: - self.vartree.dbapi.plib_registry.unregister(self.mycpv, self.settings["SLOT"], self.settings["COUNTER"]) + # Process ebuild logfiles elog_process(self.mycpv, self.settings, phasefilter=filter_mergephases) if retval == os.EX_OK and "noclean" not in self.settings.features: diff --git a/pym/portage/sets/libs.py b/pym/portage/sets/libs.py index b41653575..3722c8462 100644 --- a/pym/portage/sets/libs.py +++ b/pym/portage/sets/libs.py @@ -27,7 +27,8 @@ class PreservedLibraryConsumerSet(LibraryConsumerSet): reg = self.dbapi.plib_registry consumers = set() if reg: - for libs in reg.getPreservedLibs().values(): + plib_dict = reg.getPreservedLibs() + for libs in plib_dict.itervalues(): for lib in libs: if self.debug: print lib @@ -35,6 +36,10 @@ class PreservedLibraryConsumerSet(LibraryConsumerSet): print " ", x print "-"*40 consumers.update(self.dbapi.linkmap.findConsumers(lib)) + # Don't rebuild packages just because they contain preserved + # libs that happen to be consumers of other preserved libs. + for libs in plib_dict.itervalues(): + consumers.difference_update(libs) else: return if not consumers: -- 2.26.2