From d098b35c455c161714ea6fbdb36766a9663c92ae Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 30 Oct 2008 21:13:23 +0000 Subject: [PATCH] Merged from trunk -r11745:11753 | 11746 | Bug #225429 - In dblink.unmerge(), eliminate consumers | | zmedico | having providers with the same soname as an installed | | | library that is not preserved. This eliminates libraries | | | that are erroneously preserved due to a move from one | | | directory to another. | | 11747 | In dblink.unmerge(), update the CONTENTS entries when | | zmedico | preserved libs are removed. | | 11748 | Bug # 225429 - Try to remove unneeded preserved libs just | | zmedico | before returning from dblink.treewalk(), after the library | | | path has been updated. This is intended to remove unneeded | | | preserved libs after a gcc upgrade. TODO: Figure out why | | | libgomp.so.1 still isn't properly removed. | | 11749 | Bug #225429 - Inside LinkageMap.rebuild(), update | | zmedico | self._defpath since it can change during a gcc upgrade. | | 11751 | define myunadded such that repoman doesn't crash when being | | zmedico | run in a non-{cvs,svn} dir (branches/prefix r11750) | | 11752 | Avoid unnecessary aux_get calls inside Scheduler._pkg() by | | zmedico | getting the existing Package instance from the digraph. | | 11753 | Make sure mychanged and mynew are always defined even if | | zmedico | there is no vcs because commit mode is allowed without a vcs | | | in pretend mode. | svn path=/main/branches/prefix/; revision=11754 --- bin/repoman | 2 + pym/_emerge/__init__.py | 11 +- pym/portage/dbapi/vartree.py | 227 ++++++++++++++++++++++++++--------- 3 files changed, 181 insertions(+), 59 deletions(-) diff --git a/bin/repoman b/bin/repoman index dc8543fdf..b0fc66478 100755 --- a/bin/repoman +++ b/bin/repoman @@ -748,6 +748,8 @@ else: new_ebuilds = set() modified_changelogs = set() +mychanged = [] +mynew = [] if vcs == "cvs": mycvstree = cvstree.getentries("./", recursive=1) mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./") diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 2295d9e19..8122cb76a 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -10387,6 +10387,13 @@ class Scheduler(PollScheduler): if installed: operation = "nomerge" + if self._digraph is not None: + # Reuse existing instance when available. + pkg = self._digraph.get( + (type_name, root_config.root, cpv, operation)) + if pkg is not None: + return pkg + tree_type = depgraph.pkg_tree_map[type_name] db = root_config.trees[tree_type].dbapi db_keys = list(self.trees[root_config.root][ @@ -10399,10 +10406,6 @@ class Scheduler(PollScheduler): settings.setcpv(pkg) pkg.metadata["USE"] = settings["PORTAGE_USE"] - if self._digraph is not None: - # Reuse existing instance when available. - pkg = self._digraph.get(pkg, pkg) - return pkg class MetadataRegen(PollScheduler): diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 2c6e32a35..e200a36c5 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -220,6 +220,7 @@ class LinkageMap(object): def rebuild(self, exclude_pkgs=None, include_file=None): root = self._root + self._defpath = set(getlibpaths(root)) libs = {} obj_key_cache = {} obj_properties = {} @@ -480,6 +481,11 @@ class LinkageMap(object): """ if not self._libs: self.rebuild() + if isinstance(obj, self._ObjectKey): + obj_key = obj + if obj_key not in self._obj_properties: + raise KeyError("%s not in object list" % obj_key) + return self._obj_properties[obj_key][3] if obj not in self._obj_key_cache: raise KeyError("%s not in object list" % obj) return self._obj_properties[self._obj_key_cache[obj]][3] @@ -930,6 +936,11 @@ class LinkageMapMachO(object): """ if not self._libs: self.rebuild() + if isinstance(obj, self._ObjectKey): + obj_key = obj + if obj_key not in self._obj_properties: + raise KeyError("%s not in object list" % obj_key) + return self._obj_properties[obj_key][2] if obj not in self._obj_key_cache: raise KeyError("%s not in object list" % obj) return self._obj_properties[self._obj_key_cache[obj]][2] @@ -2393,61 +2404,21 @@ class dblink(object): self.vartree.dbapi.linkmap.rebuild(exclude_pkgs=(self.mycpv,)) # remove preserved libraries that don't have any consumers left - # Since preserved libraries can be consumers of other preserved - # libraries, use a graph to track consumer relationships. - plib_dict = plib_registry.getPreservedLibs() - lib_graph = digraph() - preserved_nodes = set() - root = self.myroot - for plibs in plib_dict.itervalues(): - for f in plibs: - preserved_node = LinkageMap._LibGraphNode(f, root) - if not preserved_node.file_exists(): + cpv_lib_map = self._find_unused_preserved_libs() + if cpv_lib_map: + self._remove_preserved_libs(cpv_lib_map) + for cpv, removed in cpv_lib_map.iteritems(): + if not self.vartree.dbapi.cpv_exists(cpv): + for dblnk in others_in_slot: + if dblnk.mycpv == cpv: + # This one just got merged so it doesn't + # register with cpv_exists() yet. + self.vartree.dbapi.removeFromContents( + dblnk, removed) + break continue - existing_node = lib_graph.get(preserved_node) - if existing_node is not None: - preserved_node = existing_node - else: - lib_graph.add(preserved_node, None) - preserved_node.alt_paths.add(f) - preserved_nodes.add(preserved_node) - for c in self.vartree.dbapi.linkmap.findConsumers(f): - consumer_node = LinkageMap._LibGraphNode(c, root) - if not consumer_node.file_exists(): - continue - # Note that consumers may also be providers. - existing_node = lib_graph.get(consumer_node) - if existing_node is not None: - consumer_node = existing_node - consumer_node.alt_paths.add(c) - lib_graph.add(preserved_node, consumer_node) - - while not lib_graph.empty(): - root_nodes = preserved_nodes.intersection(lib_graph.root_nodes()) - if not root_nodes: - break - lib_graph.difference_update(root_nodes) - unlink_list = set() - for node in root_nodes: - unlink_list.update(node.alt_paths) - unlink_list = sorted(unlink_list) - for obj in unlink_list: - obj = os.path.join(root, obj.lstrip(os.sep)) - if os.path.islink(obj): - obj_type = "sym" - else: - obj_type = "obj" - try: - os.unlink(obj) - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e - else: - showMessage("<<< !needed %s %s\n" % (obj_type, obj)) + self.vartree.dbapi.removeFromContents(cpv, removed) - plib_registry.pruneNonExisting() - finally: if builddir_lock: try: @@ -3023,8 +2994,143 @@ class dblink(object): # keep track of the libs we preserved self.vartree.dbapi.plib_registry.register(self.mycpv, self.settings["SLOT"], counter, preserve_paths) - del preserve_paths - + def _find_unused_preserved_libs(self): + """ + Find preserved libraries that don't have any consumers left. + """ + + # Since preserved libraries can be consumers of other preserved + # libraries, use a graph to track consumer relationships. + plib_dict = self.vartree.dbapi.plib_registry.getPreservedLibs() + lib_graph = digraph() + preserved_nodes = set() + preserved_paths = set() + path_cpv_map = {} + path_node_map = {} + root = self.myroot + + def path_to_node(path): + node = path_node_map.get(path) + if node is None: + node = LinkageMap._LibGraphNode(path, root) + alt_path_node = lib_graph.get(node) + if alt_path_node is not None: + node = alt_path_node + node.alt_paths.add(path) + path_node_map[path] = node + return node + + linkmap = self.vartree.dbapi.linkmap + for cpv, plibs in plib_dict.iteritems(): + for f in plibs: + path_cpv_map[f] = cpv + preserved_node = path_to_node(f) + if not preserved_node.file_exists(): + continue + lib_graph.add(preserved_node, None) + preserved_paths.add(f) + preserved_nodes.add(preserved_node) + for c in self.vartree.dbapi.linkmap.findConsumers(f): + consumer_node = path_to_node(c) + if not consumer_node.file_exists(): + continue + # Note that consumers may also be providers. + lib_graph.add(preserved_node, consumer_node) + + # Eliminate consumers having providers with the same soname as an + # installed library that is not preserved. This eliminates + # libraries that are erroneously preserved due to a move from one + # directory to another. + provider_cache = {} + for preserved_node in preserved_nodes: + soname = linkmap.getSoname(preserved_node) + for consumer_node in lib_graph.parent_nodes(preserved_node): + if consumer_node in preserved_nodes: + continue + providers = provider_cache.get(consumer_node) + if providers is None: + providers = linkmap.findProviders(consumer_node) + provider_cache[consumer_node] = providers + providers = providers.get(soname) + if providers is None: + continue + for provider in providers: + if provider in preserved_paths: + continue + provider_node = path_to_node(provider) + if not provider_node.file_exists(): + continue + if provider_node in preserved_nodes: + continue + # An alternative provider seems to be + # installed, so drop this edge. + lib_graph.remove_edge(preserved_node, consumer_node) + break + + cpv_lib_map = {} + while not lib_graph.empty(): + root_nodes = preserved_nodes.intersection(lib_graph.root_nodes()) + if not root_nodes: + break + lib_graph.difference_update(root_nodes) + unlink_list = set() + for node in root_nodes: + unlink_list.update(node.alt_paths) + unlink_list = sorted(unlink_list) + for obj in unlink_list: + cpv = path_cpv_map[obj] + removed = cpv_lib_map.get(cpv) + if removed is None: + removed = set() + cpv_lib_map[cpv] = removed + removed.add(obj) + + return cpv_lib_map + + def _remove_preserved_libs(self, cpv_lib_map): + """ + Remove files returned from _find_unused_preserved_libs(). + """ + + files_to_remove = set() + for files in cpv_lib_map.itervalues(): + files_to_remove.update(files) + files_to_remove = sorted(files_to_remove) + showMessage = self._display_merge + root = self.myroot + + parent_dirs = set() + for obj in files_to_remove: + obj = os.path.join(root, obj.lstrip(os.sep)) + parent_dirs.add(os.path.dirname(obj)) + if os.path.islink(obj): + obj_type = "sym" + else: + obj_type = "obj" + try: + os.unlink(obj) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + else: + showMessage("<<< !needed %s %s\n" % (obj_type, obj)) + + # Remove empty parent directories if possible. + while parent_dirs: + x = parent_dirs.pop() + while True: + try: + os.rmdir(x) + except OSError: + break + prev = x + x = os.path.dirname(x) + if x == prev: + break + + self.vartree.dbapi.plib_registry.pruneNonExisting() + def _collision_protect(self, srcroot, destroot, mypkglist, mycontents): collision_ignore = set([normalize_path(myignore) for myignore in \ shlex.split(self.settings.get("COLLISION_IGNORE", ""))]) @@ -3680,6 +3786,17 @@ class dblink(object): contents=contents, env=self.settings.environ(), writemsg_level=self._display_merge) + # For gcc upgrades, preserved libs have to be removed after the + # the library path has been updated. + self.vartree.dbapi.linkmap.rebuild() + cpv_lib_map = self._find_unused_preserved_libs() + if cpv_lib_map: + self._remove_preserved_libs(cpv_lib_map) + for cpv, removed in cpv_lib_map.iteritems(): + if not self.vartree.dbapi.cpv_exists(cpv): + continue + self.vartree.dbapi.removeFromContents(cpv, removed) + return os.EX_OK def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime): -- 2.26.2