Merged from trunk 10760:10764

   | 10761   | Bug #228939 - Fix display list interaction with --tree mode  |
   | zmedico | to prevent display of a duplicate list.                      |

   | 10762   | Bug #228977 - Protect against possible infinite loop in      |
   | zmedico | dblink._preserve_libs().                                     |

   | 10763   | Bug #228935 - Add a new "command not found" build log QA     |
   | zmedico | Notice. Thanks to Olivier Huber for the initial patch.       |

   | 10764   | Bug #228977 - During dblink._preserve_libs() calls, cache    |
   | zmedico | results of LinkageMap.findProviders(),                       |
   |         | LinkageMap.findConsumers(), and os.path.realpath() calls in  |
   |         | order to improve performance. This makes a huge difference   |
   |         | in performance for glibc upgrades since glibc provides so    |
   |         | many libs to check consumerge for.                           |

svn path=/main/branches/prefix/; revision=10767

diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index 0ee96893149c165dc86841c162d19093a03f45a6..16ebae618e3fd084ba00e14f1b1007282b27775f 100644 (file)
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -4436,7 +4436,10 @@ class depgraph(object):
 
        def _show_merge_list(self):
                if self._serialized_tasks_cache is not None and \
-                       self._serialized_tasks_cache != self._displayed_list:
+                       not (self._displayed_list and \
+                       (self._displayed_list == self._serialized_tasks_cache or \
+                       self._displayed_list == \
+                               list(reversed(self._serialized_tasks_cache)))):
                        display_list = self._serialized_tasks_cache[:]
                        if "--tree" in self.myopts:
                                display_list.reverse()

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 28161c1d9a3970f5742c40a2b989fc5c769b60d6..65f66f59bfb433fd277b4ae28893881323d8eaed 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -4213,6 +4213,11 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
                                pass
                        else:
                                am_maintainer_mode = []
+
+                               bash_command_not_found = []
+                               bash_command_not_found_re = re.compile(
+                                       r'(.*): line (\d*): (.*): command not found$')
+
                                configure_opts_warn = []
                                configure_opts_warn_re = re.compile(
                                        r'^configure: WARNING: Unrecognized options: .*')
@@ -4224,6 +4229,10 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
                                                if am_maintainer_mode_re.search(line) is not None and \
                                                        am_maintainer_mode_exclude_re.search(line) is None:
                                                        am_maintainer_mode.append(line.rstrip("\n"))
+
+                                               if bash_command_not_found_re.match(line) is not None:
+                                                       bash_command_not_found.append(line.rstrip("\n"))
+
                                                if configure_opts_warn_re.match(line) is not None:
                                                        configure_opts_warn.append(line.rstrip("\n"))
                                finally:
@@ -4254,6 +4263,12 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
                                                wrap_width))
                                        _eqawarn(msg)
 
+                               if bash_command_not_found:
+                                       msg = ["QA Notice: command not found:"]
+                                       msg.append("")
+                                       msg.extend("\t" + line for line in bash_command_not_found)
+                                       _eqawarn(msg)
+
                                if configure_opts_warn:
                                        msg = ["QA Notice: Unrecognized configure options:"]
                                        msg.append("")

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 3cbc8dee76e7b405af76dd8ca85067779e9ee9e8..b0e52dbe7f0ebd9ab80f1130860268ef4c11e1da 100644 (file)
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -201,30 +201,48 @@ class LinkageMap(object):
        def findProviders(self, obj):
                if not self._libs:
                        self.rebuild()
+
+               realpath_cache = {}
+               def realpath(p):
+                       real_path = realpath_cache.get(p)
+                       if real_path is None:
+                               real_path = os.path.realpath(p)
+                               realpath_cache[p] = real_path
+                       return real_path
+
                rValue = {}
                if obj not in self._obj_properties:
-                       obj = os.path.realpath(obj)
+                       obj = realpath(obj)
                        if obj not in self._obj_properties:
                                raise KeyError("%s not in object list" % obj)
                arch, needed, path, soname = self._obj_properties[obj]
                path.extend(self._defpath)
-               path = [os.path.realpath(x) for x in path]
+               path = set(realpath(x) for x in path)
                for x in needed:
                        rValue[x] = set()
                        if x not in self._libs or arch not in self._libs[x]:
                                continue
                        for y in self._libs[x][arch]["providers"]:
-                               if x[0] == os.sep and os.path.realpath(x) == os.path.realpath(y):
+                               if x[0] == os.sep and realpath(x) == realpath(y):
                                        rValue[x].add(y)
-                               elif os.path.realpath(os.path.dirname(y)) in path:
+                               elif realpath(os.path.dirname(y)) in path:
                                        rValue[x].add(y)
                return rValue
        
        def findConsumers(self, obj):
                if not self._libs:
                        self.rebuild()
+
+               realpath_cache = {}
+               def realpath(p):
+                       real_path = realpath_cache.get(p)
+                       if real_path is None:
+                               real_path = os.path.realpath(p)
+                               realpath_cache[p] = real_path
+                       return real_path
+
                if obj not in self._obj_properties:
-                       obj = os.path.realpath(obj)
+                       obj = realpath(obj)
                        if obj not in self._obj_properties:
                                raise KeyError("%s not in object list" % obj)
                rValue = set()
@@ -233,10 +251,10 @@ class LinkageMap(object):
                                if obj in self._libs[soname][arch]["providers"]:
                                        for x in self._libs[soname][arch]["consumers"]:
                                                path = self._obj_properties[x][2]
-                                               path = [os.path.realpath(y) for y in path+self._defpath]
-                                               if soname[0] == os.sep and os.path.realpath(soname) == os.path.realpath(obj):
+                                               path = [realpath(y) for y in path+self._defpath]
+                                               if soname[0] == os.sep and realpath(soname) == realpath(obj):
                                                        rValue.add(x)
-                                               elif os.path.realpath(os.path.dirname(obj)) in path:
+                                               elif realpath(os.path.dirname(obj)) in path:
                                                        rValue.add(x)
                return rValue
                                        
@@ -335,10 +353,19 @@ class LinkageMapMachO(object):
        def findProviders(self, obj):
                if not self._libs:
                        self.rebuild()
+
+               realpath_cache = {}
+               def realpath(p):
+                       real_path = realpath_cache.get(p)
+                       if real_path is None:
+                               real_path = os.path.realpath(p)
+                               realpath_cache[p] = real_path
+                       return real_path
+
                obj = os.path.normpath(obj)
                rValue = {}
                if obj not in self._obj_properties:
-                       obj = os.path.realpath(obj)
+                       obj = realpath(obj)
                        if obj not in self._obj_properties:
                                raise KeyError("%s not in object list" % obj)
                needed, install_name = self._obj_properties[obj]
@@ -348,13 +375,14 @@ class LinkageMapMachO(object):
                        if x not in self._libs:
                                continue
                        for y in self._libs[x]["providers"]:
-                               if os.path.realpath(x) == os.path.realpath(y):
+                               if realpath(x) == realpath(y):
                                        rValue[x].add(y)
                return rValue
        
        def findConsumers(self, obj):
                if not self._libs:
                        self.rebuild()
+
                obj = os.path.normpath(obj)
                if obj not in self._obj_properties:
                        obj = os.path.realpath(obj)
@@ -2128,9 +2156,15 @@ class dblink(object):
                        if os.path.islink(x) and os.path.realpath(x) in candidates and x not in mycontents:
                                candidates.add(x)
 
+               provider_cache = {}
+               consumer_cache = {}
+
                # ignore any libs that are only internally used by the package
                def has_external_consumers(lib, contents, otherlibs):
-                       consumers = linkmap.findConsumers(lib)
+                       consumers = consumer_cache.get(lib)
+                       if consumers is None:
+                               consumers = linkmap.findConsumers(lib)
+                               consumer_cache[lib] = consumers
                        contents_without_libs = [x for x in contents if x not in otherlibs]
                        
                        # just used by objects that will be autocleaned
@@ -2157,10 +2191,19 @@ class dblink(object):
                                continue
                        # only preserve the lib if there is no other copy to use for each consumer
                        keep = False
-                       for c in linkmap.findConsumers(lib):
+
+                       lib_consumers = consumer_cache.get(lib)
+                       if lib_consumers is None:
+                               lib_consumers = linkmap.findConsumers(lib)
+                               consumer_cache[lib] = lib_consumers
+
+                       for c in lib_consumers:
                                localkeep = True
-                               providers = linkmap.findProviders(c)
-                               
+                               providers = provider_cache.get(c)
+                               if providers is None:
+                                       providers = linkmap.findProviders(c)
+                                       provider_cache[c] = providers
+
                                for soname in providers:
                                        if lib in providers[soname]:
                                                for p in providers[soname]:
@@ -2201,8 +2244,9 @@ class dblink(object):
                                os.symlink(linktarget, os.path.join(srcroot, x.lstrip(os.sep)))
                                if linktarget[0] != os.sep:
                                        linktarget = os.path.join(os.path.dirname(x), linktarget)
-                               candidates.add(linktarget)
-                               candidates_stack.append(linktarget)
+                               if linktarget not in candidates:
+                                       candidates.add(linktarget)
+                                       candidates_stack.append(linktarget)
                        else:
                                shutil.copy2(os.path.join(destroot, x.lstrip(os.sep)),
                                        os.path.join(srcroot, x.lstrip(os.sep)))