Merged from trunk 5247:5256
authorFabian Groffen <grobian@gentoo.org>
Sun, 10 Dec 2006 11:23:47 +0000 (11:23 -0000)
committerFabian Groffen <grobian@gentoo.org>
Sun, 10 Dec 2006 11:23:47 +0000 (11:23 -0000)
For bug #128809, process blockers for all installed packages.  Due to
the performance penalty incurred by additional dep_check calls, this is
only enabled with the --deep option.

Avoid unnecessary dbapi.cp_list() calls inside cpv_expand().

Cache the results of match_from_list() calls.

Cache results of dep_getcpv() calls.

Use dep_getcpv() inside dep_expand() for code sharing and caching.

Cache the results of fakedbapi.match() calls and clear the cache
automatically when necessary.

Initialize the match cache in the bindbapi constructor.

For bug #128809, optimize dep_check calls and enable the blocker
detection by default since the performance now seems acceptable.

Make sure the SLOT comes from the vardb instead of the portdb (for
multislot,   etc..). Thanks to jakub for reporting.

svn path=/main/branches/prefix/; revision=5259

bin/emerge
pym/portage.py
pym/portage_dep.py

index c296e8c2b858b6472cf5ac1d087d172b02af66c7..6d744fe12e042df78322f5826aada0469327121c 100755 (executable)
@@ -1191,7 +1191,8 @@ class depgraph:
                                        missing += 1
                                        print "Missing binary for:",xs[2]
 
-               self.validate_blockers()
+               if not self.validate_blockers():
+                       return False, myfavorites
                
                # We're true here unless we are missing binaries.
                return (not missing,myfavorites)
@@ -1438,6 +1439,67 @@ class depgraph:
                correct merge order such that mutually blocking packages are never
                installed simultaneously."""
 
+               #if "deep" in self.myparams:
+               if True:
+                       # Pull in blockers from all installed packages that haven't already
+                       # been pulled into the depgraph.  This is not enabled by default
+                       # due to the performance penalty that is incurred by all the
+                       # additional dep_check calls that are required.
+                       dep_keys = ["DEPEND","RDEPEND","PDEPEND"]
+                       for myroot in self.trees:
+                               pkg_node_map = self.pkg_node_map[myroot]
+                               vardb = self.trees[myroot]["vartree"].dbapi
+                               portdb = self.trees[myroot]["porttree"].dbapi
+                               pkgsettings = self.pkgsettings[myroot]
+                               final_db = self.mydbapi[myroot]
+                               for pkg in self.trees[myroot]["vartree"].dbapi.cpv_all():
+                                       blocker_atoms = None
+                                       matching_node = pkg_node_map.get(pkg, None)
+                                       if not matching_node or \
+                                               matching_node[3] == "merge":
+                                               # If this node has any blockers, create a "nomerge"
+                                               # node for it so that they can be enforced.
+                                               self.spinner.update()
+                                               try:
+                                                       dep_vals = portdb.aux_get(pkg, dep_keys)
+                                                       ptype = "ebuild"
+                                               except KeyError:
+                                                       if myroot not in self.global_updates:
+                                                               self.global_updates[myroot] = \
+                                                                       grab_global_updates(pkgsettings["PORTDIR"])
+                                                       perform_global_updates(
+                                                               pkg, vardb, self.global_updates[myroot])
+                                                       dep_vals = vardb.aux_get(pkg, dep_keys)
+                                                       ptype = "installed"
+                                               myuse = vardb.aux_get(pkg, ["USE"])[0]
+                                               depstr = " ".join(dep_vals)
+                                               # It is crucial to pass in final_db here in order to
+                                               # optimize dep_check calls by eliminating atoms via
+                                               # dep_wordreduce and dep_eval calls.
+                                               success, atoms = portage.dep_check(depstr, final_db,
+                                                       pkgsettings, myuse=myuse, trees=self.trees,
+                                                       myroot=myroot)
+                                               if not success:
+                                                       print "\n\n"
+                                                       print "Error occurred while processing",pkg
+                                                       print str(atoms)
+                                                       return False
+                                               blocker_atoms = [myatom for myatom in atoms \
+                                                       if myatom.startswith("!")]
+                                       if blocker_atoms:
+                                               # Don't store this parent in pkg_node_map, because it's
+                                               # not needed there and it might overwrite a "merge"
+                                               # node with the same cpv.
+                                               myparent = (ptype, myroot, pkg, "nomerge")
+                                               for myatom in blocker_atoms:
+                                                       blocker = ("blocks", myroot, myatom[1:])
+                                                       myparents = \
+                                                               self.blocker_parents.get(blocker, None)
+                                                       if not myparents:
+                                                               myparents = set()
+                                                               self.blocker_parents[blocker] = myparents
+                                                       myparents.add(myparent)
+
                for blocker in self.blocker_parents.keys():
                        mytype, myroot, mydep = blocker
                        initial_db = self.trees[myroot]["vartree"].dbapi
@@ -1510,6 +1572,7 @@ class depgraph:
                # Validate blockers that depend on merge order.
                if not self.blocker_digraph.empty():
                        self.altlist()
+               return True
 
        def altlist(self, reversed=False):
                if reversed in self._altlist_cache:
@@ -1711,7 +1774,8 @@ class depgraph:
                        except ValueError:
                                missing_atoms.append(mydep)
 
-               self.validate_blockers()
+               if not self.validate_blockers():
+                       return False
 
                if world_problems:
                        print >> sys.stderr, "\n!!! Problems have been detected with your world file"
@@ -1861,7 +1925,7 @@ class depgraph:
                                        pkg_key, mydb=vardb, settings=pkgsettings)
                                print "["+x[0]+" "+addl+"]",red(resolved),
                                block_parents = self.blocker_parents[tuple(x)]
-                               block_parents = [pnode[2] for pnode in block_parents]
+                               block_parents = set([pnode[2] for pnode in block_parents])
                                block_parents = ", ".join(block_parents)
                                if resolved!=x[2]:
                                        print bad("(\"%s\" is blocking %s)") % \
@@ -3936,6 +4000,8 @@ def action_depclean(settings, trees, ldpath_mtimes,
                try:
                        # Prefer live ebuild metadata when available.
                        aux_vals = portdb.aux_get(cpv, fakedb_auxkeys)
+                       # SLOT always comes from the vardb, for multislot.
+                       aux_vals[-1] = vardb.aux_get(cpv, ["SLOT"])[0]
                        live_ebuild = True
                except KeyError:
                        aux_vals = vardb.aux_get(cpv, fakedb_auxkeys)
index d66dfee66334dfedb40e818ec8bc9e38e1853699..d7678ecc08ec6c6b1ae02e56f33a1b585147bf58 100644 (file)
@@ -3844,21 +3844,11 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
                return mydep
        if mydep[0]=="*":
                mydep=mydep[1:]
-       prefix=""
-       postfix=""
-       if mydep[-1]=="*":
-               mydep=mydep[:-1]
-               postfix="*"
-       if mydep[:2] in [ ">=", "<=" ]:
-               prefix=mydep[:2]
-               mydep=mydep[2:]
-       elif mydep[:1] in "=<>~!":
-               prefix=mydep[:1]
-               mydep=mydep[1:]
-       colon = mydep.rfind(":")
-       if colon != -1:
-               postfix = mydep[colon:]
-               mydep = mydep[:colon]
+       orig_dep = mydep
+       mydep = dep_getcpv(orig_dep)
+       myindex = orig_dep.index(mydep)
+       prefix = orig_dep[:myindex]
+       postfix = orig_dep[myindex+len(mydep):]
        return prefix + cpv_expand(
                mydep, mydb=mydb, use_cache=use_cache, settings=settings) + postfix
 
@@ -4032,10 +4022,10 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
                        mykey=myslash[0]+"/"+mysplit[0]
                else:
                        mykey=mycpv
-               if mydb:
+               if mydb and virts and mykey in virts:
                        writemsg("mydb.__class__: %s\n" % (mydb.__class__), 1)
                        if type(mydb)==types.InstanceType:
-                               if (not mydb.cp_list(mykey,use_cache=use_cache)) and virts and virts.has_key(mykey):
+                               if not mydb.cp_list(mykey, use_cache=use_cache):
                                        writemsg("virts[%s]: %s\n" % (str(mykey),virts[mykey]), 1)
                                        mykey_orig = mykey[:]
                                        for vkey in virts[mykey]:
@@ -4387,6 +4377,19 @@ class fakedbapi(dbapi):
                if settings is None:
                        settings = globals()["settings"]
                self.settings = settings
+               self._match_cache = {}
+
+       def _clear_cache(self):
+               if self._match_cache:
+                       self._match_cache = {}
+
+       def match(self, origdep, use_cache=1):
+               result = self._match_cache.get(origdep, None)
+               if result is not None:
+                       return result[:]
+               result = dbapi.match(self, origdep, use_cache=use_cache)
+               self._match_cache[origdep] = result
+               return result[:]
 
        def cpv_exists(self,mycpv):
                return self.cpvdict.has_key(mycpv)
@@ -4408,6 +4411,7 @@ class fakedbapi(dbapi):
 
        def cpv_inject(self, mycpv, metadata=None):
                """Adds a cpv from the list of available packages."""
+               self._clear_cache()
                mycp=cpv_getkey(mycpv)
                self.cpvdict[mycpv] = metadata
                myslot = None
@@ -4439,6 +4443,7 @@ class fakedbapi(dbapi):
 
        def cpv_remove(self,mycpv):
                """Removes a cpv from the list of available packages."""
+               self._clear_cache()
                mycp=cpv_getkey(mycpv)
                if self.cpvdict.has_key(mycpv):
                        del     self.cpvdict[mycpv]
@@ -4458,6 +4463,7 @@ class fakedbapi(dbapi):
                return [metadata.get(x, "") for x in wants]
 
        def aux_update(self, cpv, values):
+               self._clear_cache()
                self.cpvdict[cpv].update(values)
 
 class bindbapi(fakedbapi):
@@ -4468,6 +4474,7 @@ class bindbapi(fakedbapi):
                if settings is None:
                        settings = globals()["settings"]
                self.settings = settings
+               self._match_cache = {}
 
        def match(self, *pargs, **kwargs):
                if self.bintree and not self.bintree.populated:
index 6459137e028ead0c83e186c9cf6f6e882cf1f843..72fa3f0c355228f36596fe741ab900480538e346 100644 (file)
@@ -257,6 +257,8 @@ def get_operator(mydep):
 
        return operator
 
+_dep_getcpv_cache = {}
+
 def dep_getcpv(mydep):
        """
        Return the category-package-version with any operators/slot specifications stripped off
@@ -270,6 +272,11 @@ def dep_getcpv(mydep):
        @rtype: String
        @return: The depstring with the operator removed
        """
+       global _dep_getcpv_cache
+       retval = _dep_getcpv_cache.get(mydep, None)
+       if retval is not None:
+               return retval
+       mydep_orig = mydep
        if mydep and mydep[0] == "*":
                mydep = mydep[1:]
        if mydep and mydep[-1] == "*":
@@ -282,7 +289,8 @@ def dep_getcpv(mydep):
                mydep = mydep[1:]
        colon = mydep.rfind(":")
        if colon != -1:
-               return mydep[:colon]
+               mydep = mydep[:colon]
+       _dep_getcpv_cache[mydep_orig] = mydep
        return mydep
 
 def dep_getslot(mydep):
@@ -478,6 +486,8 @@ def best_match_to_list(mypkg, mylist):
                        bestm  = x
        return bestm
 
+_match_from_list_cache = {}
+
 def match_from_list(mydep, candidate_list):
        """
        Searches list for entries that matches the package.
@@ -490,6 +500,11 @@ def match_from_list(mydep, candidate_list):
        @return: A list of package atoms that match the given package atom
        """
 
+       global _match_from_list_cache
+       mylist = _match_from_list_cache.get((mydep, tuple(candidate_list)), None)
+       if mylist is not None:
+               return mylist[:]
+
        from portage_util import writemsg
        if mydep[0] == "!":
                mydep = mydep[1:]
@@ -585,4 +600,5 @@ def match_from_list(mydep, candidate_list):
        else:
                raise KeyError("Unknown operator: %s" % mydep)
 
+       _match_from_list_cache[(mydep, tuple(candidate_list))] = mylist
        return mylist