Optimize repoman visibility checks to access as little

metadata as possible. (improves performance especially
in cases where metadata needs to be generated). This
works by starting at the lowest version since that's
most likely to have keywords and it returns as soon as
the first visible package is found.

svn path=/main/trunk/; revision=8309

diff --git a/bin/repoman b/bin/repoman
index 88493122007933b0392a77d9c7542982c8ade9c4..a57c30052808034dcd3e3823b478884aceac6329 100755 (executable)
--- a/bin/repoman
+++ b/bin/repoman
@@ -1455,10 +1455,10 @@ for x in scanlist:
                                                        continue
                                                #we are testing deps for a masked package; give it some lee-way
                                                suffix="masked"
-                                               matchmode="match-all"
+                                               matchmode = "minimum-all"
                                        else:
                                                suffix=""
-                                               matchmode="match-visible"
+                                               matchmode = "minimum-visible"
        
                                        if prof[1] == "dev":
                                                suffix=suffix+"indev"

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index cc713c1d20fda59cbfbf3b9c4164d3e27f79d64d..06878c61c74df6267541ca19ee2c05291c162a64 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -4970,7 +4970,13 @@ def dep_wordreduce(mydeplist,mysettings,mydbapi,mode,use_cache=1):
                                deplist[mypos] = False
                        else:
                                if mode:
-                                       mydep=mydbapi.xmatch(mode,deplist[mypos])
+                                       x = mydbapi.xmatch(mode, deplist[mypos])
+                                       if mode.startswith("minimum-"):
+                                               mydep = []
+                                               if x:
+                                                       mydep.append(x)
+                                       else:
+                                               mydep = x
                                else:
                                        mydep=mydbapi.match(deplist[mypos],use_cache=use_cache)
                                if mydep!=None:

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 05fe2a767e44c37dfa46c467b24e3e1b5a51bf48..92129381894a9e8ac31deeea0cb3c796691c7f96 100644 (file)
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -14,7 +14,7 @@ from portage.exception import OperationNotPermitted, PortageException, \
 from portage.manifest import Manifest
 from portage.output import red
 from portage.util import ensure_dirs, writemsg, apply_recursive_permissions
-from portage.versions import pkgsplit, catpkgsplit, best
+from portage.versions import pkgcmp, pkgsplit, catpkgsplit, best
 
 import portage.gpg, portage.checksum
 
@@ -521,13 +521,27 @@ class portdbapi(dbapi):
                        mylist = []
                else:
                        mylist = d.keys()
+               # Always sort in ascending order here since it's handy
+               # and the result can be easily cached and reused.
+               if len(mylist) > 1:
+                       for i in xrange(len(mylist)):
+                               mylist[i] = catpkgsplit(mylist[i])[1:]
+                       mylist.sort(pkgcmp)
+                       cat = mysplit[0]
+                       for i, (pn, ver, rev) in enumerate(mylist):
+                               if rev == "r0":
+                                       cpv = cat + "/" + pn + "-" + ver
+                               else:
+                                       cpv = cat + "/" + pn + "-" + ver + "-" + rev
+                               mylist[i] = cpv
                if self.frozen and mytree is None:
                        if not (not mylist and mycp.startswith("virtual/")):
                                self.xcache["match-all"][mycp] = mylist[:]
                return mylist
 
        def freeze(self):
-               for x in ["list-visible", "bestmatch-visible", "match-visible", "match-all"]:
+               for x in "bestmatch-visible", "list-visible", "match-all", \
+                       "match-visible", "minimum-all", "minimum-visible":
                        self.xcache[x]={}
                self.frozen=1
 
@@ -550,6 +564,7 @@ class portdbapi(dbapi):
                        mydep = dep_expand(origdep, mydb=self, settings=self.mysettings)
                        mykey = dep_getkey(mydep)
 
+               myslot = dep_getslot(mydep)
                if level == "list-visible":
                        #a list of all visible packages, not called directly (just by xmatch())
                        #myval = self.visible(self.cp_list(mykey))
@@ -560,6 +575,56 @@ class portdbapi(dbapi):
                        #get all visible matches (from xmatch()), then choose the best one
 
                        myval = best(self.xmatch("match-visible", None, mydep=mydep, mykey=mykey))
+               elif level == "minimum-all":
+                       # Find the minimum matching version. This is optimized to
+                       # minimize the number of metadata accesses (improves performance
+                       # especially in cases where metadata needs to be generated).
+                       if mydep == mykey:
+                               mylist = self.cp_list(mykey)
+                       else:
+                               mylist = match_from_list(mydep, self.cp_list(mykey))
+                       myval = ""
+                       if mylist:
+                               if myslot is None:
+                                       myval = mylist[0]
+                               else:
+                                       for cpv in mylist:
+                                               try:
+                                                       if self.aux_get(cpv, ["SLOT"])[0] == myslot:
+                                                               myval = cpv
+                                                               break
+                                               except KeyError:
+                                                       pass # ebuild masked by corruption
+               elif level == "minimum-visible":
+                       # Find the minimum matching visible version. This is optimized to
+                       # minimize the number of metadata accesses (improves performance
+                       # especially in cases where metadata needs to be generated).
+                       # This does not implement LICENSE filtering since it's only
+                       # intended for use by repoman.
+                       if mydep == mykey:
+                               mylist = self.cp_list(mykey)
+                       else:
+                               mylist = match_from_list(mydep, self.cp_list(mykey))
+                       myval = ""
+                       settings = self.mysettings
+                       for cpv in mylist:
+                               try:
+                                       metadata = dict(izip(self._aux_cache_keys,
+                                               self.aux_get(cpv, self._aux_cache_keys)))
+                               except KeyError:
+                                       # ebuild masked by corruption
+                                       continue
+                               if not eapi_is_supported(metadata["EAPI"]):
+                                       continue
+                               if myslot and myslot != metadata["SLOT"]:
+                                       continue
+                               if settings.getMissingKeywords(cpv, metadata):
+                                       continue
+                               if settings.getMaskAtom(cpv, metadata):
+                                       continue
+                               if settings.getProfileMaskAtom(cpv, metadata):
+                                       continue
+                               myval = cpv
                elif level == "bestmatch-list":
                        #dep match -- find best match but restrict search to sublist
                        #no point in calling xmatch again since we're not caching list deps
@@ -584,8 +649,7 @@ class portdbapi(dbapi):
                else:
                        print "ERROR: xmatch doesn't handle", level, "query!"
                        raise KeyError
-               myslot = dep_getslot(mydep)
-               if myslot is not None:
+               if myslot is not None and isinstance(myval, list):
                        slotmatches = []
                        for cpv in myval:
                                try: