* Optimize metadata caching in pordbapi and bindbapi so that cache is
authorZac Medico <zmedico@gentoo.org>
Tue, 8 Apr 2008 05:38:33 +0000 (05:38 -0000)
committerZac Medico <zmedico@gentoo.org>
Tue, 8 Apr 2008 05:38:33 +0000 (05:38 -0000)
  never pulled for the same package twice.
* Cache a Package instances constructed during package selection
  and reuse them during later package selections.
(trunk r9741)

svn path=/main/branches/2.1.2/; revision=9752

bin/emerge
pym/portage.py

index a4865d1abe3fe5789cfc019bf01b4fd1bf8ca983..1eeae340c4c84b2c8d25f4648cf65383e7948884 100755 (executable)
@@ -1485,7 +1485,7 @@ class CompositeDbapi(object):
                                except portage_exception.InvalidDependString:
                                        arg = None
                                        arg_cp = None
-                               else:
+                               if arg:
                                        arg_cp = portage.dep_getkey(arg)
                                if arg and arg_cp != pkg.cp:
                                        # If this argument matches via PROVIDE but there is a
@@ -1619,6 +1619,8 @@ class depgraph:
                self._world_problems = False
                self._select_package = self._select_pkg_highest_available
                self._highest_pkg_cache = {}
+               # All Package instances
+               self._pkg_cache = {}
 
        def _show_slot_collision_notice(self):
                """Show an informational message advising the user to mask one of the
@@ -2316,6 +2318,11 @@ class depgraph:
                                        cpv_list = db.xmatch("match-all", atom)
                                else:
                                        cpv_list = db.match(atom)
+                               if not cpv_list:
+                                       continue
+                               pkg_status = "merge"
+                               if installed or onlydeps:
+                                       pkg_status = "nomerge"
                                # descending order
                                cpv_list.reverse()
                                for cpv in cpv_list:
@@ -2324,33 +2331,47 @@ class depgraph:
                                                cpv in vardb.match(atom):
                                                break
                                        reinstall_for_flags = None
-                                       try:
-                                               metadata = dict(izip(db_keys,
-                                                       db.aux_get(cpv, db_keys)))
-                                       except KeyError:
-                                               continue
-                                       if not built:
-                                               if "?" in metadata["LICENSE"]:
+                                       cache_key = (pkg_type, root, cpv, pkg_status)
+                                       calculated_use = True
+                                       pkg = self._pkg_cache.get(cache_key)
+                                       if pkg is None:
+                                               calculated_use = False
+                                               try:
+                                                       metadata = dict(izip(self._mydbapi_keys,
+                                                               db.aux_get(cpv, self._mydbapi_keys)))
+                                               except KeyError:
+                                                       continue
+                                               if not built and ("?" in metadata["LICENSE"] or \
+                                                       "?" in metadata["PROVIDE"]):
+                                                       # This is avoided whenever possible because
+                                                       # it's expensive. It only needs to be done here
+                                                       # if it has an effect on visibility.
                                                        pkgsettings.setcpv(cpv, mydb=metadata)
                                                        metadata["USE"] = pkgsettings["PORTAGE_USE"]
-                                               else:
-                                                       metadata["USE"] = ""
+                                                       calculated_use = True
+                                               pkg = Package(built=built, cpv=cpv,
+                                                       installed=installed, metadata=metadata,
+                                                       onlydeps=onlydeps, root=root, type_name=pkg_type)
+                                               self._pkg_cache[pkg] = pkg
                                        myarg = None
                                        if root == self.target_root:
                                                try:
                                                        myarg = self._set_atoms.findAtomForPackage(
-                                                               cpv, metadata)
+                                                               pkg.cpv, pkg.metadata)
                                                except portage_exception.InvalidDependString:
                                                        if not installed:
                                                                # masked by corruption
                                                                continue
-                                       pkg = Package(built=built, cpv=cpv, installed=installed,
-                                               metadata=metadata, type_name=pkg_type)
                                        if not installed:
                                                if myarg:
                                                        found_available_arg = True
                                                if not visible(pkgsettings, pkg):
                                                        continue
+                                       if not built and not calculated_use:
+                                               # This is avoided whenever possible because
+                                               # it's expensive.
+                                               pkgsettings.setcpv(cpv, mydb=pkg.metadata)
+                                               pkg.metadata["USE"] = pkgsettings["PORTAGE_USE"]
                                        if pkg.cp == atom_cp:
                                                if highest_version is None:
                                                        highest_version = pkg
@@ -2362,9 +2383,7 @@ class depgraph:
                                        # will always end with a break statement below
                                        # this point.
                                        if find_existing_node:
-                                               slot_atom = "%s:%s" % (
-                                                       portage.cpv_getkey(cpv), metadata["SLOT"])
-                                               e_pkg = self._slot_pkg_map[root].get(slot_atom)
+                                               e_pkg = self._slot_pkg_map[root].get(pkg.slot_atom)
                                                if not e_pkg:
                                                        break
                                                cpv_slot = "%s:%s" % \
@@ -2388,9 +2407,9 @@ class depgraph:
                                                ("--newuse" in self.myopts or \
                                                "--reinstall" in self.myopts):
                                                iuses = set(filter_iuse_defaults(
-                                                       metadata["IUSE"].split()))
-                                               old_use = metadata["USE"].split()
-                                               mydb = metadata
+                                                       pkg.metadata["IUSE"].split()))
+                                               old_use = pkg.metadata["USE"].split()
+                                               mydb = pkg.metadata
                                                if myeb and not usepkgonly:
                                                        mydb = portdb
                                                if myeb:
@@ -2416,7 +2435,7 @@ class depgraph:
                                                ("--newuse" in self.myopts or \
                                                "--reinstall" in self.myopts) and \
                                                cpv in vardb.match(atom):
-                                               pkgsettings.setcpv(cpv, mydb=metadata)
+                                               pkgsettings.setcpv(cpv, mydb=pkg.metadata)
                                                forced_flags = set()
                                                forced_flags.update(pkgsettings.useforce)
                                                forced_flags.update(pkgsettings.usemask)
@@ -2425,7 +2444,7 @@ class depgraph:
                                                        vardb.aux_get(cpv, ["IUSE"])[0].split()))
                                                cur_use = pkgsettings["PORTAGE_USE"].split()
                                                cur_iuse = set(filter_iuse_defaults(
-                                                       metadata["IUSE"].split()))
+                                                       pkg.metadata["IUSE"].split()))
                                                reinstall_for_flags = \
                                                        self._reinstall_for_flags(
                                                        forced_flags, old_use, old_iuse,
@@ -2439,22 +2458,8 @@ class depgraph:
                                                        not must_reinstall and \
                                                        cpv in vardb.match(atom):
                                                        break
-                                       # Metadata accessed above is cached internally by
-                                       # each db in order to optimize visibility checks.
-                                       # Now that all possible checks visibility checks
-                                       # are complete, it's time to pull the rest of the
-                                       # metadata (including *DEPEND). This part is more
-                                       # expensive, so avoid it whenever possible.
-                                       metadata.update(izip(self._mydbapi_keys,
-                                               db.aux_get(cpv, self._mydbapi_keys)))
                                        if not built:
-                                               pkgsettings.setcpv(cpv, mydb=metadata)
-                                               metadata["USE"] = pkgsettings["PORTAGE_USE"]
                                                myeb = cpv
-                                       pkg = Package(type_name=pkg_type, root=root,
-                                               cpv=cpv, metadata=metadata,
-                                               built=built, installed=installed,
-                                               onlydeps=onlydeps)
                                        matched_packages.append(pkg)
                                        if reinstall_for_flags:
                                                self._reinstall_nodes[pkg] = \
index 0cc187da16d2f4b41c1f1ca4ea519c5345031f7d..4fee4e47a513fc953c996e47b4abe7deb5741076 100644 (file)
@@ -5945,6 +5945,17 @@ def getmaskingstatus(mycpv, settings=None, portdb=None):
                rValue.append(kmask+" keyword")
        return rValue
 
+
+auxdbkeys=[
+  'DEPEND',    'RDEPEND',   'SLOT',      'SRC_URI',
+       'RESTRICT',  'HOMEPAGE',  'LICENSE',   'DESCRIPTION',
+       'KEYWORDS',  'INHERITED', 'IUSE',      'CDEPEND',
+       'PDEPEND',   'PROVIDE', 'EAPI',
+       'UNUSED_01', 'UNUSED_02', 'UNUSED_03', 'UNUSED_04',
+       'UNUSED_05', 'UNUSED_06', 'UNUSED_07',
+       ]
+auxdbkeylen=len(auxdbkeys)
+
 class portagetree:
        def __init__(self, root="/", virtual=None, clone=None, settings=None):
 
@@ -6020,10 +6031,12 @@ class portagetree:
                return myslot
 
 
-class dbapi:
+class dbapi(object):
        _category_re = re.compile(r'^\w[-.+\w]*$')
        _pkg_dir_name_re = re.compile(r'^\w[-+\w]*$')
        _categories = None
+       _known_keys = frozenset(x for x in auxdbkeys
+               if not x.startswith("UNUSED_0"))
        def __init__(self):
                pass
 
@@ -6243,6 +6256,8 @@ class fakedbapi(dbapi):
                self.cpvdict[cpv].update(values)
 
 class bindbapi(fakedbapi):
+       _known_keys = frozenset(list(fakedbapi._known_keys) + \
+               ["CHOST", "repository", "USE"])
        def __init__(self, mybintree=None, settings=None):
                self.bintree = mybintree
                self.move_ent = mybintree.move_ent
@@ -6254,8 +6269,9 @@ class bindbapi(fakedbapi):
                self._match_cache = {}
                # Selectively cache metadata in order to optimize dep matching.
                self._aux_cache_keys = set(
-                                       ["CHOST", "EAPI", "IUSE", "KEYWORDS",
-                                       "LICENSE", "PROVIDE", "SLOT", "USE"])
+                       ["CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
+                       "LICENSE", "PDEPEND", "PROVIDE",
+                       "RDEPEND", "repository", "RESTRICT", "SLOT", "USE"])
                self._aux_cache = {}
 
        def match(self, *pargs, **kwargs):
@@ -6267,10 +6283,11 @@ class bindbapi(fakedbapi):
                if self.bintree and not self.bintree.populated:
                        self.bintree.populate()
                cache_me = False
-               if not set(wants).difference(self._aux_cache_keys):
+               if not self._known_keys.intersection(
+                       wants).difference(self._aux_cache_keys):
                        aux_cache = self._aux_cache.get(mycpv)
                        if aux_cache is not None:
-                               return [aux_cache[x] for x in wants]
+                               return [aux_cache.get(x, "") for x in wants]
                        cache_me = True
                mysplit = mycpv.split("/")
                mylist  = []
@@ -6970,16 +6987,6 @@ class vartree(object):
        def populate(self):
                self.populated=1
 
-auxdbkeys=[
-  'DEPEND',    'RDEPEND',   'SLOT',      'SRC_URI',
-       'RESTRICT',  'HOMEPAGE',  'LICENSE',   'DESCRIPTION',
-       'KEYWORDS',  'INHERITED', 'IUSE',      'CDEPEND',
-       'PDEPEND',   'PROVIDE', 'EAPI',
-       'UNUSED_01', 'UNUSED_02', 'UNUSED_03', 'UNUSED_04',
-       'UNUSED_05', 'UNUSED_06', 'UNUSED_07',
-       ]
-auxdbkeylen=len(auxdbkeys)
-
 def close_portdbapi_caches():
        for i in portdbapi.portdbapi_instances:
                i.close_caches()
@@ -7051,11 +7058,13 @@ class portdbapi(dbapi):
                self.porttrees = [self.porttree_root] + \
                        [os.path.realpath(t) for t in self.mysettings["PORTDIR_OVERLAY"].split()]
                self.treemap = {}
+               self._repository_map = {}
                for path in self.porttrees:
                        repo_name_path = os.path.join(path, portage_const.REPO_NAME_LOC)
                        try:
                                repo_name = open(repo_name_path, 'r').readline().strip()
                                self.treemap[repo_name] = path
+                               self._repository_map[path] = repo_name
                        except (OSError,IOError):
                                # warn about missing repo_name at some other time, since we
                                # don't want to see a warning every time the portage module is
@@ -7084,7 +7093,9 @@ class portdbapi(dbapi):
                                        self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid)
                # Selectively cache metadata in order to optimize dep matching.
                self._aux_cache_keys = set(
-                       ["EAPI", "IUSE", "KEYWORDS", "LICENSE", "PROVIDE", "SLOT"])
+                       ["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE",
+                       "PDEPEND", "PROVIDE", "RDEPEND", "repository",
+                       "RESTRICT", "SLOT"])
                self._aux_cache = {}
                self._broken_ebuilds = set()
 
@@ -7179,10 +7190,11 @@ class portdbapi(dbapi):
                cache_me = False
                if not mytree:
                        cache_me = True
-               if not mytree and not set(mylist).difference(self._aux_cache_keys):
+               if not mytree and not self._known_keys.intersection(
+                       mylist).difference(self._aux_cache_keys):
                        aux_cache = self._aux_cache.get(mycpv)
                        if aux_cache is not None:
-                               return [aux_cache[x] for x in mylist]
+                               return [aux_cache.get(x, "") for x in mylist]
                        cache_me = True
                global auxdbkeys,auxdbkeylen
                cat,pkg = mycpv.split("/", 1)
@@ -7304,6 +7316,10 @@ class portdbapi(dbapi):
                if not mydata.setdefault("EAPI", "0"):
                        mydata["EAPI"] = "0"
 
+               # do we have a origin repository name for the current package
+               mydata["repository"] = self._repository_map.get(
+                       os.path.sep.join(myebuild.split(os.path.sep)[:-3]), "")
+
                #finally, we look at our internal cache entry and return the requested data.
                returnme = []
                for x in mylist: