From: Fabian Groffen Date: Tue, 9 Oct 2007 18:01:51 +0000 (-0000) Subject: Merged from trunk 8007:8016 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=95f34db4bb1c8368af403fd42d2d031bb0e3cfd9;p=portage.git Merged from trunk 8007:8016 | 8008 | Optimize away a match_from_list() call inside | | zmedico | portdbapi.xmatch("match-all") when given atom has no | | | operator or version. Also, make cp_list() use the | | | xmatch("match-all") cache when possible. | | 8009 | Remove a redundant os.access() call in portdbapi.aux_get() | | zmedico | and handle the potential OSError instead. | | 8010 | Properly exclude old-style virtuals from the | | zmedico | portdbapi.cp_list() cache. | | 8011 | Make portdbapi.aux_get() cache more often. | | zmedico | | | 8012 | Bug #149816 - Implement visibility filtering support for | | zmedico | binary packages. This works by creating a virtual filtered | | | repository that can be composed of multiple package types, | | | including ebuilds, binary packages, and installed packages. | | 8013 | In depgraph.xcreate(), installed packages are excluded from | | zmedico | the filtered tree. Clear out the history of atoms so that | | | they can be processed again to satisfy dependencies with | | | installed packages if necessary. | | 8014 | When an ebuild has conditionals in it's LICENSE, calculate | | zmedico | USE before the visibility check. | | 8015 | Fix InvalidDependString namespace and initialize USE before | | zmedico | visibility check. | | 8016 | Replace hardcoded path with new constant | | genone | | svn path=/main/branches/prefix/; revision=8018 --- diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 062f87c5b..adbea9741 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -31,7 +31,7 @@ except ImportError: import portage del os.environ["PORTAGE_LEGACY_GLOBALS"] from portage import digraph, portdbapi -from portage.const import NEWS_LIB_PATH, CACHE_PATH, PRIVATE_PATH, USER_CONFIG_PATH +from portage.const import NEWS_LIB_PATH, CACHE_PATH, PRIVATE_PATH, USER_CONFIG_PATH, GLOBAL_CONFIG_PATH import emerge.help import portage.xpak, commands, errno, re, socket, time, types @@ -780,9 +780,13 @@ class FakeVartree(portage.vartree): user doesn't necessarily need write access to the vardb in cases where global updates are necessary (updates are performed when necessary if there is not a matching ebuild in the tree).""" - def __init__(self, real_vartree, portdb): + def __init__(self, real_vartree, portdb, db_keys): self.root = real_vartree.root self.settings = real_vartree.settings + mykeys = db_keys[:] + for required_key in ("COUNTER", "SLOT"): + if required_key not in mykeys: + mykeys.append(required_key) self.dbapi = portage.fakedbapi(settings=real_vartree.settings) vdb_path = os.path.join(self.root, portage.VDB_PATH) try: @@ -794,8 +798,6 @@ class FakeVartree(portage.vartree): try: if os.access(vdb_path, os.W_OK): vdb_lock = portage.locks.lockdir(vdb_path) - mykeys = ["SLOT", "COUNTER", "PROVIDE", "USE", "IUSE", - "RESTRICT", "DEPEND", "RDEPEND", "PDEPEND", "repository"] real_dbapi = real_vartree.dbapi slot_counters = {} for cpv in real_dbapi.cpv_all(): @@ -871,6 +873,61 @@ def perform_global_updates(mycpv, mydb, mycommands): if updates: mydb.aux_update(mycpv, updates) +def cpv_sort_descending(cpv_list): + """Sort in place, returns None.""" + if len(cpv_list) <= 1: + return + first_split = portage.catpkgsplit(cpv_list[0]) + cat = first_split[0] + cpv_list[0] = first_split[1:] + for i in xrange(1, len(cpv_list)): + cpv_list[i] = portage.catpkgsplit(cpv_list[i])[1:] + cpv_list.sort(portage.pkgcmp, reverse=True) + for i, (pn, ver, rev) in enumerate(cpv_list): + if rev == "r0": + cpv = cat + "/" + pn + "-" + ver + else: + cpv = cat + "/" + pn + "-" + ver + "-" + rev + cpv_list[i] = cpv + +def visible(pkgsettings, cpv, metadata, built=False, installed=False): + """ + Check if a package is visible. This can raise an InvalidDependString + exception if LICENSE is invalid. + TODO: optionally generate a list of masking reasons + @rtype: Boolean + @returns: True if the package is visible, False otherwise. + """ + if built and not installed and \ + metadata["CHOST"] != pkgsettings["CHOST"]: + return False + if not portage.eapi_is_supported(metadata["EAPI"]): + return False + if pkgsettings.getMissingKeywords(cpv, metadata): + return False + if pkgsettings.getMaskAtom(cpv, metadata): + return False + if pkgsettings.getProfileMaskAtom(cpv, metadata): + return False + if pkgsettings.getMissingLicenses(cpv, metadata): + return False + return True + +def iter_atoms(deps): + """Take a dependency structure as returned by paren_reduce or use_reduce + and iterate over all the atoms.""" + i = iter(deps) + for x in i: + if isinstance(x, basestring): + if x == '||' or x.endswith('?'): + for x in iter_atoms(i.next()): + yield x + else: + yield x + else: + for x in iter_atoms(x): + yield x + class BlockerCache(DictMixin): """This caches blockers of installed packages so that dep_check does not have to be done for every single installed package on every invocation of @@ -1020,6 +1077,13 @@ class depgraph(object): "binary":"bintree", "installed":"vartree"} + _mydbapi_keys = [ + "CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS", + "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", + "repository", "RESTRICT", "SLOT", "USE"] + + _dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] + def __init__(self, settings, trees, myopts, myparams, spinner): self.settings = settings self.target_root = settings["ROOT"] @@ -1037,17 +1101,19 @@ class depgraph(object): # Maps nodes to the reasons they were selected for reinstallation. self._reinstall_nodes = {} self.mydbapi = {} - self._mydbapi_keys = ["SLOT", "DEPEND", "RDEPEND", "PDEPEND", - "USE", "IUSE", "PROVIDE", "RESTRICT", "repository"] self.trees = {} self.roots = {} + # Contains a filtered view of preferred packages that are selected + # from available repositories. + self._filtered_trees = {} for myroot in trees: self.trees[myroot] = {} for tree in ("porttree", "bintree"): self.trees[myroot][tree] = trees[myroot][tree] self.trees[myroot]["vartree"] = \ FakeVartree(trees[myroot]["vartree"], - trees[myroot]["porttree"].dbapi) + trees[myroot]["porttree"].dbapi, + self._mydbapi_keys) self.pkgsettings[myroot] = portage.config( clone=self.trees[myroot]["vartree"].settings) self.pkg_node_map[myroot] = {} @@ -1068,6 +1134,14 @@ class depgraph(object): metadata=dict(izip(self._mydbapi_keys, vardb.aux_get(pkg, self._mydbapi_keys)))) del vardb, fakedb + self._filtered_trees[myroot] = {} + self._filtered_trees[myroot]["vartree"] = self.trees[myroot]["vartree"] + def filtered_tree(): + pass + filtered_tree.dbapi = portage.fakedbapi( + settings=self.pkgsettings[myroot], exclusive_slots=False) + self._filtered_trees[myroot]["porttree"] = filtered_tree + self._filtered_trees[myroot]["atoms"] = set() if "--usepkg" in self.myopts: self.trees[myroot]["bintree"].populate( "--getbinpkg" in self.myopts, @@ -1649,6 +1723,158 @@ class depgraph(object): # We're true here unless we are missing binaries. return (not missing,myfavorites) + def _populate_filtered_repo(self, myroot, depstring, + myparent=None, myuse=None, exclude_installed=False): + """Extract all of the atoms from the depstring, select preferred + packages from appropriate repositories, and use them to populate + the filtered repository.""" + + filtered_db = self._filtered_trees[myroot]["porttree"].dbapi + pkgsettings = self.pkgsettings[myroot] + if myparent: + p_type, p_root, p_key, p_status = myparent + + from portage.dep import paren_reduce, use_reduce + try: + if myparent and p_type == "installed": + portage.dep._dep_check_strict = False + try: + atoms = paren_reduce(depstring) + atoms = use_reduce(atoms, uselist=myuse) + atoms = list(iter_atoms(atoms)) + for x in atoms: + if portage.dep._dep_check_strict and \ + not portage.isvalidatom(x, allow_blockers=True): + raise portage.exception.InvalidDependString( + "Invalid atom: %s" % x) + except portage.exception.InvalidDependString, e: + if myparent: + show_invalid_depstring_notice( + myparent, depstring, str(e)) + else: + sys.stderr.write("\n%s\n%s\n" % (depstring, str(e))) + return 0 + finally: + portage.dep._dep_check_strict = True + + filtered_atoms = self._filtered_trees[myroot]["atoms"] + dbs = self._filtered_trees[myroot].get("dbs") + if dbs is None: + dbs = [] + portdb = self.trees[myroot]["porttree"].dbapi + bindb = self.trees[myroot]["bintree"].dbapi + vardb = self.trees[myroot]["vartree"].dbapi + # (db, pkg_type, built, installed, db_keys) + if "--usepkgonly" not in self.myopts: + db_keys = list(portdb._aux_cache_keys) + dbs.append((portdb, "ebuild", False, False, db_keys)) + if "--usepkg" in self.myopts: + db_keys = list(bindb._aux_cache_keys) + dbs.append((bindb, "binary", True, False, db_keys)) + if "--usepkgonly" in self.myopts: + db_keys = self._mydbapi_keys + dbs.append((vardb, "installed", True, True, db_keys)) + self._filtered_trees[myroot]["dbs"] = dbs + old_virts = pkgsettings.getvirtuals() + while atoms: + x = atoms.pop() + if x.startswith("!"): + continue + if x in filtered_atoms: + continue + filtered_atoms.add(x) + cp = portage.dep_getkey(x) + cat = portage.catsplit(cp)[0] + slot = portage.dep.dep_getslot(x) + is_virt = cp.startswith("virtual/") + atom_populated = False + for db, pkg_type, built, installed, db_keys in dbs: + if installed and exclude_installed: + continue + cpv_list = db.cp_list(cp) + if not cpv_list: + if is_virt: + # old-style virtual + # Create a transformed atom for each choice + # and add it to the stack for processing. + for choice in old_virts.get(cp, []): + atoms.append(x.replace(cp, choice)) + # Maybe a new-style virtual exists in another db, so + # we have to try all of them to prevent the old-style + # virtuals from overriding available new-styles. + continue + cpv_sort_descending(cpv_list) + for cpv in cpv_list: + if filtered_db.cpv_exists(cpv): + continue + if not portage.match_from_list(x, [cpv]): + continue + if is_virt: + mykeys = db_keys[:] + mykeys.extend(self._dep_keys) + else: + mykeys = db_keys + try: + metadata = dict(izip(mykeys, + db.aux_get(cpv, mykeys))) + except KeyError: + # masked by corruption + continue + if slot is not None: + if slot != metadata["SLOT"]: + continue + if not built and \ + (is_virt or "?" in metadata["LICENSE"]): + pkgsettings.setcpv(cpv, mydb=metadata) + metadata["USE"] = pkgsettings["USE"] + else: + metadata["USE"] = "" + + try: + if not visible(pkgsettings, cpv, metadata, + built=built, installed=installed): + continue + except portage.exception.InvalidDependString: + # masked by corruption + continue + + filtered_db.cpv_inject(cpv, metadata=metadata) + if not is_virt: + # break here since we only want the best version + # for now (eventually will be configurable). + atom_populated = True + break + # For new-style virtuals, we explore all available + # versions and recurse on their deps. This is a + # preparation for the lookahead that happens when + # new-style virtuals are expanded by dep_check(). + virtual_deps = " ".join(metadata[k] \ + for k in self._dep_keys) + try: + if installed: + portage.dep._dep_check_strict = False + try: + deps = paren_reduce(virtual_deps) + deps = use_reduce(deps, + uselist=metadata["USE"].split()) + for y in iter_atoms(deps): + if portage.dep._dep_check_strict and \ + not portage.isvalidatom(y, + allow_blockers=True): + raise portage.exception.InvalidDependString( + "Invalid atom: %s" % y) + atoms.append(y) + except portage.exception.InvalidDependString, e: + show_invalid_depstring_notice( + (pkg_type, myroot, cpv, "nomerge"), + virtual_deps, str(e)) + return 0 + finally: + portage.dep._dep_check_strict = True + if atom_populated: + break + return 1 + def select_dep(self, myroot, depstring, myparent=None, arg=None, myuse=None, raise_on_missing=False, priority=DepPriority(), rev_deps=False, parent_arg=None): @@ -1662,6 +1888,10 @@ class depgraph(object): return 1 on success, 0 for failure """ + if not depstring: + return 1 # nothing to do + + filtered_db = self._filtered_trees[myroot]["porttree"].dbapi portdb = self.trees[myroot]["porttree"].dbapi bindb = self.trees[myroot]["bintree"].dbapi vardb = self.trees[myroot]["vartree"].dbapi @@ -1677,6 +1907,10 @@ class depgraph(object): print "Reverse:", rev_deps print "Priority:", priority + if not self._populate_filtered_repo( + myroot, depstring, myparent=myparent, myuse=myuse): + return 0 + #processing dependencies """ Call portage.dep_check to evaluate the use? conditionals and make sure all dependencies are satisfiable. """ @@ -1688,12 +1922,11 @@ class depgraph(object): mymerge = [] else: try: - if myparent and p_status == "nomerge": + if myparent and p_type == "installed": portage.dep._dep_check_strict = False mycheck = portage.dep_check(depstring, None, pkgsettings, myuse=myuse, - use_binaries=("--usepkgonly" in self.myopts), - myroot=myroot, trees=self.trees) + myroot=myroot, trees=self._filtered_trees) finally: portage.dep._dep_check_strict = True @@ -1758,18 +1991,40 @@ class depgraph(object): else: # List of acceptable packages, ordered by type preference. matched_packages = [] - myeb_matches = portdb.xmatch("match-visible", x) + slot = portage.dep.dep_getslot(x) + cpv_list = portdb.xmatch("match-all", x) + cpv_sort_descending(cpv_list) myeb = None myeb_pkg = None metadata = None existing_node = None - if myeb_matches: - myeb = portage.best(myeb_matches) + db_keys = list(portdb._aux_cache_keys) + for cpv in cpv_list: + try: + metadata = dict(izip(db_keys, + portdb.aux_get(cpv, db_keys))) + except KeyError: + # masked by corruption + continue + if "?" in metadata["LICENSE"]: + pkgsettings.setcpv(cpv, mydb=metadata) + metadata["USE"] = pkgsettings["USE"] + else: + metadata["USE"] = "" + try: + if not visible(pkgsettings, cpv, metadata, + built=False, installed=False): + continue + except portage.exception.InvalidDependString: + # masked by corruption + continue + + myeb = cpv # For best performance, try to reuse an exising node # and it's cached metadata. The portdbapi caches SLOT # metadata in memory so it's really only pulled once. - slot_atom = "%s:%s" % (portage.dep_getkey(myeb), - portdb.aux_get(myeb, ["SLOT"])[0]) + slot_atom = "%s:%s" % (portage.cpv_getkey(myeb), + metadata["SLOT"]) existing_node = self._slot_node_map[myroot].get(slot_atom) if existing_node: e_type, myroot, e_cpv, e_status = existing_node @@ -1781,20 +2036,20 @@ class depgraph(object): ([e_type, myroot, e_cpv], metadata)) else: existing_node = None + break if not existing_node and \ "--usepkg" in self.myopts: # The next line assumes the binarytree has been populated. # XXX: Need to work out how we use the binary tree with roots. usepkgonly = "--usepkgonly" in self.myopts - chost = pkgsettings["CHOST"] eprefix = pkgsettings["EPREFIX"] myeb_pkg_matches = [] - bindb_keys = ["CHOST","EAPI"] - for pkg in bindb.match(x): - metadata = dict(izip(bindb_keys, - bindb.aux_get(pkg, bindb_keys))) - if chost != metadata["CHOST"]: + cpv_list = bindb.match(x) + cpv_sort_descending(cpv_list) + db_keys = list(bindb._aux_cache_keys) + for pkg in cpv_list: + if not filtered_db.cpv_exists(pkg): continue pkg_eprefix = bindb.aux_get(pkg, ["EPREFIX"])[0] @@ -1806,13 +2061,17 @@ class depgraph(object): if not portage.eapi_is_supported(metadata["EAPI"]): continue - # Remove any binary package entries that are - # masked in the portage tree (#55871). - if not usepkgonly and \ - not (pkg in myeb_matches or \ - not portdb.cpv_exists(pkg)): - continue + + metadata = dict(izip(db_keys, + bindb.aux_get(pkg, db_keys))) + try: + if not visible(pkgsettings, pkg, metadata, + built=True, installed=False): + continue + except portage.exception.InvalidDependString: + # masked by corruption myeb_pkg_matches.append(pkg) + break if myeb_pkg_matches: myeb_pkg = portage.best(myeb_pkg_matches) # For best performance, try to reuse an exising node @@ -1881,20 +2140,22 @@ class depgraph(object): """Fall back to the installed package database. This is a last resort because the metadata tends to diverge from that of the ebuild in the tree.""" - myeb_inst_matches = vardb.match(x) - if "--usepkgonly" not in self.myopts: - """ TODO: Improve masking check for installed and - binary packages. bug #149816""" - myeb_inst_matches = [pkg for pkg in myeb_inst_matches \ - if not portdb.cpv_exists(pkg)] - myeb_inst = None - if myeb_inst_matches: - myeb_inst = portage.best(myeb_inst_matches) - if myeb_inst: + cpv_list = vardb.match(x) + cpv_sort_descending(cpv_list) + for cpv in cpv_list: metadata = dict(izip(self._mydbapi_keys, - vardb.aux_get(myeb_inst, self._mydbapi_keys))) + vardb.aux_get(cpv, self._mydbapi_keys))) + # TODO: Handle masking for installed packages. + #try: + # if not visible(pkgsettings, cpv, metadata, + # built=True, installed=True): + # continue + #except: InvalidDependString: + # # masked by corruption + # continue matched_packages.append( - (["installed", myroot, myeb_inst], metadata)) + (["installed", myroot, cpv], metadata)) + break if not matched_packages: if raise_on_missing: @@ -2565,12 +2826,7 @@ class depgraph(object): def xcreate(self,mode="system"): vardb = self.trees[self.target_root]["vartree"].dbapi - portdb = self.trees[self.target_root]["porttree"].dbapi - bindb = self.trees[self.target_root]["bintree"].dbapi - def visible(mylist): - matches = portdb.gvisible(portdb.visible(mylist)) - return [x for x in mylist \ - if x in matches or not portdb.cpv_exists(x)] + filtered_db = self._filtered_trees[self.target_root]["porttree"].dbapi world_problems = False root_config = self.roots[self.target_root] @@ -2586,17 +2842,10 @@ class depgraph(object): continue elif not vardb.match(x): world_problems = True - available = False - if "--usepkgonly" not in self.myopts and \ - portdb.match(x): - available = True - elif "--usepkg" in self.myopts: - mymatches = bindb.match(x) - if "--usepkgonly" not in self.myopts: - mymatches = visible(mymatches) - if mymatches: - available = True - if not available: + if not self._populate_filtered_repo(self.target_root, x, + exclude_installed=True): + return 0 + if not filtered_db.match(x): continue mylist.append(x) @@ -2615,46 +2864,33 @@ class depgraph(object): for cpv in vardb.match(mykey): myslots.add(vardb.aux_get(cpv, ["SLOT"])[0]) if myslots: - best_pkgs = [] - if "--usepkg" in self.myopts: - mymatches = bindb.match(atom) - if "--usepkgonly" not in self.myopts: - mymatches = visible(mymatches) - best_pkg = portage.best(mymatches) - if best_pkg: - best_slot = bindb.aux_get(best_pkg, ["SLOT"])[0] - best_pkgs.append(("binary", best_pkg, best_slot)) - if "--usepkgonly" not in self.myopts: - best_pkg = portage.best(portdb.match(atom)) - if best_pkg: - best_slot = portdb.aux_get(best_pkg, ["SLOT"])[0] - best_pkgs.append(("ebuild", best_pkg, best_slot)) - if best_pkgs: - best_pkg = portage.best([x[1] for x in best_pkgs]) - best_pkgs = [x for x in best_pkgs if x[1] == best_pkg] - best_slot = best_pkgs[0][2] + if not self._populate_filtered_repo(self.target_root, atom, + exclude_installed=True): + return 0 + mymatches = filtered_db.match(atom) + best_pkg = portage.best(mymatches) + if best_pkg: + best_slot = filtered_db.aux_get(best_pkg, ["SLOT"])[0] myslots.add(best_slot) if len(myslots) > 1: for myslot in myslots: myslot_atom = "%s:%s" % (mykey, myslot) - available = False - if "--usepkgonly" not in self.myopts and \ - self.trees[self.target_root][ - "porttree"].dbapi.match(myslot_atom): - available = True - elif "--usepkg" in self.myopts: - mymatches = bindb.match(myslot_atom) - if "--usepkgonly" not in self.myopts: - mymatches = visible(mymatches) - if mymatches: - available = True - if available: + if not self._populate_filtered_repo( + self.target_root, myslot_atom, + exclude_installed=True): + return 0 + if filtered_db.match(myslot_atom): newlist.append(myslot_atom) mylist = newlist for myatom in mylist: self._set_atoms.add(myatom) + # Since populate_filtered_repo() was called with the exclude_installed + # flag, these atoms will need to be processed again in case installed + # packages are required to satisfy dependencies. + self._filtered_trees[self.target_root]["atoms"].clear() + missing_atoms = [] for mydep in mylist: try: @@ -5586,7 +5822,9 @@ def action_depclean(settings, trees, ldpath_mtimes, dep_check_trees = {} dep_check_trees[myroot] = {} dep_check_trees[myroot]["vartree"] = \ - FakeVartree(trees[myroot]["vartree"], trees[myroot]["porttree"].dbapi) + FakeVartree(trees[myroot]["vartree"], + trees[myroot]["porttree"].dbapi, + depgraph._mydbapi_keys) vardb = dep_check_trees[myroot]["vartree"].dbapi # Constrain dependency selection to the installed packages. dep_check_trees[myroot]["porttree"] = dep_check_trees[myroot]["vartree"] @@ -6466,7 +6704,7 @@ def emerge_main(): print colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n") break - setconfigpaths = ["/usr/share/portage/config/sets.conf"] + setconfigpaths = [os.path.join(GLOBAL_CONFIG_PATH, "sets.conf")] setconfigpaths.append(os.path.join(settings["PORTDIR"], "sets.conf")) setconfigpaths += [os.path.join(x, "sets.conf") for x in settings["PORDIR_OVERLAY"].split()] setconfigpaths.append(os.path.join(os.sep, settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "sets.conf")) diff --git a/pym/portage/const.py b/pym/portage/const.py index 847d6f650..6b07b909f 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -24,6 +24,7 @@ DEPCACHE_PATH = CACHE_PATH+"/dep" USER_CONFIG_PATH = EPREFIX+"/etc/portage" MODULES_FILE_PATH = USER_CONFIG_PATH+"/modules" CUSTOM_PROFILE_PATH = USER_CONFIG_PATH+"/profile" +GLOBAL_CONFIG_PATH = EPREFIX+"/usr/share/portage/config" PORTAGE_BASE_PATH = PORTAGE_BASE PORTAGE_BIN_PATH = PORTAGE_BASE_PATH+"/bin" diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index d2d645e86..1ed7b0359 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -18,18 +18,15 @@ import os, errno, stat from itertools import izip class bindbapi(fakedbapi): - def __init__(self, mybintree=None, settings=None): + def __init__(self, mybintree=None, **kwargs): + fakedbapi.__init__(self, **kwargs) self.bintree = mybintree self.move_ent = mybintree.move_ent self.cpvdict={} self.cpdict={} - if settings is None: - from portage import settings - self.settings = settings - self._match_cache = {} # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( - ["CHOST", "EAPI", "KEYWORDS", "LICENSE", "SLOT", "EPREFIX"]) + ["CHOST", "EAPI", "KEYWORDS", "LICENSE", "SLOT", "USE", "EPREFIX"]) self._aux_cache = {} def match(self, *pargs, **kwargs): diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index ee8c4776d..91b5b488e 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -220,6 +220,8 @@ class portdbapi(dbapi): 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error' cache_me = False + if not mytree: + cache_me = True if not mytree and not set(mylist).difference(self._aux_cache_keys): aux_cache = self._aux_cache.get(mycpv) if aux_cache is not None: @@ -272,14 +274,14 @@ class portdbapi(dbapi): noiselevel=-1) - if os.access(myebuild, os.R_OK): + try: emtime = os.stat(myebuild)[stat.ST_MTIME] - else: + except OSError: writemsg("!!! aux_get(): ebuild for '%(cpv)s' does not exist at:\n" % {"cpv":mycpv}, noiselevel=-1) writemsg("!!! %s\n" % myebuild, noiselevel=-1) - raise KeyError + raise KeyError(mycpv) try: mydata = self.auxdb[mylocation][mycpv] @@ -499,6 +501,11 @@ class portdbapi(dbapi): return d.keys() def cp_list(self, mycp, use_cache=1, mytree=None): + if self.frozen and mytree is None: + mylist = self.xcache["match-all"].get(mycp) + # cp_list() doesn't expand old-style virtuals + if mylist and mylist[0].startswith(mycp): + return mylist[:] mysplit = mycp.split("/") invalid_category = mysplit[0] not in self._categories d={} @@ -519,8 +526,13 @@ class portdbapi(dbapi): if invalid_category and d: writemsg(("\n!!! '%s' has a category that is not listed in " + \ "/etc/portage/categories\n") % mycp, noiselevel=-1) - return [] - return d.keys() + mylist = [] + else: + mylist = d.keys() + if self.frozen and mytree is None: + if not (not mylist and mycp.startswith("virtual/")): + self.xcache["match-all"][mycp] = mylist[:] + return mylist def freeze(self): for x in ["list-visible", "bestmatch-visible", "match-visible", "match-all"]: @@ -573,8 +585,10 @@ class portdbapi(dbapi): self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey)) elif level == "match-all": #match *all* visible *and* masked packages - - myval = match_from_list(mydep, self.cp_list(mykey)) + if mydep == mykey: + myval = self.cp_list(mykey) + else: + myval = match_from_list(mydep, self.cp_list(mykey)) else: print "ERROR: xmatch doesn't handle", level, "query!" raise KeyError diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index dc1c23f14..7aae8fe3c 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -187,8 +187,10 @@ class vardbapi(dbapi): from portage import db vartree = db[root]["vartree"] self.vartree = vartree - self._aux_cache_keys = set(["SLOT", "COUNTER", "PROVIDE", "USE", - "IUSE", "DEPEND", "RDEPEND", "PDEPEND", "NEEDED", "repository"]) + self._aux_cache_keys = set( + ["CHOST", "COUNTER", "DEPEND", "EAPI", "IUSE", "KEYWORDS", + "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "NEEDED", + "repository", "RESTRICT" , "SLOT", "USE"]) self._aux_cache = None self._aux_cache_version = "1" self._aux_cache_filename = os.path.join(self.root, diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py index 30f51ceaf..65ac0a8d0 100644 --- a/pym/portage/dbapi/virtual.py +++ b/pym/portage/dbapi/virtual.py @@ -10,7 +10,14 @@ class fakedbapi(dbapi): """A fake dbapi that allows consumers to inject/remove packages to/from it portage.settings is required to maintain the dbAPI. """ - def __init__(self, settings=None): + def __init__(self, settings=None, exclusive_slots=True): + """ + @param exclusive_slots: When True, injecting a package with SLOT + metadata causes an existing package in the same slot to be + automatically removed (default is True). + @type exclusive_slots: Boolean + """ + self._exclusive_slots = exclusive_slots self.cpvdict = {} self.cpdict = {} if settings is None: @@ -46,12 +53,19 @@ class fakedbapi(dbapi): return self.cpvdict.keys() def cpv_inject(self, mycpv, metadata=None): - """Adds a cpv from the list of available packages.""" + """Adds a cpv to the list of available packages. See the + exclusive_slots constructor parameter for behavior with + respect to SLOT metadata. + @param mycpv: cpv for the package to inject + @type mycpv: str + @param metadata: dictionary of raw metadata for aux_get() calls + @param metadata: dict + """ self._clear_cache() mycp = cpv_getkey(mycpv) self.cpvdict[mycpv] = metadata myslot = None - if metadata: + if self._exclusive_slots and metadata: myslot = metadata.get("SLOT", None) if myslot and mycp in self.cpdict: # If necessary, remove another package in the same SLOT.