From e0a9a7d0f2b6884344ad77b53928ca174ea23981 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 12 Oct 2007 08:48:20 +0000 Subject: [PATCH] Merged from trunk 8032:8036 | 8033 | Unify the masking reasons output so that the same code path | | zmedico | is used for both ebuilds and binary packages. | | 8034 | Initialize the list of package repos inside the depgraph | | zmedico | constructor so it's always available. | | 8035 | Migrate depgraph.select_files() to use the filtered repo for | | zmedico | expansion of greedy atoms. Also, use the list of repos to | | | clean up the category expansion code. | | 8036 | Cache IUSE in bindbapi to optimize --newuse calculations. | | zmedico | Also, fix filtering code for reading/writing USE and IUSE | | | from/to the metadata index. | svn path=/main/branches/prefix/; revision=8074 --- pym/emerge/__init__.py | 266 +++++++++++++++++------------------ pym/portage/__init__.py | 86 ++++++----- pym/portage/dbapi/bintree.py | 8 +- pym/portage/getbinpkg.py | 4 + 4 files changed, 180 insertions(+), 184 deletions(-) diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index dcdedb706..c830dc838 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -1142,6 +1142,21 @@ class depgraph(object): settings=self.pkgsettings[myroot], exclusive_slots=False) self._filtered_trees[myroot]["porttree"] = filtered_tree self._filtered_trees[myroot]["atoms"] = set() + dbs = [] + portdb = self.trees[myroot]["porttree"].dbapi + bindb = self.trees[myroot]["bintree"].dbapi + vardb = self.trees[myroot]["vartree"].dbapi + # (db, pkg_type, built, installed, db_keys) + if "--usepkgonly" not in self.myopts: + db_keys = list(portdb._aux_cache_keys) + dbs.append((portdb, "ebuild", False, False, db_keys)) + if "--usepkg" in self.myopts: + db_keys = list(bindb._aux_cache_keys) + dbs.append((bindb, "binary", True, False, db_keys)) + if "--usepkgonly" in self.myopts: + db_keys = self._mydbapi_keys + dbs.append((vardb, "installed", True, True, db_keys)) + self._filtered_trees[myroot]["dbs"] = dbs if "--usepkg" in self.myopts: self.trees[myroot]["bintree"].populate( "--getbinpkg" in self.myopts, @@ -1516,15 +1531,13 @@ class depgraph(object): "given a list of .tbz2s, .ebuilds and deps, create the appropriate depgraph and return a favorite list" myfavorites=[] myroot = self.target_root + dbs = self._filtered_trees[myroot]["dbs"] + filtered_db = self._filtered_trees[myroot]["porttree"].dbapi vardb = self.trees[myroot]["vartree"].dbapi portdb = self.trees[myroot]["porttree"].dbapi bindb = self.trees[myroot]["bintree"].dbapi pkgsettings = self.pkgsettings[myroot] arg_atoms = [] - def visible(mylist): - matches = portdb.gvisible(portdb.visible(mylist)) - return [x for x in mylist \ - if x in matches or not portdb.cpv_exists(x)] for x in myfiles: ext = os.path.splitext(x)[1] if ext==".tbz2": @@ -1591,19 +1604,17 @@ class depgraph(object): portage.writemsg("!!! (Did you specify a version but forget to prefix with '='?)\n") return (0,[]) try: - mykey = None - if "--usepkg" in self.myopts: - mykey = portage.dep_expand(x, mydb=bindb, - settings=pkgsettings) - if "--usepkgonly" in self.myopts or \ - (mykey and not portage.dep_getkey(mykey).startswith("null/")): - arg_atoms.append((x, mykey)) - continue - try: - mykey = portage.dep_expand(x, - mydb=portdb, settings=pkgsettings) + for db, pkg_type, built, installed, db_keys in dbs: + mykey = portage.dep_expand(x, + mydb=db, settings=pkgsettings) + if portage.dep_getkey(mykey).startswith("null/"): + continue + break except ValueError, e: + if not e.args or not isinstance(e.args[0], list) or \ + len(e.args[0]) < 2: + raise mykey = portage.dep_expand(x, mydb=vardb, settings=pkgsettings) cp = portage.dep_getkey(mykey) @@ -1613,62 +1624,52 @@ class depgraph(object): del e arg_atoms.append((x, mykey)) except ValueError, errpkgs: + if not e.args or not isinstance(e.args[0], list) or \ + len(e.args[0]) < 2: + raise print "\n\n!!! The short ebuild name \"" + x + "\" is ambiguous. Please specify" print "!!! one of the following fully-qualified ebuild names instead:\n" for i in errpkgs[0]: print " " + green(i) print - sys.exit(1) + return False, myfavorites if "--update" in self.myopts: """Make sure all installed slots are updated when possible. Do this with --emptytree also, to ensure that all slots are remerged.""" - vardb = self.trees[self.target_root]["vartree"].dbapi greedy_atoms = [] - for myarg, myatom in arg_atoms: - greedy_atoms.append((myarg, myatom)) + for myarg, atom in arg_atoms: + greedy_atoms.append((myarg, atom)) + mykey = portage.dep_getkey(atom) myslots = set() - for cpv in vardb.match(myatom): + for cpv in vardb.match(mykey): myslots.add(vardb.aux_get(cpv, ["SLOT"])[0]) if myslots: - best_pkgs = [] - if "--usepkg" in self.myopts: - mymatches = bindb.match(myatom) - if "--usepkgonly" not in self.myopts: - mymatches = visible(mymatches) - best_pkg = portage.best(mymatches) - if best_pkg: - best_slot = bindb.aux_get(best_pkg, ["SLOT"])[0] - best_pkgs.append(("binary", best_pkg, best_slot)) - if "--usepkgonly" not in self.myopts: - best_pkg = portage.best(portdb.match(myatom)) - if best_pkg: - best_slot = portdb.aux_get(best_pkg, ["SLOT"])[0] - best_pkgs.append(("ebuild", best_pkg, best_slot)) - if best_pkgs: - best_pkg = portage.best([x[1] for x in best_pkgs]) - best_pkgs = [x for x in best_pkgs if x[1] == best_pkg] - best_slot = best_pkgs[0][2] + if not self._populate_filtered_repo(myroot, atom, + exclude_installed=True): + return False, myfavorites + mymatches = filtered_db.match(atom) + best_pkg = portage.best(mymatches) + if best_pkg: + best_slot = filtered_db.aux_get(best_pkg, ["SLOT"])[0] myslots.add(best_slot) if len(myslots) > 1: for myslot in myslots: - myslot_atom = "%s:%s" % \ - (portage.dep_getkey(myatom), myslot) - available = False - if "--usepkgonly" not in self.myopts and \ - self.trees[self.target_root][ - "porttree"].dbapi.match(myslot_atom): - available = True - elif "--usepkg" in self.myopts: - mymatches = bindb.match(myslot_atom) - if "--usepkgonly" not in self.myopts: - mymatches = visible(mymatches) - if mymatches: - available = True - if available: + myslot_atom = "%s:%s" % (mykey, myslot) + if not self._populate_filtered_repo( + myroot, myslot_atom, + exclude_installed=True): + return False, myfavorites + if filtered_db.match(myslot_atom): greedy_atoms.append((myarg, myslot_atom)) arg_atoms = greedy_atoms + # Since populate_filtered_repo() was called with the + # exclude_installed flag, these atoms will need to be processed + # again in case installed packages are required to satisfy + # dependencies. + self._filtered_trees[myroot]["atoms"].clear() + oneshot = "--oneshot" in self.myopts or \ "--onlydeps" in self.myopts """ These are used inside self.create() in order to ensure packages @@ -1758,23 +1759,7 @@ class depgraph(object): portage.dep._dep_check_strict = True filtered_atoms = self._filtered_trees[myroot]["atoms"] - dbs = self._filtered_trees[myroot].get("dbs") - if dbs is None: - dbs = [] - portdb = self.trees[myroot]["porttree"].dbapi - bindb = self.trees[myroot]["bintree"].dbapi - vardb = self.trees[myroot]["vartree"].dbapi - # (db, pkg_type, built, installed, db_keys) - if "--usepkgonly" not in self.myopts: - db_keys = list(portdb._aux_cache_keys) - dbs.append((portdb, "ebuild", False, False, db_keys)) - if "--usepkg" in self.myopts: - db_keys = list(bindb._aux_cache_keys) - dbs.append((bindb, "binary", True, False, db_keys)) - if "--usepkgonly" in self.myopts: - db_keys = self._mydbapi_keys - dbs.append((vardb, "installed", True, True, db_keys)) - self._filtered_trees[myroot]["dbs"] = dbs + dbs = self._filtered_trees[myroot]["dbs"] old_virts = pkgsettings.getvirtuals() while atoms: x = atoms.pop() @@ -2169,86 +2154,89 @@ class depgraph(object): xfrom = '(dependency required by '+ \ green('"%s"' % myparent[2]) + \ red(' [%s]' % myparent[0]) + ')' - alleb = portdb.xmatch("match-all", x) - if alleb: - if "--usepkgonly" not in self.myopts: - print "\n!!! "+red("All ebuilds that could satisfy ")+green(xinfo)+red(" have been masked.") - print "!!! One of the following masked packages is required to complete your request:" - oldcomment = "" - shown_licenses = [] - portdb_keys = ["LICENSE","SLOT"] - for p in alleb: - mreasons = portage.getmaskingstatus(p, - settings=pkgsettings, portdb=portdb) - print "- "+p+" (masked by: "+", ".join(mreasons)+")" - if "package.mask" in mreasons: - comment, filename = \ - portage.getmaskingreason(p, - settings=pkgsettings, portdb=portdb, - return_location=True) - if comment and comment != oldcomment: - print filename+":" - print comment - oldcomment = comment - try: - metadata = dict(izip(portdb_keys, - portdb.aux_get(p, portdb_keys))) - except KeyError: - # Corruption will have been reported above. - continue - metadata["USE"] = "" + masked_packages = [] + missing_licenses = [] + dbs = self._filtered_trees[myroot]["dbs"] + for db, pkg_type, built, installed, db_keys in dbs: + match = db.match + if hasattr(db, "xmatch"): + def match(atom): + return db.xmatch("match-all", atom) + cpv_list = match(x) + cpv_sort_descending(cpv_list) + for cpv in cpv_list: + try: + metadata = dict(izip(db_keys, + db.aux_get(cpv, db_keys))) + except KeyError: + mreasons = ["corruption"] + metadata = None + if metadata and not built: if "?" in metadata["LICENSE"]: pkgsettings.setcpv(p, mydb=portdb) metadata["USE"] = pkgsettings.get("USE", "") - missing_licenses = [] + else: + metadata["USE"] = "" + mreasons = portage.getmaskingstatus( + cpv, metadata=metadata, + settings=pkgsettings, portdb=portdb) + comment, filename = None, None + if "package.mask" in mreasons: + comment, filename = \ + portage.getmaskingreason( + cpv, metadata=metadata, + settings=pkgsettings, portdb=portdb, + return_location=True) + if built and \ + metadata["CHOST"] != pkgsettings["CHOST"]: + mreasons.append("CHOST: %s" % \ + metadata["CHOST"]) + if built: + if not metadata["EPREFIX"]: + mreasons.append("missing EPREFIX") + elif len(metadata["EPREFIX"].strip()) < len(pkgsettings["EPREFIX"]): + mreasons.append("EPREFIX: '%s' too small" % metadata["EPREFIX") + missing_licenses = [] + if metadata: try: missing_licenses = \ pkgsettings.getMissingLicenses( - p, metadata) + cpv, metadata) except portage.exception.InvalidDependString: # This will have already been reported # above via mreasons. pass - for l in missing_licenses: - l_path = portdb.findLicensePath(l) - if l in shown_licenses: - continue - msg = ("A copy of the '%s' license" + \ - " is located at '%s'.") % (l, l_path) - print msg - print - shown_licenses.append(l) - print - print "For more information, see MASKED PACKAGES section in the emerge man page or " - print "refer to the Gentoo Handbook." - else: - print - alleb = bindb.match(x) - if alleb: - chost = pkgsettings["CHOST"] - eprefix = pkgsettings["EPREFIX"] - bindb_keys = ["CHOST","EAPI","EPREFIX"] - for p in alleb: - mreasons = [] - metadata = dict(izip(bindb_keys, - bindb.aux_get(pkg, bindb_keys))) - if chost != metadata["CHOST"]: - mreasons.append("CHOST: %s" % \ - metadata["CHOST"]) - if not portage.eapi_is_supported( - metadata["EAPI"]): - mreasons.append(("required EAPI %s" + \ - ", supported EAPI %s") % \ - (metadata["EAPI"], - portage.const.EAPI)) - if not metadata["EPREFIX"]: - mreasons.append("missing EPREFIX") - elif len(metadata["EPREFIX"].strip()) < \ - len(eprefix): - mreasons.append("EPREFIX too small") - print "- "+p+" (masked by: "+", ".join(mreasons)+")" - print "!!! "+red("There are no packages available to satisfy: ")+green(xinfo) - print "!!! Either add a suitable binary package or compile from an ebuild." + masked_packages.append((cpv, mreasons, + comment, filename, missing_licenses)) + if masked_packages: + print "\n!!! "+red("All ebuilds that could satisfy ")+green(xinfo)+red(" have been masked.") + print "!!! One of the following masked packages is required to complete your request:" + shown_licenses = set() + shown_comments = set() + # Maybe there is both an ebuild and a binary. Only + # show one of them to avoid redundant appearance. + shown_cpvs = set() + for cpv, mreasons, comment, filename, missing_licenses in masked_packages: + if cpv in shown_cpvs: + continue + shown_cpvs.add(cpv) + print "- "+cpv+" (masked by: "+", ".join(mreasons)+")" + if comment and comment not in shown_comments: + print filename+":" + print comment + shown_comments.add(comment) + for l in missing_licenses: + l_path = portdb.findLicensePath(l) + if l in shown_licenses: + continue + msg = ("A copy of the '%s' license" + \ + " is located at '%s'.") % (l, l_path) + print msg + print + shown_licenses.add(l) + print + print "For more information, see MASKED PACKAGES section in the emerge man page or " + print "refer to the Gentoo Handbook." else: print "\nemerge: there are no ebuilds to satisfy "+green(xinfo)+"." if myparent: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 05e23ad88..fc5da4ba1 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -5081,7 +5081,7 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None): else: return mykey -def getmaskingreason(mycpv, settings=None, portdb=None, return_location=False): +def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False): from portage.util import grablines if settings is None: settings = globals()["settings"] @@ -5090,8 +5090,18 @@ def getmaskingreason(mycpv, settings=None, portdb=None, return_location=False): mysplit = catpkgsplit(mycpv) if not mysplit: raise ValueError("invalid CPV: %s" % mycpv) - if not portdb.cpv_exists(mycpv): - raise KeyError("CPV %s does not exist" % mycpv) + if metadata is None: + db_keys = list(portdb._aux_cache_keys) + try: + metadata = dict(izip(db_keys, portdb.aux_get(mycpv, db_keys))) + except KeyError: + if not portdb.cpv_exists(mycpv): + raise + if metadata is None: + # Can't access SLOT due to corruption. + cpv_slot_list = [mycpv] + else: + cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])] mycp=mysplit[0]+"/"+mysplit[1] # XXX- This is a temporary duplicate of code from the config constructor. @@ -5108,7 +5118,7 @@ def getmaskingreason(mycpv, settings=None, portdb=None, return_location=False): if settings.pmaskdict.has_key(mycp): for x in settings.pmaskdict[mycp]: - if mycpv in portdb.xmatch("match-all", x): + if match_from_list(x, cpv_slot_list): comment = "" l = "\n" comment_valid = -1 @@ -5138,7 +5148,7 @@ def getmaskingreason(mycpv, settings=None, portdb=None, return_location=False): else: return None -def getmaskingstatus(mycpv, settings=None, portdb=None): +def getmaskingstatus(mycpv, metadata=None, settings=None, portdb=None): if settings is None: settings = config(clone=globals()["settings"]) if portdb is None: @@ -5146,47 +5156,37 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): mysplit = catpkgsplit(mycpv) if not mysplit: raise ValueError("invalid CPV: %s" % mycpv) - if not portdb.cpv_exists(mycpv): - raise KeyError("CPV %s does not exist" % mycpv) + if metadata is None: + db_keys = list(portdb._aux_cache_keys) + try: + metadata = dict(izip(db_keys, portdb.aux_get(mycpv, db_keys))) + except KeyError: + if not portdb.cpv_exists(mycpv): + raise + return ["corruption"] + if "?" in metadata["LICENSE"]: + settings.setcpv(p, mydb=metadata) + metadata["USE"] = settings.get("USE", "") + else: + metadata["USE"] = "" + cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])] mycp=mysplit[0]+"/"+mysplit[1] rValue = [] # profile checking - revmaskdict=settings.prevmaskdict - if revmaskdict.has_key(mycp): - for x in revmaskdict[mycp]: - if x[0]=="*": - myatom = x[1:] - else: - myatom = x - if not match_to_list(mycpv, [myatom]): - rValue.append("profile") - break + if settings.getProfileMaskAtom(mycpv, metadata): + rValue.append("profile") # package.mask checking - maskdict=settings.pmaskdict - unmaskdict=settings.punmaskdict - if maskdict.has_key(mycp): - for x in maskdict[mycp]: - if mycpv in portdb.xmatch("match-all", x): - unmask=0 - if unmaskdict.has_key(mycp): - for z in unmaskdict[mycp]: - if mycpv in portdb.xmatch("match-all",z): - unmask=1 - break - if unmask==0: - rValue.append("package.mask") + if settings.getMaskAtom(mycpv, metadata): + rValue.append("package.mask") # keywords checking - try: - eapi, mygroups, licenses, slot = portdb.aux_get( - mycpv, ["EAPI", "KEYWORDS", "LICENSE", "SLOT"]) - except KeyError: - # The "depend" phase apparently failed for some reason. An associated - # error message will have already been printed to stderr. - return ["corruption"] + eapi = metadata["EAPI"] + mygroups = metadata["KEYWORDS"] + licenses = metadata["LICENSE"] + slot = metadata["SLOT"] if eapi.startswith("-"): eapi = eapi[1:] if not eapi_is_supported(eapi): @@ -5202,7 +5202,10 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): cp = dep_getkey(mycpv) if pkgdict.has_key(cp): - matches = match_to_list(mycpv, pkgdict[cp].keys()) + matches = [] + for match in pkgdict[cp]: + if match_from_list(match, cpv_slot_list): + matches.append(match) for match in matches: pgroups.extend(pkgdict[cp][match]) if matches: @@ -5242,13 +5245,8 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): if kmask: rValue.append(kmask+" keyword") - use = "" - if "?" in licenses: - settings.setcpv(mycpv, mydb=portdb) - use = settings.get("USE", "") try: - missing_licenses = settings.getMissingLicenses( - mycpv, {"LICENSE":licenses, "SLOT":slot, "USE":use}) + missing_licenses = settings.getMissingLicenses(mycpv, metadata) if missing_licenses: allowed_tokens = set(["||", "(", ")"]) allowed_tokens.update(missing_licenses) diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 1ed7b0359..d466f493e 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -26,7 +26,7 @@ class bindbapi(fakedbapi): self.cpdict={} # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( - ["CHOST", "EAPI", "KEYWORDS", "LICENSE", "SLOT", "USE", "EPREFIX"]) + ["CHOST", "EAPI", "IUSE", "KEYWORDS", "LICENSE", "SLOT", "USE", "EPREFIX"]) self._aux_cache = {} def match(self, *pargs, **kwargs): @@ -502,6 +502,12 @@ class binarytree(object): d.update(izip(aux_keys, self.dbapi.aux_get(mycpv, aux_keys))) + use = d["USE"].split() + iuse = set(d["IUSE"].split()) + use = [f for f in use if f in iuse] + use.sort() + d["USE"] = " ".join(use) + # record location if it's non-default if mypath != mycpv + ".tbz2": d["PATH"] = mypath diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 79820256a..d19c75d4a 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -687,7 +687,9 @@ class PackageIndex(object): if not mycpv: continue d.setdefault("EAPI", "0") + d.setdefault("IUSE", "") d.setdefault("SLOT", "0") + d.setdefault("USE", "") if header_chost: d.setdefault("CHOST", header_chost) self.packages[mycpv] = d @@ -708,6 +710,8 @@ class PackageIndex(object): del metadata["CHOST"] if metadata.get("EAPI") == "0": del metadata["EAPI"] + if metadata.get("IUSE") == "": + del metadata["IUSE"] if metadata.get("SLOT") == "0": del metadata["SLOT"] if metadata.get("USE") == "": -- 2.26.2