From 594817e3704083bf8fe5ca606392d009b6fbf0f5 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Fri, 23 May 2008 08:42:35 +0000 Subject: [PATCH] Bug #2272 - Extend dependency atom sytax to specify enabled or disabled states of USE flags. Matching with the new syntax is currently only supported in the dbapi classes and dependency resolver (use matching does not work yet in config files such as package.mask). svn path=/main/trunk/; revision=10387 --- bin/repoman | 9 +++- pym/_emerge/__init__.py | 75 ++++++++++++++++++++++++++---- pym/portage/__init__.py | 86 ++++++++++++++++------------------- pym/portage/dbapi/__init__.py | 54 ++++++++++++++++++---- pym/portage/dbapi/porttree.py | 40 ++++++++-------- pym/portage/dbapi/vartree.py | 16 ++----- pym/portage/dep.py | 76 +++++++++++++++++++++++++++++-- 7 files changed, 255 insertions(+), 101 deletions(-) diff --git a/bin/repoman b/bin/repoman index 7599b957f..5ed5c0b62 100755 --- a/bin/repoman +++ b/bin/repoman @@ -509,7 +509,7 @@ portdb = trees["/"]["porttree"].dbapi portdb.mysettings = repoman_settings # We really only need to cache the metadata that's necessary for visibility # filtering. Anything else can be discarded to reduce memory consumption. -for k in ("DEPEND", "IUSE", "LICENCE", "PDEPEND", +for k in ("DEPEND", "LICENCE", "PDEPEND", "PROVIDE", "RDEPEND", "RESTRICT", "repository"): portdb._aux_cache_keys.discard(k) # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman. @@ -1258,6 +1258,7 @@ for x in scanlist: is_blocker = atom.startswith("!") if is_blocker: atom = token.lstrip("!") + atom = portage.dep.Atom(atom) if mytype == "DEPEND" and \ not is_blocker and \ not inherited_java_eclass and \ @@ -1277,6 +1278,12 @@ for x in scanlist: (relative_path + ": %s slot dependency" + \ " not supported with EAPI='%s':" + \ " '%s'") % (mytype, eapi, atom)) + if atom.use and eapi in ("0", "1"): + stats['EAPI.incompatible'] += 1 + fails['EAPI.incompatible'].append( + (relative_path + ": %s use dependency" + \ + " not supported with EAPI='%s':" + \ + " '%s'") % (mytype, eapi, atom)) type_list.extend([mytype] * (len(badsyntax) - len(type_list))) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index d0eb64cf9..0a5306bf6 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2814,6 +2814,13 @@ class depgraph(object): return selected_atoms def _show_unsatisfied_dep(self, root, atom, myparent=None, arg=None): + atom = portage.dep.Atom(atom) + atom_without_use = atom + if atom.use: + atom_without_use = portage.dep.remove_slot(atom) + if atom.slot: + atom_without_use += ":" + atom.slot + atom_without_use = portage.dep.Atom(atom_without_use) xinfo = '"%s"' % atom if arg: xinfo='"%s"' % arg @@ -2824,9 +2831,11 @@ class depgraph(object): green('"%s"' % myparent[2]) + \ red(' [%s]' % myparent[0]) + ')' masked_packages = [] + missing_use = [] missing_licenses = [] have_eapi_mask = False pkgsettings = self.pkgsettings[root] + implicit_iuse = pkgsettings._get_implicit_iuse() root_config = self.roots[root] portdb = self.roots[root].trees["porttree"].dbapi dbs = self._filtered_trees[root]["dbs"] @@ -2835,18 +2844,61 @@ class depgraph(object): continue match = db.match if hasattr(db, "xmatch"): - cpv_list = db.xmatch("match-all", atom) + cpv_list = db.xmatch("match-all", atom_without_use) else: - cpv_list = db.match(atom) + cpv_list = db.match(atom_without_use) # descending order cpv_list.reverse() for cpv in cpv_list: metadata, mreasons = get_mask_info(root_config, cpv, pkgsettings, db, pkg_type, built, installed, db_keys) - masked_packages.append( - (root_config, pkgsettings, cpv, metadata, mreasons)) - - if masked_packages: + if atom.use and not mreasons: + missing_use.append(Package(built=built, cpv=cpv, + installed=installed, metadata=metadata, root=root)) + else: + masked_packages.append( + (root_config, pkgsettings, cpv, metadata, mreasons)) + + missing_use_reasons = [] + missing_iuse_reasons = [] + for pkg in missing_use: + use = pkg.metadata["USE"].split() + iuse = implicit_iuse.union(x.lstrip("+-") \ + for x in pkg.metadata["IUSE"].split()) + iuse_re = re.compile("^(%s)$" % "|".join(iuse)) + missing_iuse = [] + for x in atom.use.required: + if iuse_re.match(x) is None: + missing_iuse.append(x) + mreasons = [] + if missing_iuse: + mreasons.append("Missing IUSE: %s" % " ".join(missing_iuse)) + missing_iuse_reasons.append((pkg, mreasons)) + else: + need_enable = sorted(atom.use.enabled.difference(use)) + need_disable = sorted(atom.use.disabled.intersection(use)) + if need_enable or need_disable: + changes = [] + changes.extend(colorize("red", "+" + x) \ + for x in need_enable) + changes.extend(colorize("blue", "-" + x) \ + for x in need_disable) + mreasons.append("Change USE: %s" % " ".join(changes)) + missing_use_reasons.append((pkg, mreasons)) + + if missing_iuse_reasons and not missing_use_reasons: + missing_use_reasons = missing_iuse_reasons + elif missing_use_reasons: + # Only show the latest version. + del missing_use_reasons[1:] + + if missing_use_reasons: + print "\nemerge: there are no ebuilds built with USE flags to satisfy "+green(xinfo)+"." + print "!!! One of the following packages is required to complete your request:" + for pkg, mreasons in missing_use_reasons: + print "- "+pkg.cpv+" ("+", ".join(mreasons)+")" + + elif masked_packages: print "\n!!! "+red("All ebuilds that could satisfy ")+green(xinfo)+red(" have been masked.") print "!!! One of the following masked packages is required to complete your request:" have_eapi_mask = show_masked_packages(masked_packages) @@ -2892,7 +2944,8 @@ class depgraph(object): # List of acceptable packages, ordered by type preference. matched_packages = [] highest_version = None - atom_cp = portage.dep_getkey(atom) + atom = portage.dep.Atom(atom) + atom_cp = atom.cp existing_node = None myeb = None usepkgonly = "--usepkgonly" in self.myopts @@ -3024,11 +3077,17 @@ class depgraph(object): # it's not the same version. continue - if not built and not calculated_use: + if not pkg.built and not calculated_use: # This is avoided whenever possible because # it's expensive. pkgsettings.setcpv(cpv, mydb=pkg.metadata) pkg.metadata["USE"] = pkgsettings["PORTAGE_USE"] + if atom.use and not pkg.built: + use = pkg.metadata["USE"].split() + if atom.use.enabled.difference(use): + continue + if atom.use.disabled.intersection(use): + continue if pkg.cp == atom_cp: if highest_version is None: highest_version = pkg diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 274354016..84c614757 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2035,53 +2035,13 @@ class config(object): # Filter out USE flags that aren't part of IUSE. This has to # be done for every setcpv() call since practically every - # package has different IUSE. Some flags are considered to - # be implicit members of IUSE: - # - # * Flags derived from ARCH - # * Flags derived from USE_EXPAND_HIDDEN variables - # * Masked flags, such as those from {,package}use.mask - # * Forced flags, such as those from {,package}use.force - # * build and bootstrap flags used by bootstrap.sh - + # package has different IUSE. use = set(self["USE"].split()) - iuse_implicit = set(x.lstrip("+-") for x in iuse.split()) - - # Flags derived from ARCH. - arch = self.configdict["defaults"].get("ARCH") - if arch: - iuse_implicit.add(arch) - iuse_implicit.update(self.get("PORTAGE_ARCHLIST", "").split()) - - # Flags derived from USE_EXPAND_HIDDEN variables - # such as ELIBC, KERNEL, and USERLAND. - use_expand_hidden = self.get("USE_EXPAND_HIDDEN", "").split() - use_expand_hidden_raw = use_expand_hidden - if use_expand_hidden: - use_expand_hidden = re.compile("^(%s)_.*" % \ - ("|".join(x.lower() for x in use_expand_hidden))) - for x in use: - if use_expand_hidden.match(x): - iuse_implicit.add(x) - - # Flags that have been masked or forced. - iuse_implicit.update(self.usemask) - iuse_implicit.update(self.useforce) + iuse_implicit = self._get_implicit_iuse() + iuse_implicit.update(x.lstrip("+-") for x in iuse.split()) - # build and bootstrap flags used by bootstrap.sh - iuse_implicit.add("build") - iuse_implicit.add("bootstrap") - - if ebuild_phase: - iuse_grep = iuse_implicit.copy() - if use_expand_hidden_raw: - for x in use_expand_hidden_raw: - iuse_grep.add(x.lower() + "_.*") - if iuse_grep: - iuse_grep = "^(%s)$" % "|".join(sorted(iuse_grep)) - else: - iuse_grep = "" - self.configdict["pkg"]["PORTAGE_IUSE"] = iuse_grep + self.configdict["pkg"]["PORTAGE_IUSE"] = \ + "^(%s)$" % "|".join(sorted(iuse_implicit)) ebuild_force_test = self.get("EBUILD_FORCE_TEST") == "1" if ebuild_force_test and ebuild_phase and \ @@ -2175,6 +2135,38 @@ class config(object): x for x in use if \ x in iuse_implicit)) + def _get_implicit_iuse(self): + """ + Some flags are considered to + be implicit members of IUSE: + * Flags derived from ARCH + * Flags derived from USE_EXPAND_HIDDEN variables + * Masked flags, such as those from {,package}use.mask + * Forced flags, such as those from {,package}use.force + * build and bootstrap flags used by bootstrap.sh + """ + iuse_implicit = set() + # Flags derived from ARCH. + arch = self.configdict["defaults"].get("ARCH") + if arch: + iuse_implicit.add(arch) + iuse_implicit.update(self.get("PORTAGE_ARCHLIST", "").split()) + + # Flags derived from USE_EXPAND_HIDDEN variables + # such as ELIBC, KERNEL, and USERLAND. + use_expand_hidden = self.get("USE_EXPAND_HIDDEN", "").split() + for x in use_expand_hidden: + iuse_implicit.add(x.lower() + "_.*") + + # Flags that have been masked or forced. + iuse_implicit.update(self.usemask) + iuse_implicit.update(self.useforce) + + # build and bootstrap flags used by bootstrap.sh + iuse_implicit.add("build") + iuse_implicit.add("bootstrap") + return iuse_implicit + def getMaskAtom(self, cpv, metadata): """ Take a package and return a matching package.mask atom, or None if no @@ -5728,8 +5720,8 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None): myindex = orig_dep.index(mydep) prefix = orig_dep[:myindex] postfix = orig_dep[myindex+len(mydep):] - return prefix + cpv_expand( - mydep, mydb=mydb, use_cache=use_cache, settings=settings) + postfix + return portage.dep.Atom(prefix + cpv_expand( + mydep, mydb=mydb, use_cache=use_cache, settings=settings) + postfix) def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, use_cache=1, use_binaries=0, myroot="/", trees=None): diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index 1431c1074..95219322f 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -4,7 +4,8 @@ import os import re -from portage.dep import dep_getslot, dep_getkey, match_from_list +from portage.dep import Atom, dep_getslot, dep_getkey, \ + dep_getusedeps, match_from_list from portage.locks import unlockfile from portage.output import red from portage.util import writemsg @@ -16,6 +17,8 @@ class dbapi(object): _category_re = re.compile(r'^\w[-.+\w]*$') _pkg_dir_name_re = re.compile(r'^\w[-+\w]*$') _categories = None + _iuse_implicit = None + _use_mutable = False _known_keys = frozenset(x for x in auxdbkeys if not x.startswith("UNUSED_0")) def __init__(self): @@ -119,13 +122,48 @@ class dbapi(object): a list of packages that match origdep """ mydep = dep_expand(origdep, mydb=self, settings=self.settings) - mykey = dep_getkey(mydep) - mylist = match_from_list(mydep, self.cp_list(mykey, use_cache=use_cache)) - myslot = dep_getslot(mydep) - if myslot is not None: - mylist = [cpv for cpv in mylist \ - if self.aux_get(cpv, ["SLOT"])[0] == myslot] - return mylist + return list(self._iter_match(mydep, + self.cp_list(mydep.cp, use_cache=use_cache))) + + def _iter_match(self, atom, cpv_iter): + cpv_iter = match_from_list(atom, cpv_iter) + if atom.slot: + cpv_iter = self._iter_match_slot(atom, cpv_iter) + if atom.use: + cpv_iter = self._iter_match_use(atom, cpv_iter) + return cpv_iter + + def _iter_match_slot(self, atom, cpv_iter): + for cpv in cpv_iter: + if self.aux_get(cpv, ["SLOT"])[0] == atom.slot: + yield cpv + + def _iter_match_use(self, atom, cpv_iter): + """ + 1) Check for required IUSE intersection (need implicit IUSE here). + 2) Check enabled/disabled flag states. + """ + if self._iuse_implicit is None: + self._iuse_implicit = self.settings._get_implicit_iuse() + for cpv in cpv_iter: + iuse, use = self.aux_get(cpv, ["IUSE", "USE"]) + use = use.split() + iuse = self._iuse_implicit.union( + x.lstrip("+-") for x in iuse.split()) + iuse_re = re.compile("^(%s)$" % "|".join(iuse)) + missing_iuse = False + for x in atom.use.required: + if iuse_re.match(x) is None: + missing_iuse = True + break + if missing_iuse: + continue + if not self._use_mutable: + if atom.use.enabled.difference(use): + continue + if atom.use.disabled.intersection(use): + continue + yield cpv def invalidentry(self, mypath): if mypath.endswith('portage_lockfile'): diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 70135be34..7fe361570 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -27,6 +27,7 @@ from itertools import izip class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" portdbapi_instances = [] + _use_mutable = True def __init__(self, porttree_root, mysettings=None): portdbapi.portdbapi_instances.append(self) @@ -36,6 +37,7 @@ class portdbapi(dbapi): else: from portage import settings self.mysettings = config(clone=settings) + self._iuse_implicit = self.mysettings._get_implicit_iuse() self._categories = set(self.mysettings.categories) # This is strictly for use in aux_get() doebuild calls when metadata # is generated by the depend phase. It's safest to use a clone for @@ -615,22 +617,14 @@ class portdbapi(dbapi): # Find the minimum matching version. This is optimized to # minimize the number of metadata accesses (improves performance # especially in cases where metadata needs to be generated). - if mydep == mykey: - mylist = self.cp_list(mykey) - else: - mylist = match_from_list(mydep, self.cp_list(mykey)) + cpv_iter = iter(self.cp_list(mykey)) + if mydep != mykey: + cpv_iter = self._iter_match(mydep, cpv_iter) myval = "" - if mylist: - if myslot is None: - myval = mylist[0] - else: - for cpv in mylist: - try: - if self.aux_get(cpv, ["SLOT"])[0] == myslot: - myval = cpv - break - except KeyError: - pass # ebuild masked by corruption + for cpv in cpv_iter: + myval = cpv + break + elif level in ("minimum-visible", "bestmatch-visible"): # Find the minimum matching visible version. This is optimized to # minimize the number of metadata accesses (improves performance @@ -674,29 +668,35 @@ class portdbapi(dbapi): continue except InvalidDependString: continue + if mydep.use: + has_iuse = False + for has_iuse in self._iter_match_use(mydep, [cpv]): + break + if not has_iuse: + continue myval = cpv break elif level == "bestmatch-list": #dep match -- find best match but restrict search to sublist #no point in calling xmatch again since we're not caching list deps - myval = best(match_from_list(mydep, mylist)) + myval = best(list(self._iter_match(mydep, mylist))) elif level == "match-list": #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible()) - myval = match_from_list(mydep, mylist) + myval = list(self._iter_match(mydep, mylist)) elif level == "match-visible": #dep match -- find all visible matches #get all visible packages, then get the matching ones - myval = match_from_list(mydep, - self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey)) + myval = list(self._iter_match(mydep, + self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey))) elif level == "match-all": #match *all* visible *and* masked packages if mydep == mykey: myval = self.cp_list(mykey) else: - myval = match_from_list(mydep, self.cp_list(mykey)) + myval = list(self._iter_match(mydep, self.cp_list(mykey))) else: print "ERROR: xmatch doesn't handle", level, "query!" raise KeyError diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index ecd17f81e..f1923b17f 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -507,13 +507,8 @@ class vardbapi(dbapi): if self.matchcache.has_key(mycat): del self.mtdircache[mycat] del self.matchcache[mycat] - mymatch = match_from_list(mydep, - self.cp_list(mykey, use_cache=use_cache)) - myslot = dep_getslot(mydep) - if myslot is not None: - mymatch = [cpv for cpv in mymatch \ - if self.aux_get(cpv, ["SLOT"])[0] == myslot] - return mymatch + return list(self._iter_match(mydep, + self.cp_list(mydep.cp, use_cache=use_cache))) try: curmtime = os.stat(self.root+VDB_PATH+"/"+mycat)[stat.ST_MTIME] except (IOError, OSError): @@ -524,11 +519,8 @@ class vardbapi(dbapi): self.mtdircache[mycat] = curmtime self.matchcache[mycat] = {} if not self.matchcache[mycat].has_key(mydep): - mymatch = match_from_list(mydep, self.cp_list(mykey, use_cache=use_cache)) - myslot = dep_getslot(mydep) - if myslot is not None: - mymatch = [cpv for cpv in mymatch \ - if self.aux_get(cpv, ["SLOT"])[0] == myslot] + mymatch = list(self._iter_match(mydep, + self.cp_list(mydep.cp, use_cache=use_cache))) self.matchcache[mycat][mydep] = mymatch return self.matchcache[mycat][mydep][:] diff --git a/pym/portage/dep.py b/pym/portage/dep.py index b12411d63..267b356fb 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -329,6 +329,37 @@ def dep_opconvert(deplist): x += 1 return retlist +class _use_dep(object): + def __init__(self, use): + enabled_flags = [] + disabled_flags = [] + for x in use: + if "-" == x[:1]: + disabled_flags.append(x[1:]) + else: + enabled_flags.append(x) + self.enabled = frozenset(enabled_flags) + self.disabled = frozenset(disabled_flags) + self.required = self.enabled.union(self.disabled) + +class Atom(str): + + def __init__(self, s): + str.__init__(self, s) + if not isvalidatom(s, allow_blockers=True): + raise InvalidAtom(s) + self.blocker = "!" == s[:1] + if self.blocker: + s = s[1:] + self.cp = dep_getkey(s) + self.cpv = dep_getcpv(s) + self.slot = dep_getslot(s) + self.operator = get_operator(s) + #self.repo = self._get_repo(s) + self.use = dep_getusedeps(s) + if self.use: + self.use = _use_dep(self.use) + def get_operator(mydep): """ Return the operator used in a depstring. @@ -344,6 +375,9 @@ def get_operator(mydep): @return: The operator. One of: '~', '=', '>', '<', '=*', '>=', or '<=' """ + operator = getattr(mydep, "operator", None) + if operator is not None: + return operator if mydep: mydep = remove_slot(mydep) if not mydep: @@ -380,6 +414,9 @@ def dep_getcpv(mydep): @rtype: String @return: The depstring with the operator removed """ + cpv = getattr(mydep, "cpv", None) + if cpv is not None: + return cpv global _dep_getcpv_cache retval = _dep_getcpv_cache.get(mydep, None) if retval is not None: @@ -413,15 +450,32 @@ def dep_getslot(mydep): @rtype: String @return: The slot """ - colon = mydep.rfind(":") + slot = getattr(mydep, "slot", None) + if slot is not None: + return slot + colon = mydep.find(":") if colon != -1: - return mydep[colon+1:] + bracket = mydep.find("[", colon) + if bracket == -1: + return mydep[colon+1:] + else: + return mydep[colon+1:bracket] return None def remove_slot(mydep): - colon = mydep.rfind(":") + """ + Removes dep components from the right side of an atom: + * slot + * use + * repo + """ + colon = mydep.find(":") if colon != -1: mydep = mydep[:colon] + else: + bracket = mydep.find("[") + if bracket != -1: + mydep = mydep[:bracket] return mydep def dep_getusedeps( depend ): @@ -437,6 +491,9 @@ def dep_getusedeps( depend ): @rtype: List @return: List of use flags ( or [] if no flags exist ) """ + use = getattr(depend, "use", None) + if use is not None: + return use use_list = [] open_bracket = depend.find('[') # -1 = failure (think c++ string::npos) @@ -452,7 +509,7 @@ def dep_getusedeps( depend ): raise InvalidAtom("USE Dependency with no use flag ([]): %s" % depend ) # Find next use flag open_bracket = depend.find( '[', open_bracket+1 ) - return use_list + return tuple(use_list) _invalid_atom_chars_regexp = re.compile("[()|?@]") @@ -473,6 +530,10 @@ def isvalidatom(atom, allow_blockers=False): 1) 0 if the atom is invalid 2) 1 if the atom is valid """ + if isinstance(atom, Atom): + if atom.blocker and not allow_blockers: + return 0 + return 1 global _invalid_atom_chars_regexp if _invalid_atom_chars_regexp.search(atom): return 0 @@ -571,6 +632,9 @@ def dep_getkey(mydep): @rtype: String @return: The package category/package-version """ + cp = getattr(mydep, "cp", None) + if cp is not None: + return cp mydep = dep_getcpv(mydep) if mydep and isspecific(mydep): mysplit = catpkgsplit(mydep) @@ -646,8 +710,10 @@ def match_from_list(mydep, candidate_list): """ from portage.util import writemsg - if mydep[0] == "!": + if "!" == mydep[:1]: mydep = mydep[1:] + if not isinstance(mydep, Atom): + mydep = Atom(mydep) mycpv = dep_getcpv(mydep) mycpv_cps = catpkgsplit(mycpv) # Can be None if not specific -- 2.26.2