From 51164f1475c6f3fa958bf9e9da44406295a2fbd1 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 26 Oct 2007 17:33:22 +0000 Subject: [PATCH] Merged from trunk 8299:8310 | 8300 | Allow sets to contain non-atoms | | genone | | | 8301 | use a @ prefix for sets, and replace world/system with their | | genone | real prefixed names if found on the commandline | | 8302 | use SetConfig.getSetAtoms instead of PackageSet.getAtoms for | | genone | expanding sets to allow recursion, and drop the default | | | 'sets/' prefix for setnames | | 8303 | break potential cycles in set expansion | | genone | | | 8304 | update notes about package sets | | genone | | | 8305 | Optimize the algorithm used for expansion of set arguments | | zmedico | into atoms. | | 8306 | Prevent premature expansion of old-style virtuals inside | | zmedico | depgraph.select_files(). | | 8307 | Allow sets to get through world file validation and expand | | zmedico | properly. It's now possible to put @system in the world file | | | and get the old behavior where world includes system. | | 8308 | Make emerge add sets to the world file as one would expect. | | zmedico | | | 8309 | Optimize repoman visibility checks to access as little | | zmedico | metadata as possible. (improves performance especially in | | | cases where metadata needs to be generated). This works by | | | starting at the lowest version since that's most likely to | | | have keywords and it returns as soon as the first visible | | | package is found. | | 8310 | Add missing break statement for xmatch minimum-visible. | | zmedico | | svn path=/main/branches/prefix/; revision=8312 --- RELEASE-NOTES | 14 +++---- bin/repoman | 4 +- pym/_emerge/__init__.py | 59 +++++++++++++++++++++++----- pym/portage/__init__.py | 8 +++- pym/portage/dbapi/porttree.py | 73 +++++++++++++++++++++++++++++++++-- pym/portage/sets/__init__.py | 14 ++++++- pym/portage/sets/base.py | 28 +++++++++++--- pym/portage/sets/files.py | 18 +++++++-- 8 files changed, 184 insertions(+), 34 deletions(-) diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 076bb12c5..8c1f6b3eb 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -18,15 +18,15 @@ portage-2.2 features in 2.2 make use of or require named repositories. The repository name is stored in profiles/repo_name in each repository. * Package set support: There are several important notes regarding package sets: - - they may currently only include simple and versioned atom, use conditionals - or any-of constructs aren't possible yet + - setnames have to be prefixed with @ (exceptions: 'world' and 'system' can be + used without the prefix) + - they may currently only include simple and versioned atoms or other sets, use + conditionals or any-of constructs aren't possible yet - emerge makes no difference atm wether you pass a setname or all atoms contained in the set to it, this means that without options packages will be remerged if - already installed and added to the worldfile (exception: "world" and "system" - are still handled specially), or in the case of --unmerge all atoms in a set - will be unmerged even if they are depended upon by other packages - - sets shadow packages with the same name - - sets can not contain other sets + already installed and added to the worldfile, or in the case of --unmerge all + atoms in a set will be unmerged even if they are depended upon by other + packages portage-2.1.3 ================================== diff --git a/bin/repoman b/bin/repoman index 3ea24c1b4..617bf9712 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1520,10 +1520,10 @@ for x in scanlist: continue #we are testing deps for a masked package; give it some lee-way suffix="masked" - matchmode="match-all" + matchmode = "minimum-all" else: suffix="" - matchmode="match-visible" + matchmode = "minimum-visible" if prof[1] == "dev": suffix=suffix+"indev" diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 8ed59ce84..b3d986a4f 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -53,7 +53,7 @@ from portage.const import EPREFIX, BPREFIX from portage.data import secpass from portage.util import normalize_path as normpath from portage.util import writemsg -from portage.sets import SetConfig, make_default_config +from portage.sets import SetConfig, make_default_config, SETPREFIX from portage.sets.profiles import PackagesSystemSet as SystemSet from portage.sets.base import InternalPackageSet from portage.sets.files import WorldSet @@ -1692,6 +1692,16 @@ class depgraph(object): portage.writemsg("!!! Please check ebuild(5) for full details.\n") portage.writemsg("!!! (Did you specify a version but forget to prefix with '='?)\n") return (0,[]) + # Don't expand categories or old-style virtuals here unless + # necessary. Expansion of old-style virtuals here causes at + # least the following problems: + # 1) It's more difficult to determine which set(s) an atom + # came from, if any. + # 2) It takes away freedom from the resolver to choose other + # possible expansions when necessary. + if "/" in x: + arg_atoms.append((x, x)) + continue try: try: for db, pkg_type, built, installed, db_keys in dbs: @@ -3526,14 +3536,27 @@ class depgraph(object): if myfavkey in added_favorites: continue added_favorites.add(myfavkey) - world_set.add(myfavkey) - print ">>> Recording",myfavkey,"in \"world\" favorites file..." except portage.exception.InvalidDependString, e: writemsg("\n\n!!! '%s' has invalid PROVIDE: %s\n" % \ (pkg_key, str(e)), noiselevel=-1) writemsg("!!! see '%s'\n\n" % os.path.join( root, portage.VDB_PATH, pkg_key, "PROVIDE"), noiselevel=-1) del e + all_added = [] + for k in self._sets: + if k in ("args", "world"): + continue + s = SETPREFIX + k + if s in world_set: + continue + all_added.append(SETPREFIX + k) + all_added.extend(added_favorites) + all_added.sort() + for a in all_added: + print ">>> Recording %s in \"world\" favorites file..." % \ + colorize("INFORM", a) + if all_added: + world_set.update(all_added) world_set.unlock() def loadResumeCommand(self, resume_data): @@ -6230,8 +6253,7 @@ def action_build(settings, trees, mtimedb, pkglist.append(pkg) else: pkglist = mydepgraph.altlist() - if favorites: - mydepgraph.saveNomergeFavorites() + mydepgraph.saveNomergeFavorites() del mydepgraph mergetask = MergeTask(settings, trees, myopts) retval = mergetask.merge(pkglist, favorites, mtimedb) @@ -6565,20 +6587,37 @@ def emerge_main(): # only expand sets for actions taking package arguments oldargs = myfiles[:] if myaction not in ["search", "metadata", "sync"]: - for s in settings.sets: - if s in myfiles: + newargs = [] + for a in myfiles: + if a in ("system", "world"): + newargs.append(SETPREFIX+a) + else: + newargs.append(a) + myfiles = newargs + del newargs + newargs = [] + for a in myfiles: + if a.startswith(SETPREFIX): + s = a[len(SETPREFIX):] + if s not in settings.sets: + print "emerge: there are no sets to satisfy %s." % \ + colorize("INFORM", s) + return 1 # TODO: check if the current setname also resolves to a package name if myaction in ["unmerge", "prune", "clean", "depclean"] and not packagesets[s].supportsOperation("unmerge"): print "emerge: the given set %s does not support unmerge operations" % s return 1 - if not settings.sets[s].getAtoms(): + if not settings.setconfig.getSetAtoms(s): print "emerge: '%s' is an empty set" % s else: - myfiles.extend(settings.sets[s].getAtoms()) + newargs.extend(settings.setconfig.getSetAtoms(s)) mysets[s] = settings.sets[s] for e in settings.sets[s].errors: print e - myfiles.remove(s) + else: + newargs.append(a) + myfiles = newargs + del newargs # Need to handle empty sets specially, otherwise emerge will react # with the help message for empty argument lists if oldargs and not myfiles: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index e29c14f43..7231934ae 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4989,7 +4989,13 @@ def dep_wordreduce(mydeplist,mysettings,mydbapi,mode,use_cache=1): deplist[mypos] = False else: if mode: - mydep=mydbapi.xmatch(mode,deplist[mypos]) + x = mydbapi.xmatch(mode, deplist[mypos]) + if mode.startswith("minimum-"): + mydep = [] + if x: + mydep.append(x) + else: + mydep = x else: mydep=mydbapi.match(deplist[mypos],use_cache=use_cache) if mydep!=None: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 05fe2a767..f0cc49ac2 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -14,7 +14,7 @@ from portage.exception import OperationNotPermitted, PortageException, \ from portage.manifest import Manifest from portage.output import red from portage.util import ensure_dirs, writemsg, apply_recursive_permissions -from portage.versions import pkgsplit, catpkgsplit, best +from portage.versions import pkgcmp, pkgsplit, catpkgsplit, best import portage.gpg, portage.checksum @@ -521,13 +521,27 @@ class portdbapi(dbapi): mylist = [] else: mylist = d.keys() + # Always sort in ascending order here since it's handy + # and the result can be easily cached and reused. + if len(mylist) > 1: + for i in xrange(len(mylist)): + mylist[i] = catpkgsplit(mylist[i])[1:] + mylist.sort(pkgcmp) + cat = mysplit[0] + for i, (pn, ver, rev) in enumerate(mylist): + if rev == "r0": + cpv = cat + "/" + pn + "-" + ver + else: + cpv = cat + "/" + pn + "-" + ver + "-" + rev + mylist[i] = cpv if self.frozen and mytree is None: if not (not mylist and mycp.startswith("virtual/")): self.xcache["match-all"][mycp] = mylist[:] return mylist def freeze(self): - for x in ["list-visible", "bestmatch-visible", "match-visible", "match-all"]: + for x in "bestmatch-visible", "list-visible", "match-all", \ + "match-visible", "minimum-all", "minimum-visible": self.xcache[x]={} self.frozen=1 @@ -550,6 +564,7 @@ class portdbapi(dbapi): mydep = dep_expand(origdep, mydb=self, settings=self.mysettings) mykey = dep_getkey(mydep) + myslot = dep_getslot(mydep) if level == "list-visible": #a list of all visible packages, not called directly (just by xmatch()) #myval = self.visible(self.cp_list(mykey)) @@ -560,6 +575,57 @@ class portdbapi(dbapi): #get all visible matches (from xmatch()), then choose the best one myval = best(self.xmatch("match-visible", None, mydep=mydep, mykey=mykey)) + elif level == "minimum-all": + # Find the minimum matching version. This is optimized to + # minimize the number of metadata accesses (improves performance + # especially in cases where metadata needs to be generated). + if mydep == mykey: + mylist = self.cp_list(mykey) + else: + mylist = match_from_list(mydep, self.cp_list(mykey)) + myval = "" + if mylist: + if myslot is None: + myval = mylist[0] + else: + for cpv in mylist: + try: + if self.aux_get(cpv, ["SLOT"])[0] == myslot: + myval = cpv + break + except KeyError: + pass # ebuild masked by corruption + elif level == "minimum-visible": + # Find the minimum matching visible version. This is optimized to + # minimize the number of metadata accesses (improves performance + # especially in cases where metadata needs to be generated). + # This does not implement LICENSE filtering since it's only + # intended for use by repoman. + if mydep == mykey: + mylist = self.cp_list(mykey) + else: + mylist = match_from_list(mydep, self.cp_list(mykey)) + myval = "" + settings = self.mysettings + for cpv in mylist: + try: + metadata = dict(izip(self._aux_cache_keys, + self.aux_get(cpv, self._aux_cache_keys))) + except KeyError: + # ebuild masked by corruption + continue + if not eapi_is_supported(metadata["EAPI"]): + continue + if myslot and myslot != metadata["SLOT"]: + continue + if settings.getMissingKeywords(cpv, metadata): + continue + if settings.getMaskAtom(cpv, metadata): + continue + if settings.getProfileMaskAtom(cpv, metadata): + continue + myval = cpv + break elif level == "bestmatch-list": #dep match -- find best match but restrict search to sublist #no point in calling xmatch again since we're not caching list deps @@ -584,8 +650,7 @@ class portdbapi(dbapi): else: print "ERROR: xmatch doesn't handle", level, "query!" raise KeyError - myslot = dep_getslot(mydep) - if myslot is not None: + if myslot is not None and isinstance(myval, list): slotmatches = [] for cpv in myval: try: diff --git a/pym/portage/sets/__init__.py b/pym/portage/sets/__init__.py index b757c43c4..9f562820b 100644 --- a/pym/portage/sets/__init__.py +++ b/pym/portage/sets/__init__.py @@ -10,6 +10,8 @@ DEFAULT_SETS = ["world", "system", "everything", "security"] \ +["package_"+x for x in ["mask", "unmask", "use", "keywords"]] del x +SETPREFIX = "@" + class SetConfigError(Exception): pass @@ -61,7 +63,7 @@ class SetConfig(SafeConfigParser): try: setname = self.get(sname, "name") except NoOptionError: - setname = "sets/"+sname + setname = sname if hasattr(setclass, "singleBuilder"): try: self.psets[setname] = setclass.singleBuilder(optdict, self.settings, self.trees) @@ -92,6 +94,16 @@ class SetConfig(SafeConfigParser): self.aliases = shortnames return self.aliases + def getSetAtoms(self, setname, ignorelist=[]): + myset = self.getSetsWithAliases()[setname] + myatoms = myset.getAtoms() + ignorelist.append(setname) + for n in myset.getNonAtoms(): + if n[0] == SETPREFIX and n[1:] in self.aliases: + if n[1:] not in ignorelist: + myatoms.update(self.getSetAtoms(n[1:])) + return myatoms + def make_default_config(settings, trees): sc = SetConfig([], settings, trees) sc.add_section("security") diff --git a/pym/portage/sets/base.py b/pym/portage/sets/base.py index b5ea88928..c22ee9953 100644 --- a/pym/portage/sets/base.py +++ b/pym/portage/sets/base.py @@ -23,9 +23,10 @@ class PackageSet(object): self._loaded = False self._loading = False self.errors = [] + self._nonatoms = set() def __contains__(self, atom): - return atom in self.getAtoms() + return atom in self.getAtoms() or atom in self._nonatoms def __iter__(self): for x in self.getAtoms(): @@ -44,13 +45,19 @@ class PackageSet(object): self._loading = False return self._atoms + def getNonAtoms(self): + self.getAtoms() + return self._nonatoms + def _setAtoms(self, atoms): atoms = map(str.strip, atoms) + nonatoms = set() for a in atoms[:]: if a == "": atoms.remove(a) elif not isvalidatom(a): - raise InvalidAtom(a) + atoms.remove(a) + self._nonatoms.add(a) self._atoms = set(atoms) self._updateAtomMap() @@ -115,9 +122,20 @@ class EditablePackageSet(PackageSet): def update(self, atoms): self.getAtoms() - self._atoms.update(atoms) - self._updateAtomMap(atoms=atoms) - self.write() + modified = False + normal_atoms = [] + for a in atoms: + if isvalidatom(a): + normal_atoms.append(a) + else: + modified = True + self._nonatoms.add(a) + if normal_atoms: + modified = True + self._atoms.update(normal_atoms) + self._updateAtomMap(atoms=normal_atoms) + if modified: + self.write() def add(self, atom): self.update([atom]) diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index 9cbb018da..d300eaad2 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -3,13 +3,14 @@ # $Id$ import os +from itertools import chain from portage.util import grabfile, write_atomic, ensure_dirs from portage.const import PRIVATE_PATH, USER_CONFIG_PATH from portage.locks import lockfile, unlockfile from portage import portage_gid from portage.sets.base import PackageSet, EditablePackageSet -from portage.sets import SetConfigError +from portage.sets import SetConfigError, SETPREFIX from portage.env.loaders import ItemFileLoader, KeyListFileLoader from portage.env.validators import ValidAtomValidator from portage import dep_getkey, cpv_getkey @@ -24,7 +25,7 @@ class StaticFileSet(EditablePackageSet): self._filename = filename self._mtime = None self.description = "Package set loaded from file %s" % self._filename - self.loader = ItemFileLoader(self._filename, ValidAtomValidator) + self.loader = ItemFileLoader(self._filename, self._validate) metadata = grabfile(self._filename + ".metadata") key = None @@ -44,9 +45,13 @@ class StaticFileSet(EditablePackageSet): else: if key != None: setattr(self, key, " ".join(value)) - + + def _validate(self, atom): + return ValidAtomValidator(atom) + def write(self): - write_atomic(self._filename, "\n".join(sorted(self._atoms))+"\n") + write_atomic(self._filename, "\n".join(sorted( + chain(self._atoms, self._nonatoms)))+"\n") def load(self): try: @@ -126,6 +131,11 @@ class WorldSet(StaticFileSet): super(WorldSet, self).__init__(os.path.join(os.sep, root, PRIVATE_PATH.lstrip(os.sep), "world")) self._lock = None + def _validate(self, atom): + if atom.startswith(SETPREFIX): + return True + return ValidAtomValidator(atom) + def _ensure_dirs(self): ensure_dirs(os.path.dirname(self._filename), gid=portage_gid, mode=02750, mask=02) -- 2.26.2