Merged from trunk 8299:8310
authorFabian Groffen <grobian@gentoo.org>
Fri, 26 Oct 2007 17:33:22 +0000 (17:33 -0000)
committerFabian Groffen <grobian@gentoo.org>
Fri, 26 Oct 2007 17:33:22 +0000 (17:33 -0000)
   | 8300    | Allow sets to contain non-atoms                              |
   | genone  |                                                              |

   | 8301    | use a @ prefix for sets, and replace world/system with their |
   | genone  | real prefixed names if found on the commandline              |

   | 8302    | use SetConfig.getSetAtoms instead of PackageSet.getAtoms for |
   | genone  | expanding sets to allow recursion, and drop the default      |
   |         | 'sets/' prefix for setnames                                  |

   | 8303    | break potential cycles in set expansion                      |
   | genone  |                                                              |

   | 8304    | update notes about package sets                              |
   | genone  |                                                              |

   | 8305    | Optimize the algorithm used for expansion of set arguments   |
   | zmedico | into atoms.                                                  |

   | 8306    | Prevent premature expansion of old-style virtuals inside     |
   | zmedico | depgraph.select_files().                                     |

   | 8307    | Allow sets to get through world file validation and expand   |
   | zmedico | properly. It's now possible to put @system in the world file |
   |         | and get the old behavior where world includes system.        |

   | 8308    | Make emerge add sets to the world file as one would expect.  |
   | zmedico |                                                              |

   | 8309    | Optimize repoman visibility checks to access as little       |
   | zmedico | metadata as possible. (improves performance especially in    |
   |         | cases where metadata needs to be generated). This works by   |
   |         | starting at the lowest version since that's most likely to   |
   |         | have keywords and it returns as soon as the first visible    |
   |         | package is found.                                            |

   | 8310    | Add missing break statement for xmatch minimum-visible.      |
   | zmedico |                                                              |

svn path=/main/branches/prefix/; revision=8312

RELEASE-NOTES
bin/repoman
pym/_emerge/__init__.py
pym/portage/__init__.py
pym/portage/dbapi/porttree.py
pym/portage/sets/__init__.py
pym/portage/sets/base.py
pym/portage/sets/files.py

index 076bb12c5d3a62c5e027553be719fa9897aeb147..8c1f6b3eb867c08d59845ecb7751b7ed46479a37 100644 (file)
@@ -18,15 +18,15 @@ portage-2.2
   features in 2.2 make use of or require named repositories. The repository name
   is stored in profiles/repo_name in each repository.
 * Package set support: There are several important notes regarding package sets:
-  - they may currently only include simple and versioned atom, use conditionals
-    or any-of constructs aren't possible yet
+  - setnames have to be prefixed with @ (exceptions: 'world' and 'system' can be 
+    used without the prefix)
+  - they may currently only include simple and versioned atoms or other sets, use
+    conditionals or any-of constructs aren't possible yet
   - emerge makes no difference atm wether you pass a setname or all atoms contained
     in the set to it, this means that without options packages will be remerged if 
-       already installed and added to the worldfile (exception: "world" and "system"
-       are still handled specially), or in the case of --unmerge all atoms in a set
-       will be unmerged even if they are depended upon by other packages
-  - sets shadow packages with the same name
-  - sets can not contain other sets    
+       already installed and added to the worldfile, or in the case of --unmerge all
+       atoms in a set will be unmerged even if they are depended upon by other 
+       packages
 
 portage-2.1.3
 ==================================
index 3ea24c1b48589217f7fd807e097b2228ac0558ee..617bf9712055ab57bbdad8b15cd2f2214718f511 100755 (executable)
@@ -1520,10 +1520,10 @@ for x in scanlist:
                                                        continue
                                                #we are testing deps for a masked package; give it some lee-way
                                                suffix="masked"
-                                               matchmode="match-all"
+                                               matchmode = "minimum-all"
                                        else:
                                                suffix=""
-                                               matchmode="match-visible"
+                                               matchmode = "minimum-visible"
        
                                        if prof[1] == "dev":
                                                suffix=suffix+"indev"
index 8ed59ce8422caf8bec3546cbcf87805dd5af1286..b3d986a4f8cbcab3c14637b18b6c5da6575c74a4 100644 (file)
@@ -53,7 +53,7 @@ from portage.const import EPREFIX, BPREFIX
 from portage.data import secpass
 from portage.util import normalize_path as normpath
 from portage.util import writemsg
-from portage.sets import SetConfig, make_default_config
+from portage.sets import SetConfig, make_default_config, SETPREFIX
 from portage.sets.profiles import PackagesSystemSet as SystemSet
 from portage.sets.base import InternalPackageSet
 from portage.sets.files import WorldSet
@@ -1692,6 +1692,16 @@ class depgraph(object):
                                        portage.writemsg("!!! Please check ebuild(5) for full details.\n")
                                        portage.writemsg("!!! (Did you specify a version but forget to prefix with '='?)\n")
                                        return (0,[])
+                               # Don't expand categories or old-style virtuals here unless
+                               # necessary. Expansion of old-style virtuals here causes at
+                               # least the following problems:
+                               #   1) It's more difficult to determine which set(s) an atom
+                               #      came from, if any.
+                               #   2) It takes away freedom from the resolver to choose other
+                               #      possible expansions when necessary.
+                               if "/" in x:
+                                       arg_atoms.append((x, x))
+                                       continue
                                try:
                                        try:
                                                for db, pkg_type, built, installed, db_keys in dbs:
@@ -3526,14 +3536,27 @@ class depgraph(object):
                                        if myfavkey in added_favorites:
                                                continue
                                        added_favorites.add(myfavkey)
-                                       world_set.add(myfavkey)
-                                       print ">>> Recording",myfavkey,"in \"world\" favorites file..."
                        except portage.exception.InvalidDependString, e:
                                writemsg("\n\n!!! '%s' has invalid PROVIDE: %s\n" % \
                                        (pkg_key, str(e)), noiselevel=-1)
                                writemsg("!!! see '%s'\n\n" % os.path.join(
                                        root, portage.VDB_PATH, pkg_key, "PROVIDE"), noiselevel=-1)
                                del e
+               all_added = []
+               for k in self._sets:
+                       if k in ("args", "world"):
+                               continue
+                       s = SETPREFIX + k
+                       if s in world_set:
+                               continue
+                       all_added.append(SETPREFIX + k)
+               all_added.extend(added_favorites)
+               all_added.sort()
+               for a in all_added:
+                       print ">>> Recording %s in \"world\" favorites file..." % \
+                               colorize("INFORM", a)
+               if all_added:
+                       world_set.update(all_added)
                world_set.unlock()
 
        def loadResumeCommand(self, resume_data):
@@ -6230,8 +6253,7 @@ def action_build(settings, trees, mtimedb,
                                                pkglist.append(pkg)
                        else:
                                pkglist = mydepgraph.altlist()
-                       if favorites:
-                               mydepgraph.saveNomergeFavorites()
+                       mydepgraph.saveNomergeFavorites()
                        del mydepgraph
                        mergetask = MergeTask(settings, trees, myopts)
                        retval = mergetask.merge(pkglist, favorites, mtimedb)
@@ -6565,20 +6587,37 @@ def emerge_main():
        # only expand sets for actions taking package arguments
        oldargs = myfiles[:]
        if myaction not in ["search", "metadata", "sync"]:
-               for s in settings.sets:
-                       if s in myfiles:
+               newargs = []
+               for a in myfiles:
+                       if a in ("system", "world"):
+                               newargs.append(SETPREFIX+a)
+                       else:
+                               newargs.append(a)
+               myfiles = newargs
+               del newargs
+               newargs = []
+               for a in myfiles:
+                       if a.startswith(SETPREFIX):
+                               s = a[len(SETPREFIX):]
+                               if s not in settings.sets:
+                                       print "emerge: there are no sets to satisfy %s." % \
+                                               colorize("INFORM", s)
+                                       return 1
                                # TODO: check if the current setname also resolves to a package name
                                if myaction in ["unmerge", "prune", "clean", "depclean"] and not packagesets[s].supportsOperation("unmerge"):
                                        print "emerge: the given set %s does not support unmerge operations" % s
                                        return 1
-                               if not settings.sets[s].getAtoms():
+                               if not settings.setconfig.getSetAtoms(s):
                                        print "emerge: '%s' is an empty set" % s
                                else:
-                                       myfiles.extend(settings.sets[s].getAtoms())
+                                       newargs.extend(settings.setconfig.getSetAtoms(s))
                                        mysets[s] = settings.sets[s]
                                for e in settings.sets[s].errors:
                                        print e
-                               myfiles.remove(s)
+                       else:
+                               newargs.append(a)
+               myfiles = newargs
+               del newargs
                # Need to handle empty sets specially, otherwise emerge will react 
                # with the help message for empty argument lists
                if oldargs and not myfiles:
index e29c14f4331a32e13eedb6f253a6ece182f3c79f..7231934aec4ba6632a4fc5b33e7e09cc11f9853f 100644 (file)
@@ -4989,7 +4989,13 @@ def dep_wordreduce(mydeplist,mysettings,mydbapi,mode,use_cache=1):
                                deplist[mypos] = False
                        else:
                                if mode:
-                                       mydep=mydbapi.xmatch(mode,deplist[mypos])
+                                       x = mydbapi.xmatch(mode, deplist[mypos])
+                                       if mode.startswith("minimum-"):
+                                               mydep = []
+                                               if x:
+                                                       mydep.append(x)
+                                       else:
+                                               mydep = x
                                else:
                                        mydep=mydbapi.match(deplist[mypos],use_cache=use_cache)
                                if mydep!=None:
index 05fe2a767e44c37dfa46c467b24e3e1b5a51bf48..f0cc49ac27c0ed227c37570cebbfa5a15e6b8ce2 100644 (file)
@@ -14,7 +14,7 @@ from portage.exception import OperationNotPermitted, PortageException, \
 from portage.manifest import Manifest
 from portage.output import red
 from portage.util import ensure_dirs, writemsg, apply_recursive_permissions
-from portage.versions import pkgsplit, catpkgsplit, best
+from portage.versions import pkgcmp, pkgsplit, catpkgsplit, best
 
 import portage.gpg, portage.checksum
 
@@ -521,13 +521,27 @@ class portdbapi(dbapi):
                        mylist = []
                else:
                        mylist = d.keys()
+               # Always sort in ascending order here since it's handy
+               # and the result can be easily cached and reused.
+               if len(mylist) > 1:
+                       for i in xrange(len(mylist)):
+                               mylist[i] = catpkgsplit(mylist[i])[1:]
+                       mylist.sort(pkgcmp)
+                       cat = mysplit[0]
+                       for i, (pn, ver, rev) in enumerate(mylist):
+                               if rev == "r0":
+                                       cpv = cat + "/" + pn + "-" + ver
+                               else:
+                                       cpv = cat + "/" + pn + "-" + ver + "-" + rev
+                               mylist[i] = cpv
                if self.frozen and mytree is None:
                        if not (not mylist and mycp.startswith("virtual/")):
                                self.xcache["match-all"][mycp] = mylist[:]
                return mylist
 
        def freeze(self):
-               for x in ["list-visible", "bestmatch-visible", "match-visible", "match-all"]:
+               for x in "bestmatch-visible", "list-visible", "match-all", \
+                       "match-visible", "minimum-all", "minimum-visible":
                        self.xcache[x]={}
                self.frozen=1
 
@@ -550,6 +564,7 @@ class portdbapi(dbapi):
                        mydep = dep_expand(origdep, mydb=self, settings=self.mysettings)
                        mykey = dep_getkey(mydep)
 
+               myslot = dep_getslot(mydep)
                if level == "list-visible":
                        #a list of all visible packages, not called directly (just by xmatch())
                        #myval = self.visible(self.cp_list(mykey))
@@ -560,6 +575,57 @@ class portdbapi(dbapi):
                        #get all visible matches (from xmatch()), then choose the best one
 
                        myval = best(self.xmatch("match-visible", None, mydep=mydep, mykey=mykey))
+               elif level == "minimum-all":
+                       # Find the minimum matching version. This is optimized to
+                       # minimize the number of metadata accesses (improves performance
+                       # especially in cases where metadata needs to be generated).
+                       if mydep == mykey:
+                               mylist = self.cp_list(mykey)
+                       else:
+                               mylist = match_from_list(mydep, self.cp_list(mykey))
+                       myval = ""
+                       if mylist:
+                               if myslot is None:
+                                       myval = mylist[0]
+                               else:
+                                       for cpv in mylist:
+                                               try:
+                                                       if self.aux_get(cpv, ["SLOT"])[0] == myslot:
+                                                               myval = cpv
+                                                               break
+                                               except KeyError:
+                                                       pass # ebuild masked by corruption
+               elif level == "minimum-visible":
+                       # Find the minimum matching visible version. This is optimized to
+                       # minimize the number of metadata accesses (improves performance
+                       # especially in cases where metadata needs to be generated).
+                       # This does not implement LICENSE filtering since it's only
+                       # intended for use by repoman.
+                       if mydep == mykey:
+                               mylist = self.cp_list(mykey)
+                       else:
+                               mylist = match_from_list(mydep, self.cp_list(mykey))
+                       myval = ""
+                       settings = self.mysettings
+                       for cpv in mylist:
+                               try:
+                                       metadata = dict(izip(self._aux_cache_keys,
+                                               self.aux_get(cpv, self._aux_cache_keys)))
+                               except KeyError:
+                                       # ebuild masked by corruption
+                                       continue
+                               if not eapi_is_supported(metadata["EAPI"]):
+                                       continue
+                               if myslot and myslot != metadata["SLOT"]:
+                                       continue
+                               if settings.getMissingKeywords(cpv, metadata):
+                                       continue
+                               if settings.getMaskAtom(cpv, metadata):
+                                       continue
+                               if settings.getProfileMaskAtom(cpv, metadata):
+                                       continue
+                               myval = cpv
+                               break
                elif level == "bestmatch-list":
                        #dep match -- find best match but restrict search to sublist
                        #no point in calling xmatch again since we're not caching list deps
@@ -584,8 +650,7 @@ class portdbapi(dbapi):
                else:
                        print "ERROR: xmatch doesn't handle", level, "query!"
                        raise KeyError
-               myslot = dep_getslot(mydep)
-               if myslot is not None:
+               if myslot is not None and isinstance(myval, list):
                        slotmatches = []
                        for cpv in myval:
                                try:
index b757c43c47c5972d66f29f1ff27aa7e3ca033021..9f562820b4dcb59870d0854707ee96555d0ee57b 100644 (file)
@@ -10,6 +10,8 @@ DEFAULT_SETS = ["world", "system", "everything", "security"] \
        +["package_"+x for x in ["mask", "unmask", "use", "keywords"]]
 del x
 
+SETPREFIX = "@"
+
 class SetConfigError(Exception):
        pass
 
@@ -61,7 +63,7 @@ class SetConfig(SafeConfigParser):
                                try:
                                        setname = self.get(sname, "name")
                                except NoOptionError:
-                                       setname = "sets/"+sname
+                                       setname = sname
                                if hasattr(setclass, "singleBuilder"):
                                        try:
                                                self.psets[setname] = setclass.singleBuilder(optdict, self.settings, self.trees)
@@ -92,6 +94,16 @@ class SetConfig(SafeConfigParser):
                        self.aliases = shortnames
                return self.aliases
 
+       def getSetAtoms(self, setname, ignorelist=[]):
+               myset = self.getSetsWithAliases()[setname]
+               myatoms = myset.getAtoms()
+               ignorelist.append(setname)
+               for n in myset.getNonAtoms():
+                       if n[0] == SETPREFIX and n[1:] in self.aliases:
+                               if n[1:] not in ignorelist:
+                                       myatoms.update(self.getSetAtoms(n[1:]))
+               return myatoms
+
 def make_default_config(settings, trees):
        sc = SetConfig([], settings, trees)
        sc.add_section("security")
index b5ea889286028f0571f6b706ec972d0137904e3d..c22ee995363d9ae3860b94f27a1c3c4978e6ed41 100644 (file)
@@ -23,9 +23,10 @@ class PackageSet(object):
                self._loaded = False
                self._loading = False
                self.errors = []
+               self._nonatoms = set()
 
        def __contains__(self, atom):
-               return atom in self.getAtoms()
+               return atom in self.getAtoms() or atom in self._nonatoms
        
        def __iter__(self):
                for x in self.getAtoms():
@@ -44,13 +45,19 @@ class PackageSet(object):
                        self._loading = False
                return self._atoms
 
+       def getNonAtoms(self):
+               self.getAtoms()
+               return self._nonatoms
+
        def _setAtoms(self, atoms):
                atoms = map(str.strip, atoms)
+               nonatoms = set()
                for a in atoms[:]:
                        if a == "":
                                atoms.remove(a)
                        elif not isvalidatom(a):
-                               raise InvalidAtom(a)
+                               atoms.remove(a)
+                               self._nonatoms.add(a)
                self._atoms = set(atoms)
                self._updateAtomMap()
 
@@ -115,9 +122,20 @@ class EditablePackageSet(PackageSet):
 
        def update(self, atoms):
                self.getAtoms()
-               self._atoms.update(atoms)
-               self._updateAtomMap(atoms=atoms)
-               self.write()
+               modified = False
+               normal_atoms = []
+               for a in atoms:
+                       if isvalidatom(a):
+                               normal_atoms.append(a)
+                       else:
+                               modified = True
+                               self._nonatoms.add(a)
+               if normal_atoms:
+                       modified = True
+                       self._atoms.update(normal_atoms)
+                       self._updateAtomMap(atoms=normal_atoms)
+               if modified:
+                       self.write()
        
        def add(self, atom):
                self.update([atom])
index 9cbb018da042e43c132d9a0144b2fbd6a6767fb1..d300eaad20d27dc528c98c4353cd5b4965eaf540 100644 (file)
@@ -3,13 +3,14 @@
 # $Id$
 
 import os
+from itertools import chain
 
 from portage.util import grabfile, write_atomic, ensure_dirs
 from portage.const import PRIVATE_PATH, USER_CONFIG_PATH
 from portage.locks import lockfile, unlockfile
 from portage import portage_gid
 from portage.sets.base import PackageSet, EditablePackageSet
-from portage.sets import SetConfigError
+from portage.sets import SetConfigError, SETPREFIX
 from portage.env.loaders import ItemFileLoader, KeyListFileLoader
 from portage.env.validators import ValidAtomValidator
 from portage import dep_getkey, cpv_getkey
@@ -24,7 +25,7 @@ class StaticFileSet(EditablePackageSet):
                self._filename = filename
                self._mtime = None
                self.description = "Package set loaded from file %s" % self._filename
-               self.loader = ItemFileLoader(self._filename, ValidAtomValidator)
+               self.loader = ItemFileLoader(self._filename, self._validate)
 
                metadata = grabfile(self._filename + ".metadata")
                key = None
@@ -44,9 +45,13 @@ class StaticFileSet(EditablePackageSet):
                else:
                        if key != None:
                                setattr(self, key, " ".join(value))
-       
+
+       def _validate(self, atom):
+               return ValidAtomValidator(atom)
+
        def write(self):
-               write_atomic(self._filename, "\n".join(sorted(self._atoms))+"\n")
+               write_atomic(self._filename, "\n".join(sorted(
+                       chain(self._atoms, self._nonatoms)))+"\n")
        
        def load(self):
                try:
@@ -126,6 +131,11 @@ class WorldSet(StaticFileSet):
                super(WorldSet, self).__init__(os.path.join(os.sep, root, PRIVATE_PATH.lstrip(os.sep), "world"))
                self._lock = None
 
+       def _validate(self, atom):
+               if atom.startswith(SETPREFIX):
+                       return True
+               return ValidAtomValidator(atom)
+
        def _ensure_dirs(self):
                ensure_dirs(os.path.dirname(self._filename), gid=portage_gid, mode=02750, mask=02)