From: Fabian Groffen Date: Fri, 5 Oct 2007 20:35:51 +0000 (-0000) Subject: Merged from trunk 7951:7959 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=0ed89ebb1e101032f52edb23406064ae5c6bf200;p=portage.git Merged from trunk 7951:7959 | 7952 | Add set configuration framework in preparation for emerge | | genone | integration | | 7956 | first version of set integration in emerge | | genone | | | 7957 | only add short names for sets if they begin with 'sets/' | | genone | | | 7958 | Generalize the first paragraph of the MASKED PACKAGES | | zmedico | section and document binary package CHOST masking. | | 7959 | replace category sets in test configuration with user sets | | genone | | svn path=/main/branches/prefix/; revision=7961 --- diff --git a/man/emerge.1 b/man/emerge.1 index c86d1f5f6..c4ca81ba0 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -529,12 +529,11 @@ and bugs resulting from misusing masked packages drains Gentoo developer time. Please be sure you are capable of handling any problems that may ensue.\fR .LP -Masks in \fBportage\fR provide three primary functions: they allow a +Masks in \fBportage\fR have many uses: they allow a testing period where the packages can be used in live machines; they prevent the use of a package when it will fail; and they mask existing -packages that are broken or could pose a security risk. Masking can be -done by two methods: \fBpackage.mask\fR and \fBKEYWORDS\fR. Read below -to find out how to unmask in either case. Also note that if you give +packages that are broken or could pose a security risk. Read below +to find out how to unmask in various cases. Also note that if you give \fBemerge\fR an ebuild, then all forms of masking will be ignored and \fBemerge\fR will attempt to emerge the package. .TP @@ -543,6 +542,12 @@ The \fBpackage.mask\fR file primarily blocks the use of packages that cause problems or are known to have issues on different systems. It resides in \fI/usr/portage/profiles\fR. .TP +.BR CHOST +The \CHOST\fR variable in \fBmake.conf\fR(5) is used to mask binary packages +that have been built with a different \CHOST\fR setting. The only +way to unmask such a binary package is to change the \CHOST\fR environment +variable so that it exactly matches that of the binary package. +.TP .BR EAPI The \fBEAPI\fR variable in an \fBebuild\fR(5) file is used to mask packages that are not supported by the current version of portage. Packages masked by diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index ad63d5762..0b550747a 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -31,7 +31,7 @@ except ImportError: import portage del os.environ["PORTAGE_LEGACY_GLOBALS"] from portage import digraph, portdbapi -from portage.const import NEWS_LIB_PATH, CACHE_PATH, PRIVATE_PATH +from portage.const import NEWS_LIB_PATH, CACHE_PATH, PRIVATE_PATH, USER_CONFIG_PATH import emerge.help import portage.xpak, commands, errno, re, socket, time, types @@ -53,7 +53,7 @@ from portage.const import EPREFIX from portage.data import secpass from portage.util import normalize_path as normpath from portage.util import writemsg -from portage.sets import InternalPackageSet +from portage.sets import InternalPackageSet, SetConfig, make_default_config from portage.sets.profiles import PackagesSystemSet as SystemSet from portage.sets.files import WorldSet @@ -6444,6 +6444,24 @@ def emerge_main(): print colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n") break + if myaction not in ["search", "metadata", "sync"]: + setconfigpaths = ["/usr/share/portage/config/sets.conf", os.path.join(settings["PORTDIR"], "sets.conf"), \ + os.path.join(os.sep, settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "sets.conf")] + #setconfig = SetConfig(setconfigpaths, settings, trees[settings["ROOT"]]) + setconfig = make_default_config(settings, trees[settings["ROOT"]]) + del setconfigpaths + packagesets, setconfig_errors = setconfig.getSetsWithAliases() + for s in packagesets: + if s in myfiles: + # TODO: check if the current setname also resolves to a package name + if myaction in ["unmerge", "prune", "clean", "depclean"] and not packagesets[s].supportsOperation("unmerge"): + print "emerge: the given set %s does not support unmerge operations" % s + sys.exit(1) + myfiles.extend(packagesets[s].getAtoms()) + for e in packagesets[s].errors: + print e + myfiles.remove(s) + if ("--tree" in myopts) and ("--columns" in myopts): print "emerge: can't specify both of \"--tree\" and \"--columns\"." sys.exit(1) diff --git a/pym/portage/sets/__init__.py b/pym/portage/sets/__init__.py index e62a9d750..de588fb4f 100644 --- a/pym/portage/sets/__init__.py +++ b/pym/portage/sets/__init__.py @@ -3,7 +3,8 @@ # $Id$ import os -from portage import flatten +from ConfigParser import SafeConfigParser, NoOptionError +from portage import flatten, load_mod from portage.dep import isvalidatom, match_from_list, \ best_match_to_list, dep_getkey, use_reduce, paren_reduce from portage.exception import InvalidAtom @@ -26,6 +27,7 @@ class PackageSet(object): self._atommap = {} self._loaded = False self._loading = False + self.errors = [] def __contains__(self, atom): return atom in self.getAtoms() @@ -145,7 +147,6 @@ class EditablePackageSet(PackageSet): # This method must be overwritten in subclasses that should be editable raise NotImplementedError() - class InternalPackageSet(EditablePackageSet): def __init__(self, initial_atoms=None): super(InternalPackageSet, self).__init__() @@ -163,78 +164,113 @@ class InternalPackageSet(EditablePackageSet): pass +class SetConfigError(Exception): + pass + +class SetConfig(SafeConfigParser): + def __init__(self, paths, settings, trees): + SafeConfigParser.__init__(self) + self.read(paths) + self.errors = [] + self.psets = {} + self.trees = trees + self.settings = settings + self._parsed = False + def _parse(self): + if self._parsed: + return + for sname in self.sections(): + # find classname for current section, default to file based sets + if not self.has_option(sname, "class"): + classname = "portage.sets.files.StaticFileSet" + else: + classname = self.get(sname, "class") + + # try to import the specified class + try: + setclass = load_mod(classname) + except (ImportError, AttributeError): + self.errors.append("Could not import '%s' for section '%s'" % (classname, sname)) + continue + # prepare option dict for the current section + optdict = {} + for oname in self.options(sname): + optdict[oname] = self.get(sname, oname) + + # create single or multiple instances of the given class depending on configuration + if self.has_option(sname, "multiset") and self.getboolean(sname, "multiset"): + if hasattr(setclass, "multiBuilder"): + try: + self.psets.update(setclass.multiBuilder(optdict, self.settings, self.trees)) + except SetConfigError, e: + self.errors.append("Configuration error in section '%s': %s" % (sname, str(e))) + continue + else: + self.errors.append("Section '%s' is configured as multiset, but '%s' doesn't support that configuration" % (sname, classname)) + continue + else: + try: + setname = self.get(sname, "name") + except NoOptionError: + setname = "sets/"+sname + if hasattr(setclass, "singleBuilder"): + try: + self.psets[setname] = setclass.singleBuilder(optdict, self.settings, self.trees) + except SetConfigError, e: + self.errors.append("Configuration error in section '%s': %s" % (sname, str(e))) + continue + else: + self.errors.append("'%s' does not support individual set creation, section '%s' must be configured as multiset" % (classname, sname)) + continue + self._parsed = True + + def getSets(self): + self._parse() + return (self.psets, self.errors) + + def getSetsWithAliases(self): + self._parse() + shortnames = {} + for name in self.psets: + mysplit = name.split("/") + if len(mysplit) > 1 and mysplit[0] == "sets" and mysplit[-1] != "": + if mysplit[-1] in shortnames: + del shortnames[mysplit[-1]] + else: + shortnames[mysplit[-1]] = self.psets[name] + shortnames.update(self.psets) + return (shortnames, self.errors) + +def make_default_config(settings, trees): + sc = SetConfig([], settings, trees) + sc.add_section("security") + sc.set("security", "class", "portage.sets.security.NewAffectedSet") + + sc.add_section("system") + sc.set("system", "class", "portage.sets.profiles.PackagesSystemSet") + + sc.add_section("world") + sc.set("world", "class", "portage.sets.files.WorldSet") + + sc.add_section("everything") + sc.set("everything", "class", "portage.sets.dbapi.EverythingSet") -def make_default_sets(configroot, root, profile_paths, settings=None, - vdbapi=None, portdbapi=None): - from portage.sets.files import StaticFileSet, ConfigFileSet - from portage.sets.profiles import PackagesSystemSet - from portage.sets.security import NewAffectedSet - from portage.sets.dbapi import EverythingSet - from portage.const import PRIVATE_PATH, USER_CONFIG_PATH + sc.add_section("config") + sc.set("config", "class", "portage.sets.files.ConfigFileSet") + sc.set("config", "multiset", "true") - rValue = {} - worldset = StaticFileSet(os.path.join(root, PRIVATE_PATH, "world")) - worldset.description = "Set of packages that were directly installed" - rValue["world"] = worldset - for suffix in ["mask", "unmask", "keywords", "use"]: - myname = "package_"+suffix - myset = ConfigFileSet(os.path.join(configroot, USER_CONFIG_PATH.lstrip(os.sep), "package."+suffix)) - rValue[myname] = myset - rValue["system"] = PackagesSystemSet(profile_paths) - if settings != None and portdbapi != None: - rValue["security"] = NewAffectedSet(settings, vdbapi, portdbapi) - else: - rValue["security"] = InternalPackageSet() - if vdbapi != None: - rValue["everything"] = EverythingSet(vdbapi) - else: - rValue["everything"] = InternalPackageSet() - - return rValue - -def make_extra_static_sets(configroot): - from portage.sets.files import StaticFileSet - from portage.const import PRIVATE_PATH, USER_CONFIG_PATH + sc.add_section("user_sets") + sc.set("user_sets", "class", "portage.sets.files.StaticFileSet") + sc.set("user_sets", "multiset", "true") - rValue = {} - mydir = os.path.join(configroot, USER_CONFIG_PATH.lstrip(os.sep), "sets") - try: - mysets = os.listdir(mydir) - except (OSError, IOError): - return rValue - for myname in mysets: - if myname in DEFAULT_SETS: - continue - rValue[myname] = StaticFileSet(os.path.join(mydir, myname)) - return rValue - -def make_category_sets(portdbapi, settings, only_visible=True): - from portage.sets.dbapi import CategorySet - rValue = {} - for c in settings.categories: - rValue["category_%s" % c] = CategorySet(c, portdbapi, only_visible=only_visible) - return rValue + return sc # adhoc test code if __name__ == "__main__": - import portage, sys, os - from portage.sets.dbapi import CategorySet - from portage.sets.files import StaticFileSet - l = make_default_sets("/", "/", portage.settings.profiles, portage.settings, portage.db["/"]["vartree"].dbapi, portage.db["/"]["porttree"].dbapi) - l.update(make_extra_static_sets("/")) - if len(sys.argv) > 1: - for s in sys.argv[1:]: - if s.startswith("category_"): - c = s[9:] - l["category_%s" % c] = CategorySet(c, portage.db['/']['porttree'].dbapi, only_visible=False) - elif os.path.exists(s): - l[os.path.basename(s)] = StaticFileSet(s) - elif s != "*": - print "ERROR: could not create set '%s'" % s - if not "*" in sys.argv: - for n in l: - if n not in sys.argv[1:]: - del l[n] + import portage + sc = make_default_config(portage.settings, portage.db["/"]) + l, e = sc.getSets() + print l, e for x in l: print x+":" print "DESCRIPTION = %s" % l[x].getMetadata("Description") diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py index 5c16b1701..1a7a66907 100644 --- a/pym/portage/sets/dbapi.py +++ b/pym/portage/sets/dbapi.py @@ -3,7 +3,7 @@ # $Id$ from portage.versions import catsplit -from portage.sets import PackageSet +from portage.sets import PackageSet, SetConfigError class EverythingSet(PackageSet): _operations = ["merge", "unmerge"] @@ -23,6 +23,10 @@ class EverythingSet(PackageSet): else: myatoms.append(cp) self._setAtoms(myatoms) + + def singleBuilder(self, options, settings, trees): + return EverythingSet(trees["vartree"].dbapi) + singleBuilder = classmethod(singleBuilder) class CategorySet(PackageSet): _operations = ["merge", "unmerge"] @@ -46,3 +50,56 @@ class CategorySet(PackageSet): myatoms.append(cp) self._setAtoms(myatoms) + def _builderGetRepository(cls, options, repositories): + repository = options.get("repository", "porttree") + if not repository in repositories: + raise SetConfigError("invalid repository class '%s'" % repository) + return repository + _builderGetRepository = classmethod(_builderGetRepository) + + def _builderGetVisible(cls, options): + visible = options.get("only_visible", "true").lower() + if visible not in ["1", "0", "yes", "no", "true", "false", "on", "off"]: + raise SetConfigError("invalid value for only_visible: %s" % visible) + return bool(visible in ["1", "yes", "true", "on"]) + _builderGetVisible = classmethod(_builderGetVisible) + + def singleBuilder(cls, options, settings, trees): + if not "category" in options: + raise SetConfigError("no category given") + + category = options["category"] + if not category in categories: + raise SetConfigError("invalid category name '%s'" % category) + + repository = cls._builderGetRepository(options, trees.keys()) + visible = cls._builderGetVisible(options) + + return CategorySet(category, dbapi=trees[repository].dbapi, only_visible=visible) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(cls, options, settings, trees): + rValue = {} + + if "categories" in options: + categories = options["categories"].split() + invalid = set(categories).difference(settings.categories) + if invalid: + raise SetConfigError("invalid categories: %s" % ", ".join(list(invalid))) + else: + categories = settings.categories + + repository = cls._builderGetRepository(options, trees.keys()) + visible = cls._builderGetVisible(options) + name_pattern = options.get("name_pattern", "$category/*") + + if not "$category" in name_pattern and not "${category}" in name_pattern: + raise SetConfigError("name_pattern doesn't include $category placeholder") + + for cat in categories: + myset = CategorySet(cat, trees[repository].dbapi, only_visible=visible) + myname = name_pattern.replace("$category", cat) + myname = myname.replace("${category}", cat) + rValue[myname] = myset + return rValue + multiBuilder = classmethod(multiBuilder) diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index 00f24097d..ab04dc846 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -5,10 +5,10 @@ import os from portage.util import grabfile, write_atomic, ensure_dirs -from portage.const import PRIVATE_PATH +from portage.const import PRIVATE_PATH, USER_CONFIG_PATH from portage.locks import lockfile, unlockfile from portage import portage_gid -from portage.sets import PackageSet, EditablePackageSet +from portage.sets import PackageSet, EditablePackageSet, SetConfigError from portage.env.loaders import ItemFileLoader, KeyListFileLoader from portage.env.validators import ValidAtomValidator @@ -52,6 +52,9 @@ class StaticFileSet(EditablePackageSet): if (not self._loaded or self._mtime != mtime): try: data, errors = self.loader.load() + for fname in errors: + for e in errors[fname]: + self.errors.append(fname+": "+e) except EnvironmentError, e: if e.errno != errno.ENOENT: raise @@ -59,6 +62,26 @@ class StaticFileSet(EditablePackageSet): data = {} self._setAtoms(data.keys()) self._mtime = mtime + + def singleBuilder(self, options, settings, trees): + if not "filename" in options: + raise SetConfigError("no filename specified") + return ConfigFileSet(options[filename]) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(self, options, settings, trees): + rValue = {} + directory = options.get("directory", os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH.lstrip(os.sep), "sets")) + name_pattern = options.get("name_pattern", "sets/$name") + if not "$name" in name_pattern and not "${name}" in name_pattern: + raise SetConfigError("name_pattern doesn't include $name placeholder") + if os.path.isdir(directory): + for filename in os.listdir(directory): + myname = name_pattern.replace("$name", filename) + myname = myname.replace("${name}", filename) + rValue[myname] = StaticFileSet(os.path.join(directory, filename)) + return rValue + multiBuilder = classmethod(multiBuilder) class ConfigFileSet(PackageSet): def __init__(self, filename): @@ -70,12 +93,31 @@ class ConfigFileSet(PackageSet): def load(self): data, errors = self.loader.load() self._setAtoms(data.keys()) + + def singleBuilder(self, options, settings, trees): + if not "filename" in options: + raise SetConfigError("no filename specified") + return ConfigFileSet(options[filename]) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(self, options, settings, trees): + rValue = {} + directory = options.get("directory", os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH.lstrip(os.sep))) + name_pattern = options.get("name_pattern", "sets/package_$suffix") + if not "$suffix" in name_pattern and not "${suffix}" in name_pattern: + raise SetConfigError("name_pattern doesn't include $suffix placeholder") + for suffix in ["keywords", "use", "mask", "unmask"]: + myname = name_pattern.replace("$suffix", suffix) + myname = myname.replace("${suffix}", suffix) + rValue[myname] = ConfigFileSet(os.path.join(directory, "package."+suffix)) + return rValue + multiBuilder = classmethod(multiBuilder) class WorldSet(StaticFileSet): description = "Set of packages that were directly installed by the user" def __init__(self, root): - super(WorldSet, self).__init__(os.path.join(os.sep, root, PRIVATE_PATH, "world")) + super(WorldSet, self).__init__(os.path.join(os.sep, root, PRIVATE_PATH.lstrip(os.sep), "world")) self._lock = None def _ensure_dirs(self): @@ -88,3 +130,7 @@ class WorldSet(StaticFileSet): def unlock(self): unlockfile(self._lock) self._lock = None + + def singleBuilder(self, options, settings, trees): + return WorldSet(settings["ROOT"]) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/sets/profiles.py b/pym/portage/sets/profiles.py index 7dbc35e36..6d629fc67 100644 --- a/pym/portage/sets/profiles.py +++ b/pym/portage/sets/profiles.py @@ -18,3 +18,7 @@ class PackagesSystemSet(PackageSet): mylist = [grabfile_package(os.path.join(x, "packages")) for x in self._profile_paths] mylist = stack_lists(mylist, incremental=1) self._setAtoms([x[1:] for x in mylist if x[0] == "*"]) + + def singleBuilder(self, options, settings, trees): + return PackagesSystemSet(settings.profiles) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/sets/security.py b/pym/portage/sets/security.py index 6c463df9a..576009364 100644 --- a/pym/portage/sets/security.py +++ b/pym/portage/sets/security.py @@ -14,12 +14,13 @@ class SecuritySet(PackageSet): description = "package set that includes all packages possibly affected by a GLSA" - def __init__(self, settings, vardbapi, portdbapi): + def __init__(self, settings, vardbapi, portdbapi, least_change=True): super(SecuritySet, self).__init__() self._settings = settings self._vardbapi = vardbapi self._portdbapi = portdbapi self._checkfile = os.path.join(os.sep, self._settings["ROOT"], CACHE_PATH.lstrip(os.sep), "glsa") + self._least_change = least_change def getGlsaList(self, skip_applied): glsaindexlist = glsa.get_glsa_list(self._settings) @@ -37,7 +38,7 @@ class SecuritySet(PackageSet): myglsa = glsa.Glsa(glsaid, self._settings, self._vardbapi, self._portdbapi) #print glsaid, myglsa.isVulnerable(), myglsa.isApplied(), myglsa.getMergeList() if self.useGlsa(myglsa): - atomlist += myglsa.getMergeList(least_change=False) + atomlist += myglsa.getMergeList(least_change=self._least_change) self._setAtoms(atomlist) def useGlsa(self, myglsa): @@ -52,6 +53,15 @@ class SecuritySet(PackageSet): applied_list.append(glsaid) write_atomic(self._checkfile, "\n".join(applied_list)) + def singleBuilder(cls, options, settings, trees): + if "use_emerge_resoler" in options \ + and options.get("use_emerge_resolver").lower() in ["1", "yes", "true", "on"]: + least_change = False + else: + least_change = True + return cls(settings, trees["vartree"].dbapi, trees["porttree"].dbapi, least_change=least_change) + singleBuilder = classmethod(singleBuilder) + class NewGlsaSet(SecuritySet): _skip_applied = True description = "Package set that includes all packages possibly affected by an unapplied GLSA" diff --git a/pym/portage/sets/shell.py b/pym/portage/sets/shell.py index 7bfaeaecb..d61c116ce 100644 --- a/pym/portage/sets/shell.py +++ b/pym/portage/sets/shell.py @@ -5,7 +5,7 @@ import subprocess import os -from portage.sets import PackageSet +from portage.sets import PackageSet, SetConfigError class CommandOutputSet(PackageSet): """This class creates a PackageSet from the output of a shell command. @@ -34,3 +34,7 @@ class CommandOutputSet(PackageSet): text = pipe.stdout.read() self._setAtoms(text.split("\n")) + def singleBuilder(self, options, settings, trees): + if not command in options: + raise SetConfigError("no command specified") + return CommandOutputSet(options["command"])