From: Fabian Groffen Date: Tue, 4 Dec 2007 20:09:57 +0000 (-0000) Subject: Merged from trunk 8820:8841 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=15c682eab60237d9471060302bb5039727f5ac51;p=portage.git Merged from trunk 8820:8841 | 8821 | implement the final part of FEATURES=preserved-libs and | | genone | remove previously preserved libs that don't have any | | | consumers left. Also fix the notice if preserved libs are | | | found to use the preserved-rebuild package set instead of | | | revdep-rebuild. | | 8822 | Bug #201082 - Don't call config.load_infodir() prior to | | zmedico | pkg_postinst() because it's not needed and the ebuild path | | | that's passed in for postinst is may be from the portage | | | tree, which causes load_infodir() to discard critical | | | metadata such as CATEGORY without realoading it. | | 8824 | revert accidential commit | | genone | | | 8825 | update TODO list | | genone | | | 8826 | Make config.load_infodir() more fault tolerant for cases | | zmedico | like bug #201082. | | 8828 | Bug #201090 - When portage creates a temp PORTAGE_BIN_PATH, | | zmedico | put it inside PORTAGE_TMPDIR since, unlike /tmp, it can't be | | | mounted with the "noexec" option. | | 8830 | * Don't load the CATEGORY in config.load_infodir() since | | zmedico | this value is critical for doebuild() operation and we | | | already know the category. * Always call config.setcpv() | | | prior to config.load_infodir() so that the CATEGORY is | | | always known. | | 8831 | Fix typo in config.load_infodir() return value logic. | | zmedico | | | 8833 | Synchronize the conditional that triggers "${T}"/environment | | zmedico | generation with the conditional that triggers distcc and | | | ccache FEATURES setup. | | 8835 | Remove redundant config.load_infodir() calls from | | zmedico | portage.pkgmerge() and dblink.unmerge() since it's already | | | called in doebuild.() | | 8837 | make sure we only remove files that were previously | | genone | preserved | | 8838 | enable FEATURES=preserve-libs globally for extended testing | | genone | | | 8839 | Fix setconfig loader to use config files instead of | | genone | hardcoded fallbacks | | 8840 | perform sanity checks for set configuration even if myaction | | genone | is None | | 8841 | fix another typo | | genone | | svn path=/main/branches/prefix/; revision=8842 --- diff --git a/TODO b/TODO index b3123f1fe..1704a0d27 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,8 @@ TODO for portage-2.2 release: ============================= -- package set integration in emerge (genone) +x package set integration in emerge (genone) x user interface for preserve-libs FEATURE (genone, fuzzyray?) -- resolve the eselect-news situation (genone) +x resolve the eselect-news situation (genone) +- package set documentation (genone) (- denotes open/incomplete tasks, x denotes completed tasks) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index d5964a568..cff3591aa 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1496,9 +1496,8 @@ preprocess_ebuild_env() { # === === === === === functions end, main part begins === === === === === # === === === === === === === === === === === === === === === === === === -if [[ ${EBUILD_SH_ARGS} != "depend" ]] && [[ ${EBUILD_SH_ARGS} != "clean" ]] && [[ ${EBUILD_SH_ARGS} != "setup" ]]; then - cd ${PORTAGE_TMPDIR} &> /dev/null - cd ${BUILD_PREFIX} &> /dev/null +if [ -n "${EBUILD_SH_ARGS}" ] && \ + ! hasq ${EBUILD_SH_ARGS} clean depend help info nofetch ; then if [ "$(id -nu)" == "portage" ] ; then export USER=portage @@ -1799,7 +1798,7 @@ fi # Save the env only for relevant phases. if [ -n "${EBUILD_SH_ARGS}" ] && \ - ! hasq ${EBUILD_SH_ARGS} clean help info nofetch ; then + ! hasq ${EBUILD_SH_ARGS} clean depend help info nofetch ; then # Save current environment and touch a success file. (echo for success) umask 002 save_ebuild_env | filter_readonly_variables > "${T}/environment" diff --git a/bin/emaint b/bin/emaint index 297050c5f..157bea805 100755 --- a/bin/emaint +++ b/bin/emaint @@ -23,8 +23,8 @@ class WorldHandler(object): self.not_installed = [] self.invalid_category = [] self.okay = [] - from portage.sets import make_default_config - setconfig = make_default_config(portage.settings, + from portage.sets import load_default_config + setconfig = load_default_config(portage.settings, portage.db[portage.settings["ROOT"]]) self._sets = setconfig.getSetsWithAliases() diff --git a/cnf/make.globals b/cnf/make.globals index aa5e025f9..8ea92f189 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -35,6 +35,9 @@ RESUMECOMMAND="wget -c -t 5 -T 60 --passive-ftp -O \${DISTDIR}/\${FILE} \${URI}" # Default user options FEATURES="distlocks metadata-transfer sandbox sfperms strict unmerge-orphans userfetch" +# Enable for global testing +FEATURES="${FEATURES} preserve-libs" + # Default chunksize for binhost comms PORTAGE_BINHOST_CHUNKSIZE="3000" diff --git a/cnf/sets.conf b/cnf/sets.conf index 6e9bb4259..47ff265f9 100644 --- a/cnf/sets.conf +++ b/cnf/sets.conf @@ -32,3 +32,8 @@ class = portage.sets.dbapi.EverythingSet class = portage.sets.files.StaticFileSet multiset = true directory = @DOMAIN_PREFIX@/etc/portage/sets + +# Set to rebuild all packages that need a preserved lib that only remains due +# to FEATURES=preserved-libs +[preserved-rebuild] +class = portage.sets.dbapi.PreservedConsumerSet diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 3d8e4c4c2..06ba9acaa 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -53,7 +53,7 @@ from portage.const import EPREFIX, BPREFIX from portage.data import secpass from portage.util import normalize_path as normpath from portage.util import writemsg -from portage.sets import make_default_config, SETPREFIX +from portage.sets import load_default_config, SETPREFIX from portage.sets.base import InternalPackageSet from itertools import chain, izip @@ -587,7 +587,7 @@ class RootConfig(object): self.settings = trees["vartree"].settings self.root = self.settings["ROOT"] self.setconfig = setconfig - self.sets = self.setconfig.getSetsWithAliases() + self.sets = self.setconfig.getSets() def create_world_atom(pkg_key, metadata, args_set, root_config): """Create a new atom for the world file if one does not exist. If the @@ -5097,7 +5097,7 @@ def post_emerge(trees, mtimedb, retval): print colorize("WARN", ">>>") + " package: %s" % cpv for f in plibdata[cpv]: print colorize("WARN", " * ") + " - %s" % f - print "Use " + colorize("GOOD", "revdep-rebuild") + " to rebuild packages using these libraries" + print "Use " + colorize("GOOD", "emerge @preserved-rebuild") + " to rebuild packages using these libraries" print "and then remerge the packages listed above." sys.exit(retval) @@ -6762,7 +6762,7 @@ def load_emerge_config(trees=None): for root, root_trees in trees.iteritems(): settings = root_trees["vartree"].settings - setconfig = make_default_config(settings, root_trees) + setconfig = load_default_config(settings, root_trees) root_trees["root_config"] = RootConfig(root_trees, setconfig) settings = trees["/"]["vartree"].settings @@ -6984,10 +6984,16 @@ def emerge_main(): # only expand sets for actions taking package arguments oldargs = myfiles[:] - if myaction in ("clean", "config", "depclean", "info", "prune", "unmerge"): + if myaction in ("clean", "config", "depclean", "info", "prune", "unmerge", None): root_config = trees[settings["ROOT"]]["root_config"] setconfig = root_config.setconfig sets = root_config.sets + # emerge relies on the existance of sets with names "world" and "system" + for s in ("world", "system"): + if s not in sets: + print "emerge: incomplete set configuration, no \"%s\" set defined" % s + print " sets defined: %s" % ", ".join(sets) + return 1 newargs = [] for a in myfiles: if a in ("system", "world"): @@ -7004,15 +7010,16 @@ def emerge_main(): print "emerge: there are no sets to satisfy %s." % \ colorize("INFORM", s) return 1 - # TODO: check if the current setname also resolves to a package name if myaction in ["unmerge", "prune", "clean", "depclean"] and \ - not sets[s].supportsOperation("unmerge"): + not sets[s].supportsOperation("unmerge"): print "emerge: the given set %s does not support unmerge operations" % s return 1 if not setconfig.getSetAtoms(s): print "emerge: '%s' is an empty set" % s - else: + elif myaction != None: newargs.extend(setconfig.getSetAtoms(s)) + else: + newargs.append(SETPREFIX+s) for e in sets[s].errors: print e else: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 7b68e085d..a938699b3 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1787,14 +1787,19 @@ class config(object): def load_infodir(self,infodir): self.modifying() - if "pkg" in self.configdict: + backup_pkg_metadata = dict(self.configdict["pkg"].iteritems()) + if "pkg" in self.configdict and \ + "CATEGORY" in self.configdict["pkg"]: self.configdict["pkg"].clear() + self.configdict["pkg"]["CATEGORY"] = \ + backup_pkg_metadata["CATEGORY"] else: - writemsg("No pkg setup for settings instance?\n", - noiselevel=-1) - sys.exit(17) + raise portage.exception.PortageException( + "No pkg setup for settings instance?") - if os.path.exists(infodir): + retval = 0 + found_category_file = False + if os.path.isdir(infodir): if os.path.exists(infodir+"/environment"): self.configdict["pkg"]["PORT_ENV_FILE"] = infodir+"/environment" @@ -1805,6 +1810,9 @@ class config(object): # FEATURES from the build host shouldn't be interpreted as # FEATURES on the client system. continue + if filename == "CATEGORY": + found_category_file = True + continue if myre.match(filename): try: file_path = os.path.join(infodir, filename) @@ -1821,22 +1829,23 @@ class config(object): else: self.configdict["pkg"][filename] = mydata self.configdict["env"][filename] = mydata - # CATEGORY is important because it's used in doebuild - # to infer the cpv. If it's corrupted, it leads to - # strange errors later on, so we'll validate it and - # print a warning if necessary. - if filename == "CATEGORY": - matchobj = re.match("[-a-zA-Z0-9_.+]+", mydata) - if not matchobj or matchobj.start() != 0 or \ - matchobj.end() != len(mydata): - writemsg("!!! CATEGORY file is corrupt: %s\n" % \ - os.path.join(infodir, filename), noiselevel=-1) except (OSError, IOError): writemsg("!!! Unable to read file: %s\n" % infodir+"/"+filename, noiselevel=-1) pass - return 1 - return 0 + retval = 1 + + # Missing or corrupt CATEGORY will cause problems for + # doebuild(), which uses it to infer the cpv. We already + # know the category, so there's no need to trust this + # file. Show a warning if the file is missing though, + # because it's required (especially for binary packages). + if not found_category_file: + writemsg("!!! CATEGORY file is missing: %s\n" % \ + os.path.join(infodir, "CATEGORY"), noiselevel=-1) + self.configdict["pkg"].update(backup_pkg_metadata) + retval = 0 + return retval def setcpv(self, mycpv, use_cache=1, mydb=None): """ @@ -4528,7 +4537,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, noiselevel=-1) return phase_retval elif mydo == "postinst": - mysettings.load_infodir(mysettings["O"]) phase_retval = spawn( _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings, debug=debug, free=1, logfile=logfile) @@ -5818,7 +5826,7 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes gid=portage_gid, mode=0755) writemsg_stdout(">>> Extracting info\n") xptbz2.unpackinfo(infloc) - mysettings.load_infodir(infloc) + mysettings.setcpv(mycat + "/" + mypkg, mydb=mydbapi) # Store the md5sum in the vdb. fp = open(os.path.join(infloc, "BINPKGMD5"), "w") fp.write(str(portage.checksum.perform_md5(mytbz2))+"\n") diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 1253fa58b..81c120ef5 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1083,7 +1083,7 @@ class dblink(object): write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n") break - self.settings.load_infodir(self.dbdir) + self.settings.setcpv(self.mycpv, mydb=self.vartree.dbapi) if myebuildpath: try: doebuild_environment(myebuildpath, "prerm", self.myroot, @@ -1125,7 +1125,8 @@ class dblink(object): self._unmerge_pkgfiles(pkgfiles, others_in_slot) # Remove the registration of preserved libs for this pkg instance - self.vartree.dbapi.plib_registry.unregister(self.mycpv, self.settings["SLOT"], self.settings["COUNTER"]) + plib_registry = self.vartree.dbapi.plib_registry + plib_registry.unregister(self.mycpv, self.settings["SLOT"], self.settings["COUNTER"]) if myebuildpath: ebuild_phase = "postrm" @@ -1140,7 +1141,46 @@ class dblink(object): # regenerate reverse NEEDED map self.vartree.dbapi.libmap.update() - + + # remove preserved libraries that don't have any consumers left + # FIXME: this code is quite ugly and can likely be optimized in several ways + plib_dict = plib_registry.getPreservedLibs() + for cpv in plib_dict: + keeplist = [] + plib_dict[cpv].sort() + for f in plib_dict[cpv]: + if not os.path.exists(f) or os.path.realpath(f) in keeplist: + continue + unlink_list = [] + while os.path.islink(f): + if os.path.basename(f) in self.vartree.dbapi.libmap.get(): + unlink_list = [] + keeplist.append(os.path.realpath(f)) + break + else: + unlink_list.append(f) + # only follow symlinks if the target is also a preserved lib object + if os.readlink(f) in plib_dict[cpv]: + f = os.readlink(f) + else: + break + if not os.path.islink(f) and not os.path.basename(f) in self.vartree.dbapi.libmap.get(): + unlink_list.append(f) + for obj in unlink_list: + try: + if os.path.islink(f): + obj_type = "sym" + else: + obj_type = "obj" + os.unlink(obj) + writemsg_stdout("<<< !needed %s %s\n" % (obj_type, obj)) + except OSError, e: + if e.errno == errno.ENOENT: + pass + else: + raise e + plib_registry.pruneNonExisting() + finally: if builddir_lock: try: @@ -2371,7 +2411,10 @@ class dblink(object): base_path_orig = os.path.dirname(settings["PORTAGE_BIN_PATH"]) from tempfile import mkdtemp import shutil - base_path_tmp = mkdtemp() + # Make the temp directory inside PORTAGE_TMPDIR since, unlike + # /tmp, it can't be mounted with the "noexec" option. + base_path_tmp = mkdtemp("", "._portage_reinstall_.", + settings["PORTAGE_TMPDIR"]) from portage.process import atexit_register atexit_register(shutil.rmtree, base_path_tmp) dir_perms = 0755 diff --git a/pym/portage/sets/__init__.py b/pym/portage/sets/__init__.py index 940aa1020..2d36f4814 100644 --- a/pym/portage/sets/__init__.py +++ b/pym/portage/sets/__init__.py @@ -5,10 +5,7 @@ import os from ConfigParser import SafeConfigParser, NoOptionError from portage import load_mod - -DEFAULT_SETS = ["world", "system", "everything", "security"] \ - +["package_"+x for x in ["mask", "unmask", "use", "keywords"]] -del x +from portage.const import USER_CONFIG_PATH, GLOBAL_CONFIG_PATH SETPREFIX = "@" @@ -79,65 +76,31 @@ class SetConfig(SafeConfigParser): self._parse() return self.psets - def getSetsWithAliases(self): - self._parse() - if not self.aliases: - shortnames = {} - for name in self.psets: - mysplit = name.split("/") - if len(mysplit) > 1 and mysplit[0] == "sets" and mysplit[-1] != "": - if mysplit[-1] in shortnames: - del shortnames[mysplit[-1]] - else: - shortnames[mysplit[-1]] = self.psets[name] - shortnames.update(self.psets) - self.aliases = shortnames - return self.aliases - def getSetAtoms(self, setname, ignorelist=None): - myset = self.getSetsWithAliases()[setname] + myset = self.getSets()[setname] myatoms = myset.getAtoms() if ignorelist is None: ignorelist = set() ignorelist.add(setname) for n in myset.getNonAtoms(): - if n[0] == SETPREFIX and n[1:] in self.aliases: + if n[0] == SETPREFIX and n[1:] in self.psets: if n[1:] not in ignorelist: myatoms.update(self.getSetAtoms(n[1:], ignorelist=ignorelist)) return myatoms -def make_default_config(settings, trees): - sc = SetConfig([], settings, trees) - sc.add_section("security") - sc.set("security", "class", "portage.sets.security.NewAffectedSet") - - sc.add_section("system") - sc.set("system", "class", "portage.sets.profiles.PackagesSystemSet") - - sc.add_section("world") - sc.set("world", "class", "portage.sets.files.WorldSet") - - sc.add_section("everything") - sc.set("everything", "class", "portage.sets.dbapi.EverythingSet") - - sc.add_section("config") - sc.set("config", "class", "portage.sets.files.ConfigFileSet") - sc.set("config", "multiset", "true") - - sc.add_section("user-sets") - sc.set("user-sets", "class", "portage.sets.files.StaticFileSet") - sc.set("user-sets", "multiset", "true") - - sc.add_section("rebuild-needed") - sc.set("rebuild-needed", "class", "portage.sets.dbapi.MissingLibraryConsumerSet") - - return sc +def load_default_config(settings, trees): + setconfigpaths = [os.path.join(GLOBAL_CONFIG_PATH, "sets.conf")] + setconfigpaths.append(os.path.join(settings["PORTDIR"], "sets.conf")) + setconfigpaths += [os.path.join(x, "sets.conf") for x in settings["PORDIR_OVERLAY"].split()] + setconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], + USER_CONFIG_PATH.lstrip(os.path.sep), "sets.conf")) + return SetConfig(setconfigpaths, settings, trees) # adhoc test code if __name__ == "__main__": import portage - sc = make_default_config(portage.settings, portage.db["/"]) + sc = load_default_config(portage.settings, portage.db["/"]) l, e = sc.getSets() for x in l: print x+":"