From 03fca004cbff4f8eb1fee9016b6fccbec3f3856b Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sun, 3 Aug 2008 19:17:54 +0000 Subject: [PATCH] Merged from trunk 11302:11320 | 11303 | Bug #233487 - Fix --fetch-all-uri breakage: * Make | | zmedico | --fetch-all-uri imply --fetchonly so that all the | | | --fetchonly conditionals apply to --fetch-all-uri as well. * | | | Add an EbuildFetcher.fetchall attribute, and also add the | | | required support to portage.doebuild() for a "fetchall" | | | phase that EbuildFetcher can invoke in the background via | | | the ebuild(1) command. | | 11304 | Bug #229033 - Use `svn propget -R svn:keywords` to detect | | zmedico | which will have changed headers that require an additional | | | manifest commit. Thanks to Fabian Groffen for this patch | | | which I've made some modifications to: * For clarity, use | | | separate "no_expansion" variable for cvs bin blobs. * Fix | | | svn keyword parsing to properly handle multiple keywords | | | delimited by newlines. | | 11305 | Remove the list of FEATURES descriptions from | | zmedico | make.conf.example and replace it with a reference to the | | | make.conf(5) man page. | | 11306 | Fix new rejects in cnf/make.conf.*.diff. | | zmedico | | | 11307 | Optimize `emaint` --fix binhost so that it the Packages file | | zmedico | isn't re-read and re-written for each package that's | | | updated. Instead, hold a lock for the whole time the command | | | is running and just update it once. | | 11308 | Make InheritSet more like DowngradeSet, and check INHERITED | | zmedico | from the highest visible ebuild in the portage tree rather | | | than from the installed package itself. | | 11309 | Bug #233252 - Handle InvalidAtom exceptions that can be | | zmedico | raised from depgraph._add_dep() calls inside | | | _add_pkg_deps(). | | 11310 | Add DowngradeSet and OwnerSet to __all__. | | zmedico | | | 11311 | Update the Parallel Scheduling section to describe the | | zmedico | current algorithm. | | 11312 | Make EbuildFetcher pass a copy of os.environ to the | | zmedico | ebuild(1) subprocess so that any incremental variables have | | | been overridden are correctly considered by the config | | | instance in the subproccess. | | 11314 | Add debug support to PackagesSystemSet since robbat2 is | | zmedico | reporting an empty set for some unknown reason. | | 11315 | * Add INHERITED to vardbapi._aux_cache_keys since | | zmedico | Package.metadata_keys also includes this one now. * Fix | | | pull_me calculation inside vardbapi.aux_get() so that it | | | correctly subtracts _mtime_ from the list of keys to pull. | | 11316 | In the file collision elog message, only say 'NOT merged' | | zmedico | when collision-protect is enabled. Thanks to dberkholz for | | | reporting. | | 11317 | Add a new RestrictSet which contains all packages that match | | zmedico | specified RESTRICT values. This could be useful for | | | implementing something like a @live-rebuild based on | | | RESTRICT metadata, as suggested in bug #233589. | | 11318 | Document the fact that use of @installed/EverythingSet makes | | zmedico | it impossible for emerge to solve blockers by automatic | | | uninstallation of blocked packages. | | 11319 | Handle missing sets, to avoid KeyError raised from | | zmedico | unmerge(). | | 11320 | Add a more generic VariableSet to replace the InheritSet and | | zmedico | RestrictSet classes. Thanks to Arfrever Frehtes Taifersar | | | Arahesis for this patch from bug #233589 (with some minor | | | adjustments from me). | svn path=/main/branches/prefix/; revision=11322 --- bin/emaint | 60 +++++++++--- bin/repoman | 40 ++++++-- cnf/make.conf | 60 +----------- cnf/make.conf.sparc-fbsd.diff | 10 -- cnf/make.conf.x86-fbsd.diff | 10 -- cnf/sets.conf | 10 +- doc/config/sets.docbook | 34 ++++--- .../task_scheduling.docbook | 8 +- man/make.conf.5 | 8 +- pym/_emerge/__init__.py | 67 +++++++++---- pym/portage/__init__.py | 7 +- pym/portage/cache/mappings.py | 3 + pym/portage/dbapi/bintree.py | 50 ++++++---- pym/portage/dbapi/vartree.py | 16 ++-- pym/portage/sets/dbapi.py | 94 ++++++++++++------- pym/portage/sets/profiles.py | 24 ++++- 16 files changed, 305 insertions(+), 196 deletions(-) diff --git a/bin/emaint b/bin/emaint index ab8e2a58e..084c8c327 100755 --- a/bin/emaint +++ b/bin/emaint @@ -114,8 +114,7 @@ class BinhostHandler(object): self._bintree = portage.db[myroot]["bintree"] self._bintree.populate() self._pkgindex_file = os.path.join(self._bintree.pkgdir, "Packages") - from portage import getbinpkg - self._pkgindex = getbinpkg.PackageIndex() + self._pkgindex = self._bintree._new_pkgindex() f = open(self._pkgindex_file, 'r') try: self._pkgindex.read(f) @@ -151,7 +150,7 @@ class BinhostHandler(object): cpv_all = self._bintree.dbapi.cpv_all() cpv_all.sort() missing = [] - maxval = len(cpv_all) + maxval = 0 if onProgress: onProgress(maxval, 0) pkgindex = self._pkgindex @@ -159,35 +158,61 @@ class BinhostHandler(object): metadata = {} for d in pkgindex.packages: metadata[d["CPV"]] = d + for i, cpv in enumerate(cpv_all): d = metadata.get(cpv) if not d or "MD5" not in d: - bintree.inject(cpv) - if onProgress: - onProgress(maxval, i+1) + missing.append(cpv) + stale = set(metadata).difference(cpv_all) - if stale: + if missing or stale: from portage import locks pkgindex_lock = locks.lockfile( self._pkgindex_file, wantnewlockfile=1) try: - from portage import getbinpkg - pkgindex = getbinpkg.PackageIndex() + # Repopulate with lock held. + bintree._populate() + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + + pkgindex = bintree._new_pkgindex() self._pkgindex = pkgindex f = open(self._pkgindex_file, 'r') try: self._pkgindex.read(f) finally: f.close() - from portage.dbapi.bintree import binarytree - self._bintree = binarytree(bintree.root, bintree.pkgdir, - settings=bintree.settings) - del bintree - portage.db[self._bintree.root]["bintree"] = self._bintree - self._bintree._populate() + + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + # Recount missing packages, with lock held. + del missing[:] + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or "MD5" not in d: + missing.append(cpv) + + maxval = len(missing) + for i, cpv in enumerate(missing): + try: + metadata[cpv] = bintree._pkgindex_entry(cpv) + except portage.exception.InvalidDependString: + writemsg("!!! Invalid binary package: '%s'\n" % \ + bintree.getname(cpv), noiselevel=-1) + + if onProgress: + onProgress(maxval, i+1) + for cpv in set(metadata).difference( self._bintree.dbapi.cpv_all()): del metadata[cpv] + + # We've updated the pkgindex, so set it to + # repopulate when necessary. + bintree.populated = False + del pkgindex.packages[:] pkgindex.packages.extend(metadata.itervalues()) from portage.util import atomic_ofstream @@ -198,6 +223,11 @@ class BinhostHandler(object): f.close() finally: locks.unlockfile(pkgindex_lock) + + if onProgress: + if maxval == 0: + maxval = 1 + onProgress(maxval, maxval) return None class MoveHandler(object): diff --git a/bin/repoman b/bin/repoman index 999271b78..148222d4e 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1745,19 +1745,28 @@ else: mynew=portage.cvstree.findnew(mycvstree,recursive=1,basedir="./") myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./") bin_blob_pattern = re.compile("^-kb$") - bin_blobs = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern, + no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern, recursive=1, basedir="./")) if vcs == "svn": svnstatus = os.popen("svn status").readlines() mychanged = [ elem.rstrip()[7:] for elem in svnstatus if elem.startswith("M") ] - for manifest in [ file for file in mychanged if '/Manifest' in file ]: - mychanged.remove(manifest) - mynew = [ elem.rstrip()[7:] for elem in svnstatus if elem.startswith("A") ] + mynew = [ elem.rstrip()[7:] for elem in svnstatus if elem.startswith("A") ] myremoved = [ elem.rstrip()[7:] for elem in svnstatus if elem.startswith("D") ] - # no idea how to detect binaries in SVN - bin_blobs = [] + # in contrast to CVS, SVN expands nothing by default. + # bin_blobs historically + # were just there to see what files need to be checked for + # keyword expansion, which is exactly what we do here, so + # slightly change the semantic meaning of "bin_blob"... In the + # future we could store which keyword is expanded. + props = os.popen("svn propget -R svn:keywords").readlines() + + # For files with multiple props set, props are delimited by newlines, + # so exclude lines that don't contain " - " since each of those lines + # only a contain props for a file listed on a previous line. + expansion = set(prop.split(" - ")[0] \ + for prop in props if " - " in prop) if vcs: if not (mychanged or mynew or myremoved): @@ -1777,8 +1786,17 @@ else: headerstring = "'\$(Header|Id)" headerstring += ".*\$'" for myfile in myupdates: - if myfile in bin_blobs: - continue + + # for CVS, no_expansion contains files that are excluded from expansion + if vcs == "cvs": + if myfile in no_expansion: + continue + + # for SVN, expansion contains files that are included in expansion + elif vcs == "svn": + if myfile not in expansion: + continue + myout = commands.getstatusoutput("egrep -q "+headerstring+" "+myfile) if myout[0] == 0: myheaders.append(myfile) @@ -1833,6 +1851,8 @@ else: if myupdates or myremoved: myfiles = myupdates + myremoved + if not myheaders and "sign" not in repoman_settings.features: + myfiles += mymanifests fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") mymsg = os.fdopen(fd, "w") mymsg.write(commitmessage) @@ -1919,8 +1939,8 @@ else: write_atomic(x, "".join(mylines)) manifest_commit_required = True - if myheaders or myupdates or myremoved or mynew: - myfiles=myheaders+myupdates+myremoved+mynew + if myupdates or myremoved or mynew: + myfiles=myupdates+myremoved+mynew for x in range(len(myfiles)-1, -1, -1): if myfiles[x].count("/") < 4-repolevel: del myfiles[x] diff --git a/cnf/make.conf b/cnf/make.conf index 629ef52ca..0637e0492 100644 --- a/cnf/make.conf +++ b/cnf/make.conf @@ -245,61 +245,11 @@ # packages. Do not use it unless absolutely necessary! #AUTOCLEAN="yes" # -# FEATURES are settings that affect the functionality of portage. Most of -# these settings are for developer use, but some are available to non- -# developers as well. -# -# 'buildpkg' causes binary packages to be created of all packages that -# are being merged. -# 'buildsyspkg' only build binary packages for system packages. -# 'ccache' enable support for the dev-util/ccache package, which can -# noticably decrease the time needed to remerge previously built -# packages. -# 'collision-protect' -# prevents packages from overwriting files that are owned by -# another package or by no package at all. -# 'distcc' enables distcc support via CC. -# 'distlocks' enables distfiles locking using fcntl or hardlinks. This -# is enabled by default. Tools exist to help clean the locks -# after crashes: /usr/lib/portage/bin/clean_locks. -# 'installsources' -# Install source code into /usr/src/debug/${CATEGORY}/${PF} -# (also see 'splitdebug'). This feature works only if debugedit -# is installed and CFLAGS is set to include debug information -# (such as with the -ggdb flag). -# 'test' causes ebuilds to perform testing phases if they are capable -# of it. Some packages support this automatically via makefiles. -# 'metadata-transfer' -# automatically perform a metadata transfer when `emerge --sync` -# is run. -# 'noauto' causes ebuild to perform only the action requested and -# not any other required actions like clean or unpack -- for -# debugging purposes only. -# 'nostrip' prevents the stripping of binaries. -# 'notitles' disables xterm titlebar updates (which contain status info). -# 'parallel-fetch' -# do fetching in parallel to compilation -# 'preserve-libs' -# keep libraries around that would normally removed by an upgrade, -# but are still needed by other packages -# 'sandbox' enables sandboxing when running emerge and ebuild. -# 'splitdebug' Prior to stripping ELF etdyn and etexec files, the debugging -# info is stored for later use by various debuggers. This -# feature is disabled by 'nostrip'. For installation of source -# code, see 'installsources'. -# 'strict' causes portage to react strongly to conditions that are -# potentially dangerous, like missing/incorrect Manifest files. -# 'stricter' causes portage to react strongly to conditions that may -# conflict with system security provisions (for example -# textrels, executable stacks). -# 'userfetch' when portage is run as root, drop privileges to -# portage:portage during the fetching of package sources. -# 'userpriv' allows portage to drop root privileges while it is compiling, -# as a security measure. As a side effect this can remove -# sandbox access violations for users. -# 'usersandbox' enables sandboxing while portage is running under userpriv. -#FEATURES="sandbox buildpkg ccache distcc userpriv usersandbox notitles noauto" -#FEATURES="sandbox ccache distcc distlocks" +# FEATURES defines actions portage takes by default. This is an incremental +# variable. See the make.conf(5) man page for a complete list of supported +# values and their respective meanings. +#FEATURES="ccache collision-protect distcc installsources \ +# splitdebug test userpriv usersandbox" # CCACHE_SIZE and CCACHE_DIR are used to control the behavior of ccache, and # and are only used if "ccache" is in FEATURES. diff --git a/cnf/make.conf.sparc-fbsd.diff b/cnf/make.conf.sparc-fbsd.diff index 4ca3a50cd..f3d3bca8d 100644 --- a/cnf/make.conf.sparc-fbsd.diff +++ b/cnf/make.conf.sparc-fbsd.diff @@ -23,13 +23,3 @@ # Portage Directories # =================== -@@ -272,7 +286,8 @@ - # 'preserve-libs' - # keep libraries around that would normally removed by an upgrade, - # but are still needed by other packages --# 'sandbox' enables sandboxing when running emerge and ebuild. -+# 'sandbox' enables sandboxing when running emerge and ebuild. Doesn't -+# work on *BSD-based systems. - # 'splitdebug' Prior to stripping ELF etdyn and etexec files, the debugging - # info is stored for later use by various debuggers. This - # feature is disabled by 'nostrip'. For installation of source diff --git a/cnf/make.conf.x86-fbsd.diff b/cnf/make.conf.x86-fbsd.diff index a71e54032..1a49edc66 100644 --- a/cnf/make.conf.x86-fbsd.diff +++ b/cnf/make.conf.x86-fbsd.diff @@ -63,13 +63,3 @@ # Portage Directories # =================== -@@ -272,7 +313,8 @@ - # 'preserve-libs' - # keep libraries around that would normally removed by an upgrade, - # but are still needed by other packages --# 'sandbox' enables sandboxing when running emerge and ebuild. -+# 'sandbox' enables sandboxing when running emerge and ebuild. Doesn't -+# work on *BSD-based systems. - # 'splitdebug' Prior to stripping ELF etdyn and etexec files, the debugging - # info is stored for later use by various debuggers. This - # feature is disabled by 'nostrip'. For installation of source diff --git a/cnf/sets.conf b/cnf/sets.conf index a0ab06858..c02d143ff 100644 --- a/cnf/sets.conf +++ b/cnf/sets.conf @@ -24,7 +24,10 @@ class = portage.sets.profiles.PackagesSystemSet class = portage.sets.security.NewAffectedSet world-candidate = False -# Again, not much to change here, though people might prefer a different name +# A superset of the classic world target, a set created +# by this class contains SLOT atoms to match all installed packages. Note that +# use of this set makes it impossible for emerge to solve blockers by automatic +# uninstallation of blocked packages. [installed] class = portage.sets.dbapi.EverythingSet world-candidate = False @@ -44,9 +47,10 @@ world-candidate = False # Installed ebuilds that inherit from known live eclasses. [live-rebuild] -class = portage.sets.dbapi.InheritSet +class = portage.sets.dbapi.VariableSet world-candidate = False -inherits = cvs darcs git mercurial subversion +variable = INHERITED +includes = cvs darcs git mercurial subversion # Installed packages that own files inside /lib/modules. [module-rebuild] diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook index 7cccbbc93..14e171fb2 100644 --- a/doc/config/sets.docbook +++ b/doc/config/sets.docbook @@ -443,7 +443,9 @@ portage.sets.dbapi.EverythingSet A superset of the classic world target, a set created - by this class contains SLOT atoms to match all installed packages. + by this class contains SLOT atoms to match all installed packages. Note that + use of this set makes it impossible for emerge to solve blockers by automatic + uninstallation of blocked packages. @@ -453,28 +455,36 @@ - - portage.sets.dbapi.InheritSet + + portage.sets.dbapi.OwnerSet Package set which contains all packages - that inherit one or more specific eclasses. + that own one or more files. This class supports the following options: - inherits: Required. A list of eclass names - which should be used to create the package set. + files: Required. A list of file paths + that should be used to create the package set. - - portage.sets.dbapi.OwnerSet + + portage.sets.dbapi.VariableSet Package set which contains all packages - that own one or more files. + that match specified values of specified variable. This class supports the following options: - files: Required. A list of file paths - that should be used to create the package set. + variable: The name of + the specified variable whose values are checked. + + includes: A list of + values that must be contained within the specified + variable. + + excludes: A list of + values that must not be contained within the specified + variable. @@ -525,7 +535,7 @@ security: uses NewAffectedSet with default options everything: uses EverythingSet preserved-rebuild: uses PreservedLibraryConsumerSet - live-rebuild: uses InheritSet + live-rebuild: uses VariableSet module-rebuild: uses OwnerSet downgrade: uses DowngradeSet diff --git a/doc/dependency_resolution/task_scheduling.docbook b/doc/dependency_resolution/task_scheduling.docbook index 01953128e..c8e3608d1 100644 --- a/doc/dependency_resolution/task_scheduling.docbook +++ b/doc/dependency_resolution/task_scheduling.docbook @@ -68,7 +68,13 @@ Parallel Scheduling - TODO: Spawn an appropriate number of tasks in parallel when desired. + The algorithm used to choose packages that will execute concurrently + with other packages is as conservative as possible in the sense that + a given package will not be executed if the subgraph composed of its + direct and indirect dependencies contains any scheduled merges. By + ensuring that the subgraph of deep dependencies is fully up to date + in this way, potential problems are avoided which could be triggered + by other build orders that are less optimal. diff --git a/man/make.conf.5 b/man/make.conf.5 index f6b4b85d5..0eee4cd73 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -143,10 +143,10 @@ Contains additional options that \fBeconf\fR will append to configure script arguments (see \fBebuild\fR(5)). .TP \fBFEATURES\fR = \fI"sandbox ccache"\fR -Defines actions portage takes by default. These options should -not be changed by anyone but developers and/or maintainers. 'sandbox' is an important -part of \fBFEATURES\fR and should not be disabled by default. This is -an incremental variable. +Defines actions portage takes by default. This is an incremental variable. +Most of these settings are for developer use, but some are available to +non-developers as well. The \fBsandbox\fR feature is very important and +should not be disabled by default. .RS .TP .B assume\-digests diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 42d19117c..53193b210 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2254,7 +2254,7 @@ class MiscFunctionsProcess(SpawnProcess): class EbuildFetcher(SpawnProcess): - __slots__ = ("fetchonly", "pkg",) + __slots__ = ("fetchonly", "fetchall", "pkg",) def _start(self): @@ -2262,8 +2262,16 @@ class EbuildFetcher(SpawnProcess): portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) settings = root_config.settings + phase = "fetch" + if self.fetchall: + phase = "fetchall" + + # If any incremental variables have been overridden + # via the environment, those values need to be passed + # along here so that they are correctly considered by + # the config instance in the subproccess. + fetch_env = os.environ.copy() - fetch_env = dict(settings.iteritems()) fetch_env["PORTAGE_NICENESS"] = "0" if self.fetchonly: fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1" @@ -2271,7 +2279,7 @@ class EbuildFetcher(SpawnProcess): ebuild_binary = os.path.join( settings["PORTAGE_BIN_PATH"], "ebuild") - fetch_args = [ebuild_binary, ebuild_path, "fetch"] + fetch_args = [ebuild_binary, ebuild_path, phase] debug = settings.get("PORTAGE_DEBUG") == "1" if debug: fetch_args.append("--debug") @@ -2423,7 +2431,8 @@ class EbuildBuild(CompositeTask): if self.background: fetch_log = self.scheduler.fetch.log_file - fetcher = EbuildFetcher(fetchonly=opts.fetchonly, + fetcher = EbuildFetcher(fetchall=opts.fetch_all_uri, + fetchonly=opts.fetchonly, background=self.background, logfile=fetch_log, pkg=pkg, scheduler=self.scheduler) @@ -4647,25 +4656,30 @@ class depgraph(object): return 0 if debug: print "Candidates:", selected_atoms + for atom in selected_atoms: - if isinstance(atom, basestring) \ - and not portage.isvalidatom(atom): + try: + + blocker = atom.startswith("!") + if blocker: + atom = atom[1:] + mypriority = dep_priority.copy() + if not blocker and vardb.match(atom): + mypriority.satisfied = True + + if not self._add_dep(Dependency(atom=atom, + blocker=blocker, depth=depth, parent=pkg, + priority=mypriority, root=dep_root), + allow_unsatisfied=allow_unsatisfied): + return 0 + + except portage.exception.InvalidAtom, e: show_invalid_depstring_notice( - pkg, dep_string, str(atom)) + pkg, dep_string, str(e)) + del e if not pkg.installed: return 0 - continue - blocker = atom.startswith("!") - if blocker: - atom = atom[1:] - mypriority = dep_priority.copy() - if not blocker and vardb.match(atom): - mypriority.satisfied = True - if not self._add_dep(Dependency(atom=atom, - blocker=blocker, depth=depth, parent=pkg, - priority=mypriority, root=dep_root), - allow_unsatisfied=allow_unsatisfied): - return 0 + if debug: print "Exiting...", jbigkey except ValueError, e: @@ -10398,6 +10412,7 @@ def unmerge(root_config, myopts, unmerge_action, # we don't want to unmerge packages that are still listed in user-editable package sets # listed in "world" as they would be remerged on the next update of "world" or the # relevant package sets. + unknown_sets = set() for cp in xrange(len(pkgmap)): for cpv in pkgmap[cp]["selected"].copy(): try: @@ -10413,6 +10428,17 @@ def unmerge(root_config, myopts, unmerge_action, # removed from "world" later on) if s in root_config.setconfig.active or (s == "world" and not root_config.setconfig.active): continue + + if s not in sets: + if s in unknown_sets: + continue + unknown_sets.add(s) + out = portage.output.EOutput() + out.eerror(("Unknown set '@%s' in " + \ + "%svar/lib/portage/world_sets") % \ + (s, root_config.root)) + continue + # only check instances of EditablePackageSet as other classes are generally used for # special purposes and can be ignored here (and are usually generated dynamically, so the # user can't do much about them anyway) @@ -13489,6 +13515,9 @@ def emerge_main(): "--usepkg", "--usepkgonly"): myopts.pop(opt, None) + if "--fetch-all-uri" in myopts: + myopts["--fetchonly"] = True + if "--skipfirst" in myopts and "--resume" not in myopts: myopts["--resume"] = True diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index ff50264e5..cc3fdec92 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -5111,7 +5111,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, clean_phases = ("clean", "cleanrm") validcommands = ["help","clean","prerm","postrm","cleanrm","preinst","postinst", - "config","info","setup","depend","fetch","digest", + "config", "info", "setup", "depend", + "fetch", "fetchall", "digest", "unpack","compile","test","install","rpm","qmerge","merge", "package","unmerge", "manifest"] @@ -5126,6 +5127,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, writemsg("\n", noiselevel=-1) return 1 + if mydo == "fetchall": + fetchall = 1 + mydo = "fetch" + if mydo not in clean_phases and not os.path.exists(myebuild): writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo), noiselevel=-1) diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py index 96e9e553b..112301476 100644 --- a/pym/portage/cache/mappings.py +++ b/pym/portage/cache/mappings.py @@ -262,6 +262,9 @@ def slot_dict_class(keys, prefix="_val_"): except AttributeError: pass + def __str__(self): + return str(dict(self.iteritems())) + v = SlotDict _slot_dict_classes[v.allowed_keys] = v return v diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index bab956b57..f926f9441 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -774,9 +774,6 @@ class binarytree(object): noiselevel=-1) return slot = slot.strip() - from portage.checksum import perform_multiple_checksums - digests = perform_multiple_checksums( - full_path, hashes=self._pkgindex_hashes) self.dbapi.cpv_inject(cpv) self.dbapi._aux_cache.pop(cpv, None) @@ -807,26 +804,16 @@ class binarytree(object): del f if not self._pkgindex_version_supported(pkgindex): pkgindex = self._new_pkgindex() - d = digests - d["CPV"] = cpv - d["SLOT"] = slot - d["MTIME"] = str(long(s.st_mtime)) - d["SIZE"] = str(s.st_size) - rel_path = self._pkg_paths[cpv] - # record location if it's non-default - if rel_path != cpv + ".tbz2": - d["PATH"] = rel_path - from itertools import izip - d.update(izip(self._pkgindex_aux_keys, - self.dbapi.aux_get(cpv, self._pkgindex_aux_keys))) + try: - self._eval_use_flags(cpv, d) + d = self._pkgindex_entry(cpv) except portage.exception.InvalidDependString: writemsg("!!! Invalid binary package: '%s'\n" % \ self.getname(cpv), noiselevel=-1) self.dbapi.cpv_remove(cpv) del self._pkg_paths[cpv] return + # If found, remove package(s) with duplicate path. for i in xrange(len(pkgindex.packages) - 1, -1, -1): d2 = pkgindex.packages[i] @@ -835,6 +822,7 @@ class binarytree(object): if d2.get("PATH", "") == d.get("PATH", ""): del pkgindex.packages[i] pkgindex.packages.append(d) + self._update_pkgindex_header(pkgindex.header) from portage.util import atomic_ofstream f = atomic_ofstream(os.path.join(self.pkgdir, "Packages")) @@ -846,6 +834,36 @@ class binarytree(object): if pkgindex_lock: unlockfile(pkgindex_lock) + def _pkgindex_entry(self, cpv): + """ + Performs checksums and evaluates USE flag conditionals. + Raises InvalidDependString if necessary. + @rtype: dict + @returns: a dict containing entry for the give cpv. + """ + + pkg_path = self.getname(cpv) + from portage.checksum import perform_multiple_checksums + + d = dict(izip(self._pkgindex_aux_keys, + self.dbapi.aux_get(cpv, self._pkgindex_aux_keys))) + + d.update(perform_multiple_checksums( + pkg_path, hashes=self._pkgindex_hashes)) + + d["CPV"] = cpv + st = os.stat(pkg_path) + d["MTIME"] = str(long(st.st_mtime)) + d["SIZE"] = str(st.st_size) + + rel_path = self._pkg_paths[cpv] + # record location if it's non-default + if rel_path != cpv + ".tbz2": + d["PATH"] = rel_path + + self._eval_use_flags(cpv, d) + return d + def _new_pkgindex(self): return portage.getbinpkg.PackageIndex( allowed_pkg_keys=self._pkgindex_allowed_pkg_keys, diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 53285329b..593439d52 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -486,7 +486,7 @@ class vardbapi(dbapi): self.vartree = vartree self._aux_cache_keys = set( ["CHOST", "COUNTER", "DEPEND", "DESCRIPTION", - "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS", + "EAPI", "HOMEPAGE", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "repository", "RESTRICT" , "SLOT", "USE"]) self._aux_cache_obj = None @@ -922,7 +922,7 @@ class vardbapi(dbapi): cache_valid = cache_mtime == mydir_mtime if cache_valid: mydata.update(metadata) - pull_me.difference_update(metadata) + pull_me.difference_update(mydata) if pull_me: # pull any needed data and cache it @@ -2837,10 +2837,14 @@ class dblink(object): # it may not be visible via a scrollback buffer, especially # if the number of file collisions is large. Therefore, # show a summary at the end. - msg = ("Package '%s' NOT merged due to " + \ - "file collisions. If necessary, refer to your elog " + \ - "messages for the whole content of the above message.") % \ - self.settings.mycpv + if collision_protect: + msg = "Package '%s' NOT merged due to file collisions." % \ + self.settings.mycpv + else: + msg = "Package '%s' merged despite file collisions." % \ + self.settings.mycpv + msg += " If necessary, refer to your elog " + \ + "messages for the whole content of the above message." eerror(wrap(msg, 70)) if collision_protect: diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py index 798b322e3..2ddefb861 100644 --- a/pym/portage/sets/dbapi.py +++ b/pym/portage/sets/dbapi.py @@ -3,29 +3,50 @@ # $Id$ from portage.versions import catpkgsplit, catsplit, pkgcmp +from portage.dep import Atom from portage.sets.base import PackageSet from portage.sets import SetConfigError, get_boolean -__all__ = ["CategorySet", "EverythingSet", "InheritSet"] +__all__ = ["CategorySet", "DowngradeSet", + "EverythingSet", "OwnerSet", "VariableSet"] class EverythingSet(PackageSet): _operations = ["merge", "unmerge"] description = "Package set which contains SLOT " + \ "atoms to match all installed packages" + _filter = None def __init__(self, vdbapi): super(EverythingSet, self).__init__() self._db = vdbapi - + def load(self): myatoms = [] + db_keys = ["SLOT"] + aux_get = self._db.aux_get + cp_list = self._db.cp_list + for cp in self._db.cp_all(): - if len(self._db.cp_list(cp)) > 1: - for cpv in self._db.cp_list(cp): - myslot = self._db.aux_get(cpv, ["SLOT"])[0] - myatoms.append(cp+":"+myslot) + cpv_list = cp_list(cp) + + if len(cpv_list) > 1: + for cpv in cpv_list: + slot, = aux_get(cpv, db_keys) + atom = Atom("%s:%s" % (cp, slot)) + if self._filter: + if self._filter(atom): + myatoms.append(atom) + else: + myatoms.append(atom) + else: - myatoms.append(cp) + atom = Atom(cp) + if self._filter: + if self._filter(atom): + myatoms.append(atom) + else: + myatoms.append(atom) + self._setAtoms(myatoms) def singleBuilder(self, options, settings, trees): @@ -68,40 +89,49 @@ class OwnerSet(PackageSet): singleBuilder = classmethod(singleBuilder) -class InheritSet(PackageSet): +class VariableSet(EverythingSet): _operations = ["merge", "unmerge"] description = "Package set which contains all packages " + \ - "that inherit one or more specific eclasses." + "that match specified values of a specified variable." - def __init__(self, vardb=None, inherits=None): - super(InheritSet, self).__init__() - self._db = vardb - self._inherits = inherits + def __init__(self, vardb, portdb=None, variable=None, includes=None, excludes=None): + super(VariableSet, self).__init__(vardb) + self._portdb = portdb + self._variable = variable + self._includes = includes + self._excludes = excludes + + def _filter(self, atom): + ebuild = self._portdb.xmatch("bestmatch-visible", atom) + if not ebuild: + return False + values, = self._portdb.aux_get(ebuild, [self._variable]) + values = values.split() + if self._includes and not self._includes.intersection(values): + return False + if self._excludes and self._excludes.intersection(values): + return False + return True - def load(self): - atoms = [] - inherits = self._inherits - cp_list = self._db.cp_list - aux_get = self._db.aux_get - aux_keys = ["INHERITED", "SLOT"] - for cp in self._db.cp_all(): - for cpv in cp_list(cp): - inherited, slot = aux_get(cpv, aux_keys) - inherited = inherited.split() - if inherits.intersection(inherited): - atoms.append("%s:%s" % (cp, slot)) + def singleBuilder(cls, options, settings, trees): - self._setAtoms(atoms) + variable = options.get("variable") + if variable is None: + raise SetConfigError("missing required attribute: 'variable'") - def singleBuilder(cls, options, settings, trees): - if not "inherits" in options: - raise SetConfigError("no inherits given") + includes = options.get("includes", "") + excludes = options.get("excludes", "") - inherits = options["inherits"] - return cls(vardb=trees["vartree"].dbapi, - inherits=frozenset(inherits.split())) + if not (includes or excludes): + raise SetConfigError("no includes or excludes given") + + return cls(trees["vartree"].dbapi, + portdb=trees["porttree"].dbapi, + excludes=frozenset(excludes.split()), + includes=frozenset(includes.split()), + variable=variable) singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/sets/profiles.py b/pym/portage/sets/profiles.py index 4b690895b..b40d86c72 100644 --- a/pym/portage/sets/profiles.py +++ b/pym/portage/sets/profiles.py @@ -2,9 +2,12 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import logging import os from portage.util import grabfile_package, stack_lists from portage.sets.base import PackageSet +from portage.sets import get_boolean +from portage.util import writemsg_level from portage.const import EPREFIX __all__ = ["PackagesSystemSet"] @@ -12,9 +15,10 @@ __all__ = ["PackagesSystemSet"] class PackagesSystemSet(PackageSet): _operations = ["merge"] - def __init__(self, profile_paths): + def __init__(self, profile_paths, debug=False): super(PackagesSystemSet, self).__init__() self._profile_paths = profile_paths + self._debug = debug if profile_paths: description = self._profile_paths[-1] if description == EPREFIX+"/etc/portage/profile" and \ @@ -25,10 +29,26 @@ class PackagesSystemSet(PackageSet): self.description = "System packages for profile %s" % description def load(self): + debug = self._debug + if debug: + writemsg_level("\nPackagesSystemSet: profile paths: %s\n" % \ + (self._profile_paths,), level=logging.DEBUG, noiselevel=-1) + mylist = [grabfile_package(os.path.join(x, "packages")) for x in self._profile_paths] + + if debug: + writemsg_level("\nPackagesSystemSet: raw packages: %s\n" % \ + (mylist,), level=logging.DEBUG, noiselevel=-1) + mylist = stack_lists(mylist, incremental=1) + + if debug: + writemsg_level("\nPackagesSystemSet: stacked packages: %s\n" % \ + (mylist,), level=logging.DEBUG, noiselevel=-1) + self._setAtoms([x[1:] for x in mylist if x[0] == "*"]) def singleBuilder(self, options, settings, trees): - return PackagesSystemSet(settings.profiles) + debug = get_boolean(options, "debug", False) + return PackagesSystemSet(settings.profiles, debug=debug) singleBuilder = classmethod(singleBuilder) -- 2.26.2