From 4ab58c5a255ebc3f5e3ddd6c8e73063821687c9e Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 30 Dec 2006 16:08:50 +0000 Subject: [PATCH] Merged from trunk 4500:4514 try to give the emerge manpage a facelift, upgrade clean and prune docs, change 'dependency' to 'atom', other minor fixes (WarnerBro) Use a pickle to cache vdb metadata. This greatly improves the performance for loading the vdb when the buffer cache is cold. Make portdbapi.flush_cache() consistent with vardbapi.flush_cache(). Update emerge --metadata docs for more clarity. Allow the list of keys cached by the vardb to change without forcing a cache version bump. Make vartree.getslot() use the dbapi for aux_get caching. Make vardbapi.cpv_counter() use aux_get for caching. For bug #159252, don't preserve file attributes when copying files for installsources. Thanks to Ed Catmur for this patch. For bug #159295 and python-2.3 compatibility, don't assume that dict.update can use an iterable. Use a pickle to cache blockers for all installed packages so that dep_check doesn't have to be called for every single installed package on every invocation of emerge. For bug #159161, make emerge --info show USE_EXPAND flags similar to the way it does in merge list output. Add some docs for the blockers cache format. For bug #159310, don't show a slot collision notice if there are unresolvable blockers. Do some minor cleanup in the dep_zapdeps downgrade avoidance logic (while working toward a fix for bug #159360). svn path=/main/branches/prefix/; revision=5426 --- bin/emerge | 182 +++++++++++++++++++++++++++++++++++++++++++--- bin/prepstrip | 2 +- man/emerge.1 | 91 ++++++++++++----------- pym/emergehelp.py | 11 ++- pym/portage.py | 139 ++++++++++++++++++++++++++++++----- 5 files changed, 345 insertions(+), 80 deletions(-) diff --git a/bin/emerge b/bin/emerge index 1e191accf..47d895731 100755 --- a/bin/emerge +++ b/bin/emerge @@ -50,6 +50,12 @@ from portage_data import secpass if not hasattr(__builtins__, "set"): from sets import Set as set from itertools import chain, izip +from UserDict import DictMixin + +try: + import cPickle +except ImportError: + import pickle as cPickle class stdout_spinner(object): scroll_msgs = [ @@ -745,6 +751,7 @@ class FakeVartree(portage.vartree): continue slot_counters[myslot_atom] = mycounter self.dbapi.cpv_inject(cpv, metadata=metadata) + real_dbapi.flush_cache() finally: if vdb_lock: portage_locks.unlockdir(vdb_lock) @@ -801,6 +808,105 @@ def perform_global_updates(mycpv, mydb, mycommands): if updates: mydb.aux_update(mycpv, updates) +class BlockerCache(DictMixin): + """This caches blockers of installed packages so that dep_check does not + have to be done for every single installed package on every invocation of + emerge. The cache is invalidated whenever it is detected that something + has changed that might alter the results of dep_check() calls: + 1) the set of installed packages (including COUNTER) has changed + 2) the old-style virtuals have changed + """ + class BlockerData(object): + def __init__(self, counter, atoms): + self.counter = counter + self.atoms = atoms + + def __init__(self, myroot, vardb): + self._vardb = vardb + self._installed_pkgs = set(vardb.cpv_all()) + self._virtuals = vardb.settings.getvirtuals() + self._cache_filename = os.path.join(myroot, + portage.CACHE_PATH.lstrip(os.path.sep), "vdb_blockers.pickle") + self._cache_version = "1" + self._cache_data = None + self._modified = False + self._load() + + def _load(self): + try: + f = open(self._cache_filename) + mypickle = cPickle.Unpickler(f) + mypickle.find_global = None + self._cache_data = mypickle.load() + f.close() + del f + except (IOError, OSError, EOFError, cPickle.UnpicklingError): + pass + cache_valid = self._cache_data and \ + isinstance(self._cache_data, dict) and \ + self._cache_data.get("version") == self._cache_version and \ + self._cache_data["virtuals"] == self._virtuals and \ + set(self._cache_data["blockers"]) == self._installed_pkgs + if cache_valid: + for pkg in self._installed_pkgs: + if long(self._vardb.aux_get(pkg, ["COUNTER"])[0]) != \ + self[pkg].counter: + cache_valid = False + break + if not cache_valid: + self._cache_data = {"version":self._cache_version} + self._cache_data["blockers"] = {} + self._cache_data["virtuals"] = self._virtuals + self._modified = False + + def flush(self): + """If the current user has permission and the internal blocker cache + been updated, save it to disk and mark it unmodified. This is called + by emerge after it has proccessed blockers for all installed packages. + Currently, the cache is only written if the user has superuser + privileges (since that's required to obtain a lock), but all users + have read access and benefit from faster blocker lookups (as long as + the entire cache is still valid). The cache is stored as a pickled + dict object with the following format: + + { + version : "1", + "blockers" : {cpv1:(counter,(atom1, atom2...)), cpv2...}, + "virtuals" : vardb.settings.getvirtuals() + } + """ + if self._modified and \ + secpass >= 2: + try: + f = portage_util.atomic_ofstream(self._cache_filename) + cPickle.dump(self._cache_data, f, -1) + f.close() + portage_util.apply_secpass_permissions( + self._cache_filename, gid=portage.portage_gid, mode=0644) + except (IOError, OSError), e: + pass + self._modified = False + + def __setitem__(self, cpv, blocker_data): + """ + Update the cache and mark it as modified for a future call to + self.flush(). + + @param cpv: Package for which to cache blockers. + @type cpv: String + @param blocker_data: An object with counter and atoms attributes. + @type blocker_data: BlockerData + """ + self._cache_data["blockers"][cpv] = \ + (blocker_data.counter, blocker_data.atoms) + self._modified = True + + def __getitem__(self, cpv): + """ + @rtype: BlockerData + @returns: An object with counter and atoms attributes. + """ + return self.BlockerData(*self._cache_data["blockers"][cpv]) def show_invalid_depstring_notice(parent_node, depstring, error_msg): @@ -905,6 +1011,7 @@ class depgraph: self.args_keys = [] self.blocker_digraph = digraph() self.blocker_parents = {} + self._slot_collision_info = [] self._altlist_cache = {} self._pprovided_args = [] @@ -1071,15 +1178,30 @@ class depgraph: priority=priority) return 1 else: + # A slot collision has occurred. Sometimes this coincides + # with unresolvable blockers, so the slot collision will be + # shown later if there are no unresolvable blockers. e_parents = self._parent_child_digraph.parent_nodes( existing_node) myparents = [] if myparent: myparents.append(myparent) - self._show_slot_collision_notice( - ((jbigkey, myparents), - (existing_node, e_parents))) - return 0 + self._slot_collision_info.append( + ((jbigkey, myparents), (existing_node, e_parents))) + + # Now add this node to the graph so that self.display() + # can work with it show use flags and --tree output. + self.useFlags[myroot][mykey] = myuse + self._parent_child_digraph.add(jbigkey, myparent) + if rev_dep and myparent: + self.digraph.add(myparent, jbigkey, + priority=priority) + else: + self.digraph.add(jbigkey, myparent, + priority=priority) + # The slot collision has rendered the graph invalid, so + # there's no need to process dependencies of this node. + return 1 self._slot_node_map[myroot][slot_atom] = jbigkey self.pkg_node_map[myroot][mykey] = jbigkey @@ -1654,14 +1776,21 @@ class depgraph: portdb = self.trees[myroot]["porttree"].dbapi pkgsettings = self.pkgsettings[myroot] final_db = self.mydbapi[myroot] - for pkg in self.trees[myroot]["vartree"].dbapi.cpv_all(): + cpv_all_installed = self.trees[myroot]["vartree"].dbapi.cpv_all() + blocker_cache = BlockerCache(myroot, vardb) + for pkg in cpv_all_installed: blocker_atoms = None matching_node = pkg_node_map.get(pkg, None) - if not matching_node or \ - matching_node[3] == "merge": - # If this node has any blockers, create a "nomerge" - # node for it so that they can be enforced. - self.spinner.update() + if matching_node and \ + matching_node[3] == "nomerge": + continue + # If this node has any blockers, create a "nomerge" + # node for it so that they can be enforced. + self.spinner.update() + blocker_data = blocker_cache.get(pkg) + if blocker_data: + blocker_atoms = blocker_data.atoms + else: dep_vals = vardb.aux_get(pkg, dep_keys) myuse = vardb.aux_get(pkg, ["USE"])[0].split() depstr = " ".join(dep_vals) @@ -1690,6 +1819,9 @@ class depgraph: return False blocker_atoms = [myatom for myatom in atoms \ if myatom.startswith("!")] + counter = long(vardb.aux_get(pkg, ["COUNTER"])[0]) + blocker_cache[pkg] = \ + blocker_cache.BlockerData(counter, blocker_atoms) if blocker_atoms: # Don't store this parent in pkg_node_map, because it's # not needed there and it might overwrite a "merge" @@ -1703,6 +1835,8 @@ class depgraph: myparents = set() self.blocker_parents[blocker] = myparents myparents.add(myparent) + blocker_cache.flush() + del blocker_cache for blocker in self.blocker_parents.keys(): mytype, myroot, mydep = blocker @@ -1788,6 +1922,14 @@ class depgraph: # Validate blockers that depend on merge order. if not self.blocker_digraph.empty(): self.altlist() + if self._slot_collision_info: + # The user is only notified of a slot collision if there are no + # unresolvable blocks. + for x in self.altlist(): + if x[0] == "blocks": + return True + self._show_slot_collision_notice(self._slot_collision_info[0]) + return False return True def altlist(self, reversed=False): @@ -4104,8 +4246,24 @@ def action_info(settings, trees, myopts, myfiles): unset_vars = [] myvars.sort() for x in myvars: - if settings.has_key(x): - print x+'="'+settings[x]+'"' + if x in settings: + if x != "USE": + print '%s="%s"' % (x, settings[x]) + else: + use = set(settings["USE"].split()) + use_expand = settings["USE_EXPAND"].split() + use_expand.sort() + for varname in use_expand: + flag_prefix = varname.lower() + "_" + for f in list(use): + if f.startswith(flag_prefix): + use.remove(f) + print 'USE="%s"' % " ".join(use), + for varname in use_expand: + myval = settings.get(varname) + if myval: + print '%s="%s"' % (varname, myval), + print else: unset_vars.append(x) if unset_vars: diff --git a/bin/prepstrip b/bin/prepstrip index 372ff1b0e..e4e7a0d7f 100755 --- a/bin/prepstrip +++ b/bin/prepstrip @@ -33,7 +33,7 @@ save_elf_debug() { if [[ -s ${T}/debug.sources ]] ; then [[ -d ${ED}${sources_dir} ]] || mkdir -p "${ED}${sources_dir}" cat "${T}"/debug.sources | (cd "${WORKDIR}"; LANG=C sort -z -u | \ - rsync -aL0 --files-from=- "${WORKDIR}/" "${ED}${sources_dir}/" ) + rsync -rtL0 --files-from=- "${WORKDIR}/" "${ED}${sources_dir}/" ) fi fi diff --git a/man/emerge.1 b/man/emerge.1 index 61a3bfeea..44f641c6f 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -1,10 +1,10 @@ -.TH "EMERGE" "1" "Dec 2005" "Portage 2.1" "Portage" +.TH "EMERGE" "1" "Dec 2006" "Portage 2.1" "Portage" .SH "NAME" emerge \- Command\-line interface to the Portage system .SH "SYNOPSIS" .TP .BR emerge -[\fIoptions\fR] [\fIaction\fR] [\fIebuild\fR | \fItbz2file\fR | \fIset\fR | \fIdependency\fR] ... +[\fIoptions\fR] [\fIaction\fR] [\fIebuild\fR | \fItbz2file\fR | \fIset\fR | \fIatom\fR] ... .TP .BR emerge \fB\-\-sync\fR | \fB\-\-info\fR | \fB\-\-version\fR @@ -26,7 +26,7 @@ distribution. .SH "EBUILDS, TBZ2S, SETS AND DEPENDENCIES" \fBemerge\fR primarily installs packages. You can specify packages to install in one of four main ways: an \fIebuild\fR, -a \fItbz2file\fR, a \fIset\fR, or a \fIdependency\fR. +a \fItbz2file\fR, a \fIset\fR, or a \fIatom\fR. .LP .TP .BR ebuild @@ -57,14 +57,14 @@ other packages listed in \fB/var/lib/portage/world\fR. [See \fBFILES\fR below for more information.] Note that a \fIset\fR is generally used in conjunction with \fB\-\-update\fR. .TP -.BR dependency -A \fIdependency\fR describes bounds on a package that you wish to install. -\fISee portage(5) for the details on these 'atoms'.\fR For example, +.BR atom +An \fIatom\fR describes bounds on a package that you wish to install. +\fISee portage(5) for the details on atom syntax.\fR For example, \fB>=dev\-lang/python\-2.2.1\-r2\fR matches the latest available version of Python greater than or equal to 2.2.1\-r2. Similarly, \fB= 0: - is_okay = True + if pkgcmp(avail_split, catpkgsplit(pkg)[1:]) < 0: + is_downgrade = True break - if not is_okay: - all_installed = False + if is_downgrade: break - if all_installed: + if all_installed and not is_downgrade: preferred.append((atoms, all_available)) else: other.append((atoms, all_available)) @@ -4713,6 +4714,12 @@ class vardbapi(dbapi): if vartree is None: vartree = globals()["db"][root]["vartree"] self.vartree = vartree + self._aux_cache_keys = set(["SLOT", "COUNTER", "PROVIDE", "USE", + "IUSE", "DEPEND", "RDEPEND", "PDEPEND"]) + self._aux_cache = None + self._aux_cache_version = "1" + self._aux_cache_filename = os.path.join(self.root, + CACHE_PATH.lstrip(os.path.sep), "vdb_metadata.pickle") def cpv_exists(self,mykey): "Tells us whether an actual ebuild exists on disk (no masking)" @@ -4720,6 +4727,10 @@ class vardbapi(dbapi): def cpv_counter(self,mycpv): "This method will grab the COUNTER. Returns a counter value." + try: + return long(self.aux_get(mycpv, ["COUNTER"])[0]) + except KeyError, ValueError: + pass cdir=self.root+VDB_PATH+"/"+mycpv cpath=self.root+VDB_PATH+"/"+mycpv+"/COUNTER" @@ -4976,7 +4987,100 @@ class vardbapi(dbapi): def findname(self, mycpv): return self.root+VDB_PATH+"/"+str(mycpv)+"/"+mycpv.split("/")[1]+".ebuild" + def flush_cache(self): + """If the current user has permission and the internal aux_get cache has + been updated, save it to disk and mark it unmodified. This is called + by emerge after it has loaded the full vdb for use in dependency + calculations. Currently, the cache is only written if the user has + superuser privileges (since that's required to obtain a lock), but all + users have read access and benefit from faster metadata lookups (as + long as at least part of the cache is still valid).""" + if self._aux_cache is not None and \ + self._aux_cache["modified"] and \ + secpass >= 2: + valid_nodes = set(self.cpv_all()) + for cpv in self._aux_cache["packages"].keys(): + if cpv not in valid_nodes: + del self._aux_cache["packages"][cpv] + del self._aux_cache["modified"] + try: + f = atomic_ofstream(self._aux_cache_filename) + cPickle.dump(self._aux_cache, f, -1) + f.close() + portage_util.apply_secpass_permissions( + self._aux_cache_filename, gid=portage_gid, mode=0644) + except (IOError, OSError), e: + pass + self._aux_cache["modified"] = False + def aux_get(self, mycpv, wants): + """This automatically caches selected keys that are frequently needed + by emerge for dependency calculations. The cached metadata is + considered valid if the mtime of the package directory has not changed + since the data was cached. The cache is stored in a pickled dict + object with the following format: + + {version:"1", "packages":{cpv1:(mtime,{k1,v1, k2,v2, ...}), cpv2...}} + + If an error occurs while loading the cache pickle or the version is + unrecognized, the cache will simple be recreated from scratch (it is + completely disposable). + """ + if not self._aux_cache_keys.intersection(wants): + return self._aux_get(mycpv, wants) + if self._aux_cache is None: + try: + f = open(self._aux_cache_filename) + mypickle = cPickle.Unpickler(f) + mypickle.find_global = None + self._aux_cache = mypickle.load() + f.close() + del f + except (IOError, OSError, EOFError, cPickle.UnpicklingError): + pass + if not self._aux_cache or \ + not isinstance(self._aux_cache, dict) or \ + self._aux_cache.get("version") != self._aux_cache_version: + self._aux_cache = {"version":self._aux_cache_version} + self._aux_cache["packages"] = {} + self._aux_cache["modified"] = False + mydir = os.path.join(self.root, VDB_PATH, mycpv) + mydir_stat = None + try: + mydir_stat = os.stat(mydir) + except OSError, e: + if e.errno != errno.ENOENT: + raise + raise KeyError(mycpv) + mydir_mtime = long(mydir_stat.st_mtime) + pkg_data = self._aux_cache["packages"].get(mycpv) + mydata = {} + cache_valid = False + if pkg_data: + cache_mtime, metadata = pkg_data + cache_valid = cache_mtime == mydir_mtime + if cache_valid and set(metadata) != self._aux_cache_keys: + # Allow self._aux_cache_keys to change without a cache version + # bump. + cache_valid = False + if cache_valid: + mydata.update(metadata) + pull_me = set(wants).difference(self._aux_cache_keys) + else: + pull_me = self._aux_cache_keys.union(wants) + if pull_me: + # pull any needed data and cache it + aux_keys = list(pull_me) + mydata.update(izip(aux_keys, self._aux_get(mycpv, aux_keys))) + if not cache_valid: + cache_data = {} + for aux_key in self._aux_cache_keys: + cache_data[aux_key] = mydata[aux_key] + self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) + self._aux_cache["modified"] = True + return [mydata[x] for x in wants] + + def _aux_get(self, mycpv, wants): mydir = os.path.join(self.root, VDB_PATH, mycpv) if not os.path.isdir(mydir): raise KeyError(mycpv) @@ -5213,14 +5317,10 @@ class vartree(object): def getslot(self,mycatpkg): "Get a slot for a catpkg; assume it exists." - myslot = "" try: - myslot=string.join(grabfile(self.root+VDB_PATH+"/"+mycatpkg+"/SLOT")) - except SystemExit, e: - raise - except Exception, e: - pass - return myslot + return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] + except KeyError: + return "" def hasnode(self,mykey,use_cache): """Does the particular node (cat/pkg key) exist?""" @@ -5306,7 +5406,6 @@ class portdbapi(dbapi): self.eclassdb = eclass_cache.cache(self.porttree_root, overlays=self.mysettings["PORTDIR_OVERLAY"].split()) - self.metadb = {} self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule") #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening) @@ -5368,8 +5467,8 @@ class portdbapi(dbapi): self.auxdb.clear() def flush_cache(self): - self.metadb = {} - self.auxdb = {} + for x in self.auxdb.values(): + x.sync() def finddigest(self,mycpv): try: -- 2.26.2