From 21151bd950b7dbdb3ba6ed8dc42854ce33c27d6a Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Mon, 26 Feb 2007 20:43:09 +0000 Subject: [PATCH] Merged from trunk 5964:5976 For bug #158100, use an exact reversal of the actual merge order for --tree output. It's more efficient than doing 2 order calculations. Optimize merge order calculation for --tree output when necessary. Fix CONFIG_PROTECT_MASK support. Thanks to piavlo for this patch from bug #166837. Add missing quotes. Add support for RSYNC_EXTRA_OPTS to webrsync (genone) Print filename when displaying package.mask comments (genone) [[ Ignore compiled files. Add new feature 'preserve-libs' that injects files into $D that - belong to a package instance that is about to be autocleaned - are not installed by the new package instance - are listed in at least one NEEDED file in vdb - are not just used by other files of the package instance As a side-effect a new file /var/db/pkg/.NEEDED is created that contains a reversed map of the NEEDED files found in a packages dbdir. ]] (genone) Print hostname in xterm title if sheridan is available in the environment (genone) svn path=/main/branches/prefix/; revision=6074 --- bin/emerge | 22 ++++++--- bin/emerge-webrsync | 4 +- bin/etc-update | 1 + pym/portage/__init__.py | 43 +++++++++-------- pym/portage/dbapi/vartree.py | 92 ++++++++++++++++++++++++++++++++++-- 5 files changed, 129 insertions(+), 33 deletions(-) diff --git a/bin/emerge b/bin/emerge index f8e4f69c0..c2d617d3a 100755 --- a/bin/emerge +++ b/bin/emerge @@ -233,10 +233,11 @@ shortmapping={ def emergelog(xterm_titles, mystr, short_msg=None): if xterm_titles: - if short_msg: - xtermTitle(short_msg) - else: - xtermTitle(mystr) + if short_msg == None: + short_msg = mystr + if "HOSTNAME" in os.environ.keys(): + short_msg = os.environ["HOSTNAME"]+": "+short_msg + xtermTitle(short_msg) try: file_path = EPREFIX+"/var/log/emerge.log" mylogfile = open(file_path, "a") @@ -1674,9 +1675,10 @@ class depgraph: mreasons = portage.getmaskingstatus(p, settings=pkgsettings, portdb=portdb) print "- "+p+" (masked by: "+", ".join(mreasons)+")" - comment = portage.getmaskingreason(p, - settings=pkgsettings, portdb=portdb) + comment, filename = portage.getmaskingreason(p, + settings=pkgsettings, portdb=portdb, return_location=True) if comment and comment != oldcomment: + print filename+":" print comment oldcomment = comment print @@ -1966,6 +1968,11 @@ class depgraph: def altlist(self, reversed=False): if reversed in self._altlist_cache: return self._altlist_cache[reversed][:] + if reversed: + retlist = self.altlist() + retlist.reverse() + self._altlist_cache[reversed] = retlist[:] + return retlist mygraph=self.digraph.copy() myblockers = self.blocker_digraph.copy() retlist=[] @@ -1983,6 +1990,7 @@ class depgraph: ignore_priority_range = [None] ignore_priority_range.extend( xrange(DepPriority.MIN, DepPriority.MEDIUM + 1)) + tree_mode = "--tree" in self.myopts while not mygraph.empty(): ignore_priority = None nodes = None @@ -2005,7 +2013,7 @@ class depgraph: selected_nodes = None if nodes: if ignore_priority <= DepPriority.SOFT: - if ignore_priority is None and not reversed: + if ignore_priority is None and not tree_mode: # Greedily pop all of these nodes since no relationship # has been ignored. This optimization destroys --tree # output, so it's disabled in reversed mode. diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync index f8ae87d63..c0be976cd 100755 --- a/bin/emerge-webrsync +++ b/bin/emerge-webrsync @@ -7,7 +7,7 @@ eval $("@PORTAGE_BASE@"/bin/portageq envvar -v FEATURES FETCHCOMMAND \ GENTOO_MIRRORS PORTAGE_INST_UID PORTAGE_INST_GID PORTAGE_NICENESS \ - PORTAGE_TMPDIR PORTDIR EPREFIX) + PORTAGE_TMPDIR PORTDIR PORTAGE_RSYNC_EXTRA_OPTS EPREFIX) DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync" # If PORTAGE_NICENESS is overriden via the env then it will @@ -70,7 +70,7 @@ sync_local() { cd portage rsync -av --progress --stats --delete --delete-after \ --exclude='/distfiles' --exclude='/packages' \ - --exclude='/local' . ${PORTDIR%%/} + --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . ${PORTDIR%%/} cd .. echo "cleaning up" rm -rf portage diff --git a/bin/etc-update b/bin/etc-update index 8911a2033..e447a807f 100755 --- a/bin/etc-update +++ b/bin/etc-update @@ -61,6 +61,7 @@ function scan() { rfile=$(echo "${file/\/\///}" | sed -e "s:^.*/::") for mpath in ${CONFIG_PROTECT_MASK}; do mpath="${EROOT}${mpath}" + mpath=$(echo "${mpath/\/\///}") if [[ "${rpath}" == "${mpath}"* ]]; then mv ${rpath}/${rfile} ${rpath}/${rfile:10} break diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 1f7ff0d06..fb705e199 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4408,7 +4408,7 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None): else: return mykey -def getmaskingreason(mycpv, settings=None, portdb=None): +def getmaskingreason(mycpv, settings=None, portdb=None, return_location=False): from portage.util import grablines if settings is None: settings = globals()["settings"] @@ -4431,11 +4431,7 @@ def getmaskingreason(mycpv, settings=None, portdb=None): locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH.lstrip(os.path.sep))) locations.reverse() - pmasklists = [grablines(os.path.join(x, "package.mask"), recursive=1) for x in locations] - pmasklines = [] - while pmasklists: # stack_lists doesn't preserve order so it can't be used - pmasklines.extend(pmasklists.pop(0)) - del pmasklists + pmasklists = [(x, grablines(os.path.join(x, "package.mask"), recursive=1)) for x in locations] if settings.pmaskdict.has_key(mycp): for x in settings.pmaskdict[mycp]: @@ -4443,22 +4439,27 @@ def getmaskingreason(mycpv, settings=None, portdb=None): comment = "" l = "\n" comment_valid = -1 - for i in xrange(len(pmasklines)): - l = pmasklines[i].strip() - if l == "": - comment = "" - comment_valid = -1 - elif l[0] == "#": - comment += (l+"\n") - comment_valid = i + 1 - elif l == x: - if comment_valid != i: + for pmask in pmasklists: + pmask_filename = os.path.join(pmask[0], "package.mask") + for i in xrange(len(pmask[1])): + l = pmask[1][i].strip() + if l == "": comment = "" - return comment - elif comment_valid != -1: - # Apparently this comment applies to muliple masks, so - # it remains valid until a blank line is encountered. - comment_valid += 1 + comment_valid = -1 + elif l[0] == "#": + comment += (l+"\n") + comment_valid = i + 1 + elif l == x: + if comment_valid != i: + comment = "" + if return_location: + return (comment, pmask_filename) + else: + return comment + elif comment_valid != -1: + # Apparently this comment applies to muliple masks, so + # it remains valid until a blank line is encountered. + comment_valid += 1 return None def getmaskingstatus(mycpv, settings=None, portdb=None): diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 9f59432b6..6269c6305 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -523,6 +523,39 @@ class vardbapi(dbapi): write_atomic(cpath, str(counter)) return counter + def get_library_map(self): + """ Read the global library->consumer map for this vdb instance """ + mapfilename = self.getpath(".NEEDED") + if not os.path.exists(mapfilename): + self.update_library_map() + rValue = {} + for l in open(mapfilename, "r").read().split("\n"): + mysplit = l.split() + if len(mysplit) > 1: + rValue[mysplit[0]] = mysplit[1].split(",") + return rValue + + def update_library_map(self): + """ Update the global library->consumer map for this vdb instance. """ + mapfilename = self.getpath(".NEEDED") + obj_dict = {} + for cpv in self.cpv_all(): + needed_list = self.aux_get(cpv, ["NEEDED"])[0] + for l in needed_list.split("\n"): + mysplit = l.split() + if len(mysplit) < 2: + continue + libs = mysplit[1].split(",") + for lib in libs: + if not obj_dict.has_key(lib): + obj_dict[lib] = [mysplit[0]] + else: + obj_dict[lib].append(mysplit[0]) + mapfile = open(mapfilename, "w") + for lib in obj_dict.keys(): + mapfile.write(lib+" "+",".join(obj_dict[lib])+"\n") + mapfile.close() + class vartree(object): "this tree will scan a var/db/pkg database located at root (passed to init)" def __init__(self, root="/", virtual=None, clone=None, categories=None, @@ -952,6 +985,9 @@ class dblink(object): doebuild(myebuildpath, "cleanrm", self.myroot, self.settings, tree="vartree", mydbapi=self.vartree.dbapi, vartree=self.vartree) + + # regenerate reverse NEEDED map + self.vartree.dbapi.update_library_map() finally: if builddir_lock: @@ -1162,6 +1198,7 @@ class dblink(object): This function does the following: + Preserve old libraries that are still used. Collision Protection. calls doebuild(mydo=pkg_preinst) Merges the package to the livefs @@ -1197,6 +1234,10 @@ class dblink(object): if not os.path.exists(self.dbcatdir): os.makedirs(self.dbcatdir) + myfilelist = listdir(srcroot, recursive=1, filesonly=1, followSymlinks=True) + mysymlinks = filter(os.path.islink, listdir(srcroot, recursive=1, filesonly=0, followSymlinks=False)) + myfilelist.extend(mysymlinks) + otherversions = [] for v in self.vartree.dbapi.cp_list(self.mysplit[0]): otherversions.append(v.split("/")[1]) @@ -1209,17 +1250,59 @@ class dblink(object): catsplit(slot_matches[0])[1], destroot, self.settings, vartree=self.vartree) + # Preserve old libs if they are still in use + if slot_matches and "preserve-libs" in self.settings.features: + # read global reverse NEEDED map + libmap = self.vartree.dbapi.get_library_map() + + # get list of libraries from old package instance + old_contents = self._installed_instance.getcontents().keys() + old_libs = set([os.path.basename(x) for x in old_contents]).intersection(libmap.keys()) + + # get list of libraries from new package instance + mylibs = set([os.path.basename(x) for x in myfilelist]).intersection(libmap.keys()) + + # check which libs are present in the old, but not the new package instance + preserve_libs = old_libs.difference(mylibs) + + # ignore any libs that are only internally used by the package + for lib in preserve_libs.copy(): + old_contents_without_libs = [x for x in old_contents if os.path.basename(x) not in preserve_libs] + if len(set(libmap[lib]).intersection(old_contents_without_libs)) == len(libmap[lib]): + preserve_libs.remove(lib) + + # get the real paths for the libs + preserve_paths = [x for x in old_contents if os.path.basename(x) in preserve_libs] + + # inject files that should be preserved into our image dir + import shutil + for x in preserve_paths: + print "injecting %s into %s" % (x, srcroot) + mydir = os.path.join(srcroot, os.path.dirname(x)) + if not os.path.exists(mydir): + os.makedirs(mydir) + + # resolve symlinks and extend preserve list + # NOTE: we're extending the list in the loop to emulate recursion to + # also get indirect symlinks + if os.path.islink(x): + linktarget = os.readlink(x) + os.symlink(linktarget, os.path.join(srcroot, x.lstrip(os.sep))) + if linktarget[0] != os.sep: + linktarget = os.path.join(os.path.dirname(x), linktarget) + preserve_paths.append(linktarget) + else: + shutil.copy2(x, os.path.join(srcroot, x.lstrip(os.sep))) + + # check for package collisions if "collision-protect" in self.settings.features: collision_ignore = set([normalize_path(myignore) for myignore in \ self.settings.get("COLLISION_IGNORE", "").split()]) - myfilelist = listdir(srcroot, recursive=1, filesonly=1, followSymlinks=False) # the linkcheck only works if we are in srcroot mycwd = os.getcwd() os.chdir(srcroot) - mysymlinks = filter(os.path.islink, listdir(srcroot, recursive=1, filesonly=0, followSymlinks=False)) - myfilelist.extend(mysymlinks) mysymlinked_directories = [s + os.path.sep for s in mysymlinks] del mysymlinks @@ -1442,6 +1525,9 @@ class dblink(object): writedict(cfgfiledict, conf_mem_file) del conf_mem_file + # regenerate reverse NEEDED map + self.vartree.dbapi.update_library_map() + #do postinst script a = doebuild(myebuild, "postinst", destroot, self.settings, use_cache=0, tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) -- 2.26.2