From c4bcc89af0a6b1b008844f1e446a79a98b6128aa Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 8 Nov 2006 07:21:21 +0000 Subject: [PATCH] Merged from trunk 4965:4984 Minimize getvirtuals() calls and dict lookups inside dep_virtual(). Fix validate_blockers() so that a package doesn't block itself via a virtual that it provides. Print the number of upgrades, downgrades, etc. next to size of downloads. Thanks to Bo ?\195?\152rsted Andresen for the initial patch from bug #153847. Thanks to Chris White for these portage_checksum docstrings. Thanks to Bo ?\195?\152rsted Andresen for this output fix for bug #153847. Plug some holes in the blocker validation logic so that nothing can slip through. Remove some stray blocker validation logic. s/grabdict/grabdict_package/ for bug #63429. For bug #154207, add a note about package.provided to the depclean warning message. Add --newuse back to the suggested options in the depclean warning message, since it's still needed in some cases. svn path=/main/branches/prefix/; revision=4986 --- bin/emerge | 104 +++++++++++++++++++++++++++++++++------- pym/portage.py | 15 +++--- pym/portage_checksum.py | 51 ++++++++++++++++++++ 3 files changed, 146 insertions(+), 24 deletions(-) diff --git a/bin/emerge b/bin/emerge index 4f6628e7f..426a29439 100755 --- a/bin/emerge +++ b/bin/emerge @@ -1221,13 +1221,6 @@ class depgraph: for x in mymerge: selected_pkg = None if x[0]=="!": - # if this package is myself, don't append it to block list. - if myparent: - mydep = x[1:] - if mydep == portage.dep_getkey(mydep) and \ - mydep == portage.dep_getkey(myparent.split()[2]): - continue - # adding block selected_pkg = ["blocks", myroot, x[1:], None] else: #We are not processing a blocker but a normal dependency @@ -1450,13 +1443,13 @@ class depgraph: blocked_after_update = self.mydbapi[myroot].match(mydep) if not blocked_pkgs and not blocked_after_update: del self.blocker_parents[blocker] - elif not blocked_after_update: + else: """It may be possible to circumvent this block via correct ordering of upgrades. If necessary, create hard deps to enforce correct merge order.""" fakedb = self.mydbapi[myroot] new_pkgs = [] - unresolveable = False + unresolvable = False for cpv in blocked_pkgs: myslot = vardb.aux_get(cpv, ["SLOT"])[0] myslot_atom = "%s:%s" % (portage.dep_getkey(cpv), myslot) @@ -1467,9 +1460,10 @@ class depgraph: """There's an installed package that's blocked and there's no upgrade found to invalidate it, so leave this blocker in the digraph.""" - unresolveable = True + unresolvable = True break - if unresolveable: + if unresolvable: + # Keep trying to invalidate as many blockers as possible. continue for parent in list(self.blocker_parents[blocker]): @@ -1477,11 +1471,17 @@ class depgraph: pdbapi = self.trees[proot][self.pkg_tree_map[ptype]].dbapi pslot = pdbapi.aux_get(pcpv, ["SLOT"])[0] pslot_atom = "%s:%s" % (portage.dep_getkey(pcpv), pslot) + depends_on_merge_order = False for myslot_atom, pkg in new_pkgs: if pslot_atom == myslot_atom: """A merge within a slot invalidates the block, so the order does not need to be enforced.""" - self.blocker_parents[blocker].remove(parent) + continue + if pkg in blocked_after_update: + """This isn't a case of a package blocking itself, + and the block is still valid in the final state, so + this one is unresolvable.""" + unresolvable = True break # Enforce correct merge order with a hard dep. node = self.pkg_node_map[pkg] @@ -1490,6 +1490,18 @@ class depgraph: """Count references to this blocker so that it can be invalidated after nodes referencing it have been merged.""" self.blocker_digraph.addnode(node, blocker) + depends_on_merge_order = True + if unresolvable: + break + elif not depends_on_merge_order: + self.blocker_parents[blocker].remove(parent) + if unresolvable: + """This blocker can not be solved, so make sure that it is + removed from the digraph if it has already been added.""" + if self.blocker_digraph.contains(blocker): + self.blocker_digraph.remove(blocker) + # Keep trying to invalidate as many blockers as possible. + continue if not self.blocker_parents[blocker]: del self.blocker_parents[blocker] # Validate blockers that depend on merge order. @@ -1746,7 +1758,8 @@ class depgraph: "--verbose" in self.myopts and 3 or 2) changelogs=[] p=[] - totalsize=0 + + counters = PackageCounters() if verbosity == 1 and "--verbose" not in self.myopts: def create_use_string(*args): @@ -1869,6 +1882,7 @@ class depgraph: if x[0]=="blocks": addl=""+red("B")+" "+fetch+" " + counters.blocks += 1 resolved = self.trees[x[1]]["vartree"].resolve_key(x[2]) print "["+x[0]+" "+addl+"]",red(resolved), block_parents = self.blocker_parents[" ".join(x)] @@ -1914,6 +1928,8 @@ class depgraph: myoldbest="" if self.trees[x[1]]["vartree"].exists_specific(pkg_key): addl=" "+yellow("R")+fetch+" " + if x[3] != "nomerge": + counters.reinst += 1 elif self.trees[x[1]]["vartree"].exists_specific_cat(pkg_key): mynewslot = mydbapi.aux_get(pkg_key, ["SLOT"])[0] myoldlist = self.trees[x[1]]["vartree"].dbapi.match( @@ -1926,12 +1942,15 @@ class depgraph: if portage.pkgcmp(portage.pkgsplit(x[2]), portage.pkgsplit(myoldbest)) < 0: # Downgrade in slot addl+=turquoise("U")+blue("D") + counters.downgrades += 1 else: # Update in slot addl+=turquoise("U")+" " + counters.upgrades += 1 else: # New slot, mark it new. addl=" "+green("NS")+fetch+" " + counters.newslot += 1 if "--changelog" in self.myopts: changelogs.extend(self.calc_changelog( @@ -1940,6 +1959,7 @@ class depgraph: '/'.join(portage.catpkgsplit(x[2])[:2])), x[2])) else: addl=" "+green("N")+" "+fetch+" " + counters.new += 1 verboseadd="" @@ -2037,7 +2057,7 @@ class depgraph: if myfetchfile not in myfetchlist: mysize+=myfilesdict[myfetchfile] myfetchlist.append(myfetchfile) - totalsize+=mysize + counters.totalsize += mysize verboseadd+=format_size(mysize)+" " # overlay verbose @@ -2150,7 +2170,7 @@ class depgraph: if verbosity == 3: print - print "Total size of downloads: "+format_size(totalsize) + print counters if overlays and display_overlays: print "Portage overlays:" y=0 @@ -2215,6 +2235,54 @@ class depgraph: def outdated(self): return self.outdatedpackages +class PackageCounters(object): + + def __init__(self): + self.upgrades = 0 + self.downgrades = 0 + self.new = 0 + self.newslot = 0 + self.reinst = 0 + self.blocks = 0 + self.totalsize = 0 + + def __str__(self): + total_installs = self.upgrades + self.downgrades + self.newslot + self.new + self.reinst + myoutput = [] + details = [] + myoutput.append("Total: %s package" % total_installs) + if total_installs != 1: + myoutput.append("s") + if total_installs != 0: + myoutput.append(" (") + if self.upgrades > 0: + details.append("%s upgrade" % self.upgrades) + if self.upgrades > 1: + details[-1] += "s" + if self.downgrades > 0: + details.append("%s downgrade" % self.downgrades) + if self.downgrades > 1: + details[-1] += "s" + if self.new > 0: + details.append("%s new" % self.new) + if self.newslot > 0: + details.append("%s in new slot" % self.newslot) + if self.newslot > 1: + details[-1] += "s" + if self.reinst > 0: + details.append("%s reinstall" % self.reinst) + if self.reinst > 1: + details[-1] += "s" + if self.blocks > 0: + details.append("%s block" % self.blocks) + if self.blocks > 1: + details[-1] += "s" + myoutput.append(", ".join(details)) + if total_installs != 0: + myoutput.append(")") + myoutput.append(", Size of downloads: %s" % format_size(self.totalsize)) + return "".join(myoutput) + class MergeTask(object): def __init__(self, settings, trees, myopts): @@ -3740,12 +3808,14 @@ def action_depclean(settings, trees, ldpath_mtimes, print warn_prefix + "Also study the list of packages to be cleaned for any obvious" print warn_prefix + "mistakes. Packages that are part of the world set will always" print warn_prefix + "be kept. They can be manually added to this set with" - print warn_prefix + good("`emerge --noreplace `") + "." + print warn_prefix + good("`emerge --noreplace `") + ". Packages that are listed in" + print warn_prefix + "package.provided (see portage(5)) will be removed by" + print warn_prefix + "depclean, even if they are part of the world set." print warn_prefix print warn_prefix + "As a safety measure, depclean will not remove any packages" print warn_prefix + "unless *all* required dependencies have been resolved. As a" print warn_prefix + "consequence, it is often necessary to run " - print warn_prefix + good("`emerge --update --deep world`") + " prior to depclean." + print warn_prefix + good("`emerge --update --newuse --deep world`") + " prior to depclean." xterm_titles = "notitles" not in settings.features portdb = trees[settings["ROOT"]]["porttree"].dbapi diff --git a/pym/portage.py b/pym/portage.py index e20e709e8..68bf31953 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -1830,7 +1830,7 @@ class config: return self.virtuals self.dirVirtuals = stack_dictlist( - [grabdict(os.path.join(x, "virtuals")) \ + [grabdict_package(os.path.join(x, "virtuals")) \ for x in self.profiles], incremental=True) for virt in self.dirVirtuals: @@ -3460,23 +3460,24 @@ def getCPFromCPV(mycpv): def dep_virtual(mysplit, mysettings): "Does virtual dependency conversion" newsplit=[] + myvirtuals = mysettings.getvirtuals() for x in mysplit: if type(x)==types.ListType: newsplit.append(dep_virtual(x, mysettings)) else: mykey=dep_getkey(x) - myvirtuals = mysettings.getvirtuals() - if myvirtuals.has_key(mykey): - if len(myvirtuals[mykey]) == 1: - a = string.replace(x, mykey, myvirtuals[mykey][0]) + mychoices = myvirtuals.get(mykey, None) + if mychoices: + if len(mychoices) == 1: + a = x.replace(mykey, mychoices[0]) else: if x[0]=="!": # blocker needs "and" not "or(||)". a=[] else: a=['||'] - for y in myvirtuals[mykey]: - a.append(string.replace(x, mykey, y)) + for y in mychoices: + a.append(x.replace(mykey, y)) newsplit.append(a) else: newsplit.append(x) diff --git a/pym/portage_checksum.py b/pym/portage_checksum.py index 699652514..6ba58e75c 100644 --- a/pym/portage_checksum.py +++ b/pym/portage_checksum.py @@ -85,6 +85,22 @@ def get_valid_checksum_keys(): return hashfunc_map.keys() def verify_all(filename, mydict, calc_prelink=0, strict=0): + """ + Verify all checksums against a file. + + @param filename: File to run the checksums against + @type filename: String + @param calc_prelink: Whether or not to reverse prelink before running the checksum + @type calc_prelink: Integer + @param strict: Enable/Disable strict checking (which stops exactly at a checksum failure and throws an exception) + @type strict: Integer + @rtype: Tuple + @return: Result of the checks and possible message: + 1) If size fails, False, and a tuple containing a message, the given size, and the actual size + 2) If there is an os error, False, and a tuple containing the system error followed by 2 nulls + 3) If a checksum fails, False and a tuple containing a message, the given hash, and the actual hash + 4) If all checks succeed, return True and a fake reason + """ # Dict relates to single file only. # returns: (passed,reason) file_is_ok = True @@ -112,6 +128,15 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0): return file_is_ok,reason def pyhash(filename, hashobject): + """ + Run a checksum against a file. + + @param filename: File to run the checksum against + @type filename: String + @param hashname: The hash object that will execute the checksum on the file + @type hashname: Object + @return: The hash and size of the data + """ f = open(filename, 'rb') blocksize = HASHING_BLOCKSIZE data = f.read(blocksize) @@ -126,6 +151,18 @@ def pyhash(filename, hashobject): return (sum.hexdigest(), size) def perform_checksum(filename, hashname="MD5", calc_prelink=0): + """ + Run a specific checksum against a file. + + @param filename: File to run the checksum against + @type filename: String + @param hashname: The type of hash function to run + @type hashname: String + @param calc_prelink: Whether or not to reverse prelink before running the checksum + @type calc_prelink: Integer + @rtype: Tuple + @return: The hash and size of the data + """ myfilename = filename[:] prelink_tmpfile = os.path.join("/", PRIVATE_PATH, "prelink-checksum.tmp." + str(os.getpid())) mylock = None @@ -160,6 +197,20 @@ def perform_checksum(filename, hashname="MD5", calc_prelink=0): return (myhash,mysize) def perform_multiple_checksums(filename, hashes=["MD5"], calc_prelink=0): + """ + Run a group of checksums against a file. + + @param filename: File to run the checksums against + @type filename: String + @param hashes: A list of checksum functions to run against the file + @type hashname: List + @param calc_prelink: Whether or not to reverse prelink before running the checksum + @type calc_prelink: Integer + @rtype: Tuple + @return: A dictionary in the form: + return_value[hash_name] = (hash_result,size) + for each given checksum + """ rVal = {} for x in hashes: if x not in hashfunc_map: -- 2.26.2