From: Fabian Groffen Date: Sun, 18 Nov 2007 21:08:28 +0000 (-0000) Subject: Merged from trunk 8515:8521 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=8ce02504695c273f416d281faa9c3e65ec6453dc;p=portage.git Merged from trunk 8515:8521 | 8516 | Speed up LibraryPackageMap.update() by pulling NEEDED from | | zmedico | the vardbapi.aux_get() cache. This greatly reduces the | | | amount of needless IO that has been happening just prior to | | | the postinst phase in dblink.treewalk(). We may want to | | | consider having LibraryPackageMap work directly from the | | | vardbapi instead of creating the intermediate | | | /var/cache/edb/library_consumers file. | | 8518 | Clean up vardbapi.counter_tick_core() and replace shell code | | zmedico | with pure python. | | 8519 | Fix --usepkgonly code in depgraph.select_files() so that it | | zmedico | only operates on the Package type that it expects. | | 8520 | Fix NEEDED logic in vardbapi.aux_get() so that empty NEEDED | | zmedico | doesn't trigger unnecessary cache pulls. | | 8521 | Bug #198398 - Enable Manifest generation when there are no | | zmedico | hash functions available for types such as RMD160. Now it's | | | possible to generate a Manifest when python-2.5 is built | | | without ssl support. MANIFEST1_REQUIRED_HASH and | | | MANIFEST2_REQUIRED_HASH are the only two functions that are | | | strictly required now (MD5 and SHA1). | svn path=/main/branches/prefix/; revision=8523 --- diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 5a1c8929c..f8d355448 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1993,6 +1993,8 @@ class depgraph(object): missing=0 if "--usepkgonly" in self.myopts: for xs in self.digraph.all_nodes(): + if not isinstance(xs, Package): + continue if len(xs) >= 4 and xs[0] != "binary" and xs[3] == "merge": if missing == 0: print diff --git a/pym/portage/const.py b/pym/portage/const.py index 571e18932..8e9ccbc01 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -73,6 +73,8 @@ EAPI = 1 HASHING_BLOCKSIZE = 32768 MANIFEST1_HASH_FUNCTIONS = ["MD5","SHA256","RMD160"] MANIFEST2_HASH_FUNCTIONS = ["SHA1","SHA256","RMD160"] + +MANIFEST1_REQUIRED_HASH = "MD5" MANIFEST2_REQUIRED_HASH = "SHA1" MANIFEST2_IDENTIFIERS = ["AUX","MISC","DIST","EBUILD"] diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 548fda91d..7f90a8e33 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -146,8 +146,9 @@ class LibraryPackageMap(object): def update(self): """ Update the global library->consumer map for the given vdb instance. """ obj_dict = {} + aux_get = self._dbapi.aux_get for cpv in self._dbapi.cpv_all(): - needed_list = grabfile(self._dbapi.getpath(cpv, "NEEDED")) + needed_list = aux_get(cpv, ["NEEDED"])[0].splitlines() for l in needed_list: mysplit = l.split() if len(mysplit) < 2: @@ -517,6 +518,12 @@ class vardbapi(dbapi): cache_valid = cache_mtime == mydir_mtime if cache_valid: cache_incomplete = self._aux_cache_keys.difference(metadata) + needed = metadata.get("NEEDED") + if needed is None or needed and "\n" not in needed: + # Cached value has whitespace filtered, so it has to be pulled + # again. This is temporary migration code which can be removed + # later, since it only affects users who are running trunk. + cache_incomplete.add("NEEDED") if cache_incomplete: # Allow self._aux_cache_keys to change without a cache version # bump and efficiently recycle partial cache whenever possible. @@ -558,7 +565,8 @@ class vardbapi(dbapi): myd = myf.read() finally: myf.close() - myd = " ".join(myd.split()) + if x != "NEEDED": + myd = " ".join(myd.split()) except IOError: myd = "" if x == "EAPI" and not myd: @@ -591,58 +599,38 @@ class vardbapi(dbapi): def counter_tick_core(self, myroot, incrementing=1, mycpv=None): "This method will grab the next COUNTER value and record it back to the global file. Returns new counter value." cpath = os.path.join(myroot, CACHE_PATH.lstrip(os.sep), "counter") - changed = 0 - min_counter = 0 - if mycpv: - mysplit = pkgsplit(mycpv) - for x in self.match(mysplit[0], use_cache=0): - if x == mycpv: - continue - try: - old_counter = long(self.aux_get(x, ["COUNTER"])[0]) - writemsg("COUNTER '%d' '%s'\n" % (old_counter, x), 1) - except (ValueError, KeyError): # valueError from long(), KeyError from aux_get - old_counter = 0 - writemsg("!!! BAD COUNTER in '%s'\n" % (x), noiselevel=-1) - if old_counter > min_counter: - min_counter = old_counter - - # We write our new counter value to a new file that gets moved into - # place to avoid filesystem corruption. - find_counter = ("find '%s' -type f -name COUNTER | " + \ - "while read f; do echo $(<\"${f}\"); done | " + \ - "sort -n | tail -n1") % os.path.join(self.root, VDB_PATH) - if os.path.exists(cpath): + changed = False + counter = -1 + try: cfile = open(cpath, "r") - try: - counter = long(cfile.readline()) - except (ValueError,OverflowError): - try: - counter = long(commands.getoutput(find_counter).strip()) - writemsg("!!! COUNTER was corrupted; resetting to value of %d\n" % counter, - noiselevel=-1) - changed=1 - except (ValueError, OverflowError): - writemsg("!!! COUNTER data is corrupt in pkg db. The values need to be\n", - noiselevel=-1) - writemsg("!!! corrected/normalized so that portage can operate properly.\n", - noiselevel=-1) - writemsg("!!! A simple solution is not yet available so try #gentoo on IRC.\n") - sys.exit(2) - cfile.close() + except EnvironmentError: + writemsg("!!! COUNTER file is missing: '%s'\n" % cpath, + noiselevel=-1) else: try: - counter = long(commands.getoutput(find_counter).strip()) - writemsg("!!! Global counter missing. Regenerated from counter files to: %s\n" % counter, + try: + counter = long(cfile.readline().strip()) + finally: + cfile.close() + except (OverflowError, ValueError): + writemsg("!!! COUNTER file is corrupt: '%s'\n" % cpath, noiselevel=-1) - except ValueError: # Value Error for long(), probably others for commands.getoutput - writemsg("!!! Initializing global counter.\n", noiselevel=-1) - counter = long(0) - changed = 1 - if counter < min_counter: - counter = min_counter + 1000 - changed = 1 + if counter < 0: + changed = True + max_counter = 0 + cp_list = self.cp_list + for cp in self.cp_all(): + for cpv in cp_list(cp): + try: + counter = int(self.aux_get(cpv, ["COUNTER"])[0]) + except (KeyError, OverflowError, ValueError): + continue + if counter > max_counter: + max_counter = counter + counter = max_counter + writemsg("!!! Initializing COUNTER to " + \ + "value of %d\n" % counter, noiselevel=-1) if incrementing or changed: diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 93182f764..82ca37efb 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -107,11 +107,16 @@ class Manifest(object): a Manifest (not needed for parsing and checking sums).""" self.pkgdir = pkgdir.rstrip(os.sep) + os.sep self.fhashdict = {} - self.hashes = portage.const.MANIFEST2_HASH_FUNCTIONS[:] - self.hashes.append("size") + self.hashes = set() + self.hashes.update(portage.const.MANIFEST2_HASH_FUNCTIONS) if manifest1_compat: - self.hashes.extend(portage.const.MANIFEST1_HASH_FUNCTIONS) - self.hashes = set(self.hashes) + self.hashes.update(portage.const.MANIFEST1_HASH_FUNCTIONS) + self.hashes.difference_update(hashname for hashname in \ + list(self.hashes) if hashname not in hashfunc_map) + self.hashes.add("size") + if manifest1_compat: + self.hashes.add(portage.const.MANIFEST1_REQUIRED_HASH) + self.hashes.add(portage.const.MANIFEST2_REQUIRED_HASH) for t in portage.const.MANIFEST2_IDENTIFIERS: self.fhashdict[t] = {} self.compat = manifest1_compat @@ -327,11 +332,16 @@ class Manifest(object): if self.compat: cvp_list = self.fetchlist_dict.keys() cvp_list.sort() + manifest1_hashes = set(hashname for hashname in \ + portage.const.MANIFEST1_HASH_FUNCTIONS \ + if hashname in hashfunc_map) + manifest1_hashes.add(portage.const.MANIFEST1_REQUIRED_HASH) + manifest1_hashes.add("size") for cpv in cvp_list: digest_path = os.path.join("files", "digest-%s" % self._catsplit(cpv)[1]) dname = os.path.join(self.pkgdir, digest_path) try: - myhashes = perform_multiple_checksums(dname, portage.const.MANIFEST1_HASH_FUNCTIONS+["size"]) + myhashes = perform_multiple_checksums(dname, manifest1_hashes) myhashkeys = myhashes.keys() myhashkeys.sort() for h in myhashkeys: