Skip whirlpool digest check when unaccelerated.

If multiple digests are available and whirlpool is unaccelerated, then
skip it. This allows extreme performance problems like bug #425046 to
be avoided whenever practical, especially for cases like stage builds
where acceleration may not be available for some hashes due to
minimization of dependencies.

diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index bd416ac12890fc6e32632819e06b37bc96d21558..daf4a0cbf037c510205d7099c8a7817313dc0fdf 100644 (file)
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -137,8 +137,10 @@ try:
 except ImportError:
        pass
 
+_whirlpool_unaccelerated = False
 if "WHIRLPOOL" not in hashfunc_map:
        # Bundled WHIRLPOOL implementation
+       _whirlpool_unaccelerated = True
        from portage.util.whirlpool import new as _new_whirlpool
        whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool, origin="bundled")
 
@@ -197,6 +199,24 @@ def get_hash_origin(hashtype):
                raise KeyError(hashtype)
        return hashorigin_map.get(hashtype, "unknown")
 
+def _filter_unaccelarated_hashes(digests):
+       """
+       If multiple digests are available and some are unaccelerated,
+       then return a new dict that omits the unaccelerated ones. This
+       allows extreme performance problems like bug #425046 to be
+       avoided whenever practical, especially for cases like stage
+       builds where acceleration may not be available for some hashes
+       due to minimization of dependencies.
+       """
+       if _whirlpool_unaccelerated and "WHIRLPOOL" in digests:
+               verifiable_hash_types = set(digests).intersection(hashfunc_map)
+               verifiable_hash_types.discard("size")
+               if len(verifiable_hash_types) > 1:
+                       digests = dict(digests)
+                       digests.pop("WHIRLPOOL")
+
+       return digests
+
 def verify_all(filename, mydict, calc_prelink=0, strict=0):
        """
        Verify all checksums against a file.

diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 90324eebe1508137746485b80f698170c43e7c9e..ab9186245651202d683ce5deb00377b6c9c8c328 100644 (file)
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -8,7 +8,8 @@ import warnings
 
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
-       'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all',
+       'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
+               'verify_all,_filter_unaccelarated_hashes',
        'portage.util:write_atomic',
 )
 
@@ -508,7 +509,8 @@ class Manifest(object):
        def checkFileHashes(self, ftype, fname, ignoreMissing=False):
                myhashes = self.fhashdict[ftype][fname]
                try:
-                       ok,reason = verify_all(self._getAbsname(ftype, fname), self.fhashdict[ftype][fname])
+                       ok, reason = verify_all(self._getAbsname(ftype, fname),
+                               _filter_unaccelarated_hashes(self.fhashdict[ftype][fname]))
                        if not ok:
                                raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason)))
                        return ok, reason

diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
index b795b282604c42f44fe74462333c20fcc4c44cd7..60ed04da28c119b728ff4e92025f7a945695bf65 100644 (file)
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -25,7 +25,8 @@ portage.proxy.lazyimport.lazyimport(globals(),
 
 from portage import OrderedDict, os, selinux, shutil, _encodings, \
        _shell_quote, _unicode_encode
-from portage.checksum import hashfunc_map, perform_md5, verify_all
+from portage.checksum import (hashfunc_map, perform_md5, verify_all,
+       _filter_unaccelarated_hashes)
 from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
        GLOBAL_CONFIG_PATH
 from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
@@ -210,6 +211,7 @@ def _check_distfile(filename, digests, eout, show_errors=1):
                        # Zero-byte distfiles are always invalid.
                        return (False, st)
        else:
+               digests = _filter_unaccelarated_hashes(digests)
                if _check_digests(filename, digests, show_errors=show_errors):
                        eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
                                " ".join(sorted(digests))))
@@ -793,8 +795,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
                                                        eout.eend(0)
                                                        continue
                                                else:
-                                                       verified_ok, reason = verify_all(
-                                                               myfile_path, mydigests[myfile])
+                                                       digests = _filter_unaccelarated_hashes(mydigests[myfile])
+                                                       verified_ok, reason = verify_all(myfile_path, digests)
                                                        if not verified_ok:
                                                                writemsg(_("!!! Previously fetched"
                                                                        " file: '%s'\n") % myfile, noiselevel=-1)
@@ -816,7 +818,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
                                                                eout = EOutput()
                                                                eout.quiet = \
                                                                        mysettings.get("PORTAGE_QUIET", None) == "1"
-                                                               digests = mydigests.get(myfile)
                                                                if digests:
                                                                        digests = list(digests)
                                                                        digests.sort()
@@ -1051,7 +1052,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
                                                                # file NOW, for those users who don't have a stable/continuous
                                                                # net connection. This way we have a chance to try to download
                                                                # from another mirror...
-                                                               verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile])
+                                                               digests = _filter_unaccelarated_hashes(mydigests[myfile])
+                                                               verified_ok, reason = verify_all(myfile_path, digests)
                                                                if not verified_ok:
                                                                        writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
                                                                                noiselevel=-1)
@@ -1085,7 +1087,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
                                                                else:
                                                                        eout = EOutput()
                                                                        eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
-                                                                       digests = mydigests.get(myfile)
                                                                        if digests:
                                                                                eout.ebegin("%s %s ;-)" % \
                                                                                        (myfile, " ".join(sorted(digests))))