Bug #197965 - When fetch() encounters a checksum failure,

look for an identical file from a previous checksum failure
in order to avoid consuming unnecessary disk space with
duplicate files.

svn path=/main/trunk/; revision=8401

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index d609d2b6ede1f0aaf97f42b203d84ea3e26f6893..3597fa3f829afe451be0f2df719a4df07a282bd3 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -2717,6 +2717,45 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero
                return retval >> 8
        return retval
 
+def _checksum_failure_temp_file(distdir, basename):
+       """
+       First try to find a duplicate temp file with the same checksum and return
+       that filename if available. Otherwise, use mkstemp to create a new unique
+       filename._checksum_failure_.$RANDOM, rename the given file, and return the
+       new filename. In any case, filename will be renamed or removed before this
+       function returns a temp filename.
+       """
+
+       filename = os.path.join(distdir, basename)
+       size = os.stat(filename).st_size
+       checksum = None
+       tempfile_re = re.compile(re.escape(basename) + r'\._checksum_failure_\..*')
+       for temp_filename in os.listdir(distdir):
+               if not tempfile_re.match(temp_filename):
+                       continue
+               temp_filename = os.path.join(distdir, temp_filename)
+               try:
+                       if size != os.stat(temp_filename).st_size:
+                               continue
+               except OSError:
+                       continue
+               try:
+                       temp_checksum = portage.checksum.perform_md5(temp_filename)
+               except portage.exception.FileNotFound:
+                       # Apparently the temp file disappeared. Let it go.
+                       continue
+               if checksum is None:
+                       checksum = portage.checksum.perform_md5(filename)
+               if checksum == temp_checksum:
+                       os.unlink(filename)
+                       return temp_filename
+
+       from tempfile import mkstemp
+       fd, temp_filename = mkstemp("", basename + "._checksum_failure_.", distdir)
+       os.close(fd)
+       os.rename(filename, temp_filename)
+       return temp_filename
+
 def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
        "fetch files.  Will use digest file if available."
 
@@ -2888,6 +2927,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                        noiselevel=-1)
                                return 0
                        del distlocks_subdir
+
        for myfile in filedict:
                """
                fetched  status
@@ -2976,12 +3016,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                                if reason[0] == "Insufficient data for checksum verification":
                                                                        return 0
                                                                if can_fetch and not restrict_fetch:
-                                                                       from tempfile import mkstemp
-                                                                       fd, temp_filename = mkstemp("",
-                                                                               myfile + "._checksum_failure_.",
-                                                                               mysettings["DISTDIR"])
-                                                                       os.close(fd)
-                                                                       os.rename(myfile_path, temp_filename)
+                                                                       temp_filename = \
+                                                                               _checksum_failure_temp_file(
+                                                                               mysettings["DISTDIR"], myfile)
                                                                        writemsg_stdout("Refetching... " + \
                                                                                "File renamed to '%s'\n\n" % \
                                                                                temp_filename, noiselevel=-1)
@@ -3158,12 +3195,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                                                (reason[1], reason[2]), noiselevel=-1)
                                                                        if reason[0] == "Insufficient data for checksum verification":
                                                                                return 0
-                                                                       from tempfile import mkstemp
-                                                                       fd, temp_filename = mkstemp("",
-                                                                               myfile + "._checksum_failure_.",
-                                                                               mysettings["DISTDIR"])
-                                                                       os.close(fd)
-                                                                       os.rename(myfile_path, temp_filename)
+                                                                       temp_filename = \
+                                                                               _checksum_failure_temp_file(
+                                                                               mysettings["DISTDIR"], myfile)
                                                                        writemsg_stdout("Refetching... " + \
                                                                                "File renamed to '%s'\n\n" % \
                                                                                temp_filename, noiselevel=-1)