Implement a PORTAGE_FETCH_RESUME_MIN_SIZE variable that is useful for

helping to ensure that small garbage files such as html 404 pages are
properly discarded.

svn path=/main/trunk/; revision=9687

diff --git a/cnf/make.globals b/cnf/make.globals
index 36d8591ae76edac173ab30769c0156bd55034bd3..5e8150131f9648bd1d8ed128e619976ba94e5493 100644 (file)
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -59,6 +59,9 @@ AUTOCLEAN="yes"
 # Number of mirrors to try when a downloaded file has an incorrect checksum.
 PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5"
 
+# Minimum size of existing file for RESUMECOMMAND to be called.
+PORTAGE_FETCH_RESUME_MIN_SIZE="350K"
+
 # Number of times 'emerge --sync' will run before giving up.
 PORTAGE_RSYNC_RETRIES="3"
 

diff --git a/man/make.conf.5 b/man/make.conf.5
index 18d750a21555cb2721a7093c0039aa313c1febd3..d35acf16b5a4ae6e7e7fc93627e80df11d654a9e 100644 (file)
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -412,6 +412,14 @@ Please see /etc/make.conf.example for elog documentation.
 \fBPORTAGE_FETCH_CHECKSUM_TRY_MIRRORS\fR = \fI5\fR
 Number of mirrors to try when a downloaded file has an incorrect checksum.
 .TP
+\PORTAGE_FETCH_RESUME_MIN_SIZE\fR = \fI350K\fR
+Minimum size of existing file for RESUMECOMMAND to be called. Files smaller
+than this size will be removed and FETCHCOMMAND will be called to download
+the file from the beginning. This is useful for helping to ensure that small
+garbage files such as html 404 pages are properly discarded. The variable
+should contain an integer number of bytes and may have a suffix such as
+K, M, or G.
+.TP
 \fBPORTAGE_NICENESS\fR = \fI[number]\fR
 The value of this variable will be added to the current nice level that
 emerge is running at.  In other words, this will not set the nice level,

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index a64e90c5c9d7cded2ab55e3a60588cb7fff4841c..4233c2eead05702b54278ce417b08825e27aa6eb 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -938,7 +938,8 @@ class config(object):
                "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES",
                "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
                "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
-               "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR",
+               "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
+               "PORTAGE_GPG_DIR",
                "PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
                "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
                "PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
@@ -3010,6 +3011,20 @@ def _checksum_failure_temp_file(distdir, basename):
        os.rename(filename, temp_filename)
        return temp_filename
 
+_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
+
+_size_suffix_map = {
+       ''  : 0,
+       'K' : 10,
+       'M' : 20,
+       'G' : 30,
+       'T' : 40,
+       'P' : 50,
+       'E' : 60,
+       'Z' : 70,
+       'Y' : 80,
+}
+
 def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
        "fetch files.  Will use digest file if available."
 
@@ -3050,6 +3065,26 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
        checksum_failure_max_tries = v
        del v
 
+       fetch_resume_size_default = "350K"
+       fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
+       if fetch_resume_size is not None:
+               fetch_resume_size = "".join(fetch_resume_size.split())
+               match = _fetch_resume_size_re.match(fetch_resume_size)
+               if match is None or \
+                       (match.group(2).upper() not in _size_suffix_map):
+                       writemsg("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE" + \
+                               " contains an unrecognized format: '%s'\n" % \
+                               mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
+                       writemsg("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE " + \
+                               "default value: %s\n" % fetch_resume_size_default,
+                               noiselevel=-1)
+                       fetch_resume_size = None
+       if fetch_resume_size is None:
+               fetch_resume_size = fetch_resume_size_default
+               match = _fetch_resume_size_re.match(fetch_resume_size)
+       fetch_resume_size = int(match.group(1)) * \
+               2 ** _size_suffix_map[match.group(2)]
+
        # Behave like the package has RESTRICT="primaryuri" after a
        # couple of checksum failures, to increase the probablility
        # of success before checksum_failure_max_tries is reached.
@@ -3405,8 +3440,27 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 
                                if fetched != 2 and has_space:
                                        #we either need to resume or start the download
-                                       #you can't use "continue" when you're inside a "try" block
-                                       if fetched==1:
+                                       if fetched == 1:
+                                               try:
+                                                       mystat = os.stat(myfile_path)
+                                               except OSError, e:
+                                                       if e.errno != errno.ENOENT:
+                                                               raise
+                                                       del e
+                                                       fetched = 0
+                                               else:
+                                                       if mystat.st_size < fetch_resume_size:
+                                                               writemsg((">>> Deleting distfile with size " + \
+                                                                       "%d (smaller than " "PORTAGE_FETCH_RESU" + \
+                                                                       "ME_MIN_SIZE)\n") % mystat.st_size)
+                                                               try:
+                                                                       os.unlink(myfile_path)
+                                                               except OSError, e:
+                                                                       if e.errno != errno.ENOENT:
+                                                                               raise
+                                                                       del e
+                                                               fetched = 0
+                                       if fetched == 1:
                                                #resume mode:
                                                writemsg(">>> Resuming download...\n")
                                                locfetch=resumecommand