From e2983690e7a22d69cb1507b55050438a6ec3d924 Mon Sep 17 00:00:00 2001
From: Zac Medico <zmedico@gentoo.org>
Date: Thu, 3 Apr 2008 19:23:16 +0000
Subject: [PATCH] Implement a PORTAGE_FETCH_RESUME_MIN_SIZE variable that is
 useful for helping to ensure that small garbage files such as html 404 pages
 are properly discarded.

svn path=/main/trunk/; revision=9687
---
 cnf/make.globals        |  3 +++
 man/make.conf.5         |  8 ++++++
 pym/portage/__init__.py | 60 ++++++++++++++++++++++++++++++++++++++---
 3 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/cnf/make.globals b/cnf/make.globals
index 36d8591ae..5e8150131 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -59,6 +59,9 @@ AUTOCLEAN="yes"
 # Number of mirrors to try when a downloaded file has an incorrect checksum.
 PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5"
 
+# Minimum size of existing file for RESUMECOMMAND to be called.
+PORTAGE_FETCH_RESUME_MIN_SIZE="350K"
+
 # Number of times 'emerge --sync' will run before giving up.
 PORTAGE_RSYNC_RETRIES="3"
 
diff --git a/man/make.conf.5 b/man/make.conf.5
index 18d750a21..d35acf16b 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -412,6 +412,14 @@ Please see /etc/make.conf.example for elog documentation.
 \fBPORTAGE_FETCH_CHECKSUM_TRY_MIRRORS\fR = \fI5\fR
 Number of mirrors to try when a downloaded file has an incorrect checksum.
 .TP
+\PORTAGE_FETCH_RESUME_MIN_SIZE\fR = \fI350K\fR
+Minimum size of existing file for RESUMECOMMAND to be called. Files smaller
+than this size will be removed and FETCHCOMMAND will be called to download
+the file from the beginning. This is useful for helping to ensure that small
+garbage files such as html 404 pages are properly discarded. The variable
+should contain an integer number of bytes and may have a suffix such as
+K, M, or G.
+.TP
 \fBPORTAGE_NICENESS\fR = \fI[number]\fR
 The value of this variable will be added to the current nice level that
 emerge is running at.  In other words, this will not set the nice level,
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index a64e90c5c..4233c2eea 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -938,7 +938,8 @@ class config(object):
 		"PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES",
 		"PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
 		"PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
-		"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR",
+		"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
+		"PORTAGE_GPG_DIR",
 		"PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
 		"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
 		"PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
@@ -3010,6 +3011,20 @@ def _checksum_failure_temp_file(distdir, basename):
 	os.rename(filename, temp_filename)
 	return temp_filename
 
+_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
+
+_size_suffix_map = {
+	''  : 0,
+	'K' : 10,
+	'M' : 20,
+	'G' : 30,
+	'T' : 40,
+	'P' : 50,
+	'E' : 60,
+	'Z' : 70,
+	'Y' : 80,
+}
+
 def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
 	"fetch files.  Will use digest file if available."
 
@@ -3050,6 +3065,26 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 	checksum_failure_max_tries = v
 	del v
 
+	fetch_resume_size_default = "350K"
+	fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
+	if fetch_resume_size is not None:
+		fetch_resume_size = "".join(fetch_resume_size.split())
+		match = _fetch_resume_size_re.match(fetch_resume_size)
+		if match is None or \
+			(match.group(2).upper() not in _size_suffix_map):
+			writemsg("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE" + \
+				" contains an unrecognized format: '%s'\n" % \
+				mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
+			writemsg("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE " + \
+				"default value: %s\n" % fetch_resume_size_default,
+				noiselevel=-1)
+			fetch_resume_size = None
+	if fetch_resume_size is None:
+		fetch_resume_size = fetch_resume_size_default
+		match = _fetch_resume_size_re.match(fetch_resume_size)
+	fetch_resume_size = int(match.group(1)) * \
+		2 ** _size_suffix_map[match.group(2)]
+
 	# Behave like the package has RESTRICT="primaryuri" after a
 	# couple of checksum failures, to increase the probablility
 	# of success before checksum_failure_max_tries is reached.
@@ -3405,8 +3440,27 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 
 				if fetched != 2 and has_space:
 					#we either need to resume or start the download
-					#you can't use "continue" when you're inside a "try" block
-					if fetched==1:
+					if fetched == 1:
+						try:
+							mystat = os.stat(myfile_path)
+						except OSError, e:
+							if e.errno != errno.ENOENT:
+								raise
+							del e
+							fetched = 0
+						else:
+							if mystat.st_size < fetch_resume_size:
+								writemsg((">>> Deleting distfile with size " + \
+									"%d (smaller than " "PORTAGE_FETCH_RESU" + \
+									"ME_MIN_SIZE)\n") % mystat.st_size)
+								try:
+									os.unlink(myfile_path)
+								except OSError, e:
+									if e.errno != errno.ENOENT:
+										raise
+									del e
+								fetched = 0
+					if fetched == 1:
 						#resume mode:
 						writemsg(">>> Resuming download...\n")
 						locfetch=resumecommand
-- 
2.26.2