From: Fabian Groffen Date: Sat, 15 Mar 2008 10:26:36 +0000 (-0000) Subject: Merged from trunk 9461:9467 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=256fda0a7767286753e6d96f22adace48c12b824;p=portage.git Merged from trunk 9461:9467 | 9462 | * Optimize parallel-fetch to avoid redundant checksum | | zmedico | verification. * Add parallel-fetch to the default FEATURES | | | since it is more efficient now. | | 9463 | Bug #42456 - When a checksum failure occurs during fetch, | | zmedico | use a new PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS config variable | | | to control how many mirrors should be tried before aborting | | | the download. This type of behavior used to be enabled only | | | in --fetchonly mode but now it's enabled for all fetch() | | | calls. | | 9466 | Fix grammar. | | zmedico | | | 9467 | Fix an UnboundLocalError for 'pkgindex'. | | zmedico | | svn path=/main/branches/prefix/; revision=9468 --- diff --git a/bin/emaint b/bin/emaint index c2e345290..0f4f7b417 100755 --- a/bin/emaint +++ b/bin/emaint @@ -172,8 +172,8 @@ class BinhostHandler(object): self._pkgindex_file, wantnewlockfile=1) try: from portage import getbinpkg - del pkgindex - self._pkgindex = getbinpkg.PackageIndex() + pkgindex = getbinpkg.PackageIndex() + self._pkgindex = pkgindex f = open(self._pkgindex_file, 'r') try: self._pkgindex.read(f) diff --git a/cnf/make.globals b/cnf/make.globals index 41bcdbc7c..6417cd767 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -33,7 +33,7 @@ FETCHCOMMAND="wget -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI RESUMECOMMAND="wget -c -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\"" # Default user options -FEATURES="distlocks metadata-transfer sandbox sfperms strict unmerge-orphans userfetch" +FEATURES="distlocks parallel-fetch metadata-transfer sandbox sfperms strict unmerge-orphans userfetch" # Enable for global testing FEATURES="${FEATURES} preserve-libs" @@ -51,6 +51,9 @@ EMERGE_WARNING_DELAY="10" # This option will be removed and forced to yes. AUTOCLEAN="yes" +# Number of mirrors to try when a downloaded file has an incorrect checksum. +PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5" + # Number of times 'emerge --sync' will run before giving up. PORTAGE_RSYNC_RETRIES="3" diff --git a/man/make.conf.5 b/man/make.conf.5 index 75bc141aa..5c8fc022b 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -402,6 +402,9 @@ This variable contains flags for the \fBPORTAGE_COMPRESS\fR command. .B PORTAGE_ELOG_MAILSUBJECT Please see /etc/make.conf.example for elog documentation. .TP +\fBPORTAGE_FETCH_CHECKSUM_TRY_MIRRORS\fR = \fI5\fR +Number of mirrors to try when a downloaded file has an incorrect checksum. +.TP \fBPORTAGE_NICENESS\fR = \fI[number]\fR The value of this variable will be added to the current nice level that emerge is running at. In other words, this will not set the nice level, diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index d1f876065..1bf636090 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -4530,6 +4530,7 @@ class MergeTask(object): fetch_env = os.environ.copy() fetch_env["FEATURES"] = fetch_env.get("FEATURES", "") + " -cvs" fetch_env["PORTAGE_NICENESS"] = "0" + fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1" fetch_args = [sys.argv[0], "--resume", "--fetchonly"] resume_opts = self.myopts.copy() # For automatic resume, we need to prevent @@ -7009,7 +7010,7 @@ def action_build(settings, trees, mtimedb, if ("--resume" in myopts): favorites=mtimedb["resume"]["favorites"] mergetask = MergeTask(settings, trees, myopts) - if "--fetchonly" in myopts: + if "PORTAGE_PARALLEL_FETCHONLY" in settings: """ parallel-fetch uses --resume --fetchonly and we don't want it to write the mtimedb""" mtimedb.filename = None diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 0dadd2d3b..1579b3d91 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -947,7 +947,8 @@ class config(object): "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_CALLER", "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", - "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", "PORTAGE_GPG_DIR", + "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", + "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR", @@ -3048,12 +3049,42 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", # every single available mirror is a waste of bandwidth # and time, so there needs to be a cap. checksum_failure_max_tries = 5 + v = checksum_failure_max_tries + try: + v = int(mysettings.get("PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", + checksum_failure_max_tries)) + except (ValueError, OverflowError): + writemsg("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS" + \ + " contains non-integer value: '%s'\n" % \ + mysettings["PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"], noiselevel=-1) + writemsg("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS " + \ + "default value: %s\n" % checksum_failure_max_tries, + noiselevel=-1) + v = checksum_failure_max_tries + if v < 1: + writemsg("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS" + \ + " contains value less than 1: '%s'\n" % v, noiselevel=-1) + writemsg("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS " + \ + "default value: %s\n" % checksum_failure_max_tries, + noiselevel=-1) + v = checksum_failure_max_tries + checksum_failure_max_tries = v + del v + # Behave like the package has RESTRICT="primaryuri" after a # couple of checksum failures, to increase the probablility # of success before checksum_failure_max_tries is reached. checksum_failure_primaryuri = 2 thirdpartymirrors = mysettings.thirdpartymirrors() + # In the background parallel-fetch process, it's safe to skip checksum + # verification of pre-existing files in $DISTDIR that have the correct + # file size. The parent process will verify their checksums prior to + # the unpack phase. + + parallel_fetchonly = fetchonly and \ + "PORTAGE_PARALLEL_FETCHONLY" in mysettings + check_config_instance(mysettings) custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"], @@ -3240,7 +3271,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if use_locks and can_fetch: waiting_msg = None - if "parallel-fetch" in features: + if not parallel_fetchonly and "parallel-fetch" in features: waiting_msg = ("Downloading '%s'... " + \ "see "+EPREFIX+"/var/log/emerge-fetch.log for details.") % myfile if locks_in_subdir: @@ -3299,6 +3330,15 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if mystat.st_size < mydigests[myfile]["size"] and \ not restrict_fetch: fetched = 1 # Try to resume this download. + elif parallel_fetchonly and \ + mystat.st_size == mydigests[myfile]["size"]: + eout = portage.output.EOutput() + eout.quiet = \ + mysettings.get("PORTAGE_QUIET") == "1" + eout.ebegin( + "%s size ;-)" % (myfile, )) + eout.eend(0) + continue else: verified_ok, reason = portage.checksum.verify_all( myfile_path, mydigests[myfile]) @@ -3484,10 +3524,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", pass fetched = 1 continue - if not fetchonly: - fetched=2 - break - else: + if True: # File is the correct size--check the checksums for the fetched # file NOW, for those users who don't have a stable/continuous # net connection. This way we have a chance to try to download @@ -4788,6 +4825,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, fetchme = newuris[:] checkme = alist[:] + if mydo == "fetch": + # Files are already checked inside fetch(), + # so do not check them again. + checkme = [] + # Only try and fetch the files if we are going to need them ... # otherwise, if user has FEATURES=noauto and they run `ebuild clean # unpack compile install`, we will try and fetch 4 times :/