Merged from trunk 9461:9467

   | 9462    | * Optimize parallel-fetch to avoid redundant checksum        |
   | zmedico | verification. * Add parallel-fetch to the default FEATURES   |
   |         | since it is more efficient now.                              |

   | 9463    | Bug #42456 - When a checksum failure occurs during fetch,    |
   | zmedico | use a new PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS config variable |
   |         | to control how many mirrors should be tried before aborting  |
   |         | the download. This type of behavior used to be enabled only  |
   |         | in --fetchonly mode but now it's enabled for all fetch()     |
   |         | calls.                                                       |

   | 9466    | Fix grammar.                                                 |
   | zmedico |                                                              |

   | 9467    | Fix an UnboundLocalError for 'pkgindex'.                     |
   | zmedico |                                                              |

svn path=/main/branches/prefix/; revision=9468

diff --git a/bin/emaint b/bin/emaint
index c2e3452902360163d0f314a9a09d9a5fb8b429e1..0f4f7b417e584bc86d29e1c387bc98232dca5c02 100755 (executable)
--- a/bin/emaint
+++ b/bin/emaint
@@ -172,8 +172,8 @@ class BinhostHandler(object):
                                self._pkgindex_file, wantnewlockfile=1)
                        try:
                                from portage import getbinpkg
-                               del pkgindex
-                               self._pkgindex = getbinpkg.PackageIndex()
+                               pkgindex = getbinpkg.PackageIndex()
+                               self._pkgindex = pkgindex
                                f = open(self._pkgindex_file, 'r')
                                try:
                                        self._pkgindex.read(f)

diff --git a/cnf/make.globals b/cnf/make.globals
index 41bcdbc7c3c76df3726e4156d1f6ce4c069100d0..6417cd767896d52e5b24fbc41899e02e824b5011 100644 (file)
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -33,7 +33,7 @@ FETCHCOMMAND="wget -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI
 RESUMECOMMAND="wget -c -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
 
 # Default user options
-FEATURES="distlocks metadata-transfer sandbox sfperms strict unmerge-orphans userfetch"
+FEATURES="distlocks parallel-fetch metadata-transfer sandbox sfperms strict unmerge-orphans userfetch"
 
 # Enable for global testing
 FEATURES="${FEATURES} preserve-libs"
@@ -51,6 +51,9 @@ EMERGE_WARNING_DELAY="10"
 # This option will be removed and forced to yes.
 AUTOCLEAN="yes"
 
+# Number of mirrors to try when a downloaded file has an incorrect checksum.
+PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5"
+
 # Number of times 'emerge --sync' will run before giving up.
 PORTAGE_RSYNC_RETRIES="3"
 

diff --git a/man/make.conf.5 b/man/make.conf.5
index 75bc141aaa90c9e595cac1c77d2672e9afc6ff88..5c8fc022b38311792e7500b53a5779448058270e 100644 (file)
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -402,6 +402,9 @@ This variable contains flags for the \fBPORTAGE_COMPRESS\fR command.
 .B PORTAGE_ELOG_MAILSUBJECT
 Please see /etc/make.conf.example for elog documentation.
 .TP
+\fBPORTAGE_FETCH_CHECKSUM_TRY_MIRRORS\fR = \fI5\fR
+Number of mirrors to try when a downloaded file has an incorrect checksum.
+.TP
 \fBPORTAGE_NICENESS\fR = \fI[number]\fR
 The value of this variable will be added to the current nice level that
 emerge is running at.  In other words, this will not set the nice level,

diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index d1f876065a53ee8caa9d43d2a841b14dc99aa57d..1bf636090c0175ef2292e8ad96b219bcdc3bca64 100644 (file)
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -4530,6 +4530,7 @@ class MergeTask(object):
                                fetch_env = os.environ.copy()
                                fetch_env["FEATURES"] = fetch_env.get("FEATURES", "") + " -cvs"
                                fetch_env["PORTAGE_NICENESS"] = "0"
+                               fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
                                fetch_args = [sys.argv[0], "--resume", "--fetchonly"]
                                resume_opts = self.myopts.copy()
                                # For automatic resume, we need to prevent
@@ -7009,7 +7010,7 @@ def action_build(settings, trees, mtimedb,
                if ("--resume" in myopts):
                        favorites=mtimedb["resume"]["favorites"]
                        mergetask = MergeTask(settings, trees, myopts)
-                       if "--fetchonly" in myopts:
+                       if "PORTAGE_PARALLEL_FETCHONLY" in settings:
                                """ parallel-fetch uses --resume --fetchonly and we don't want
                                it to write the mtimedb"""
                                mtimedb.filename = None

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 0dadd2d3bb39a3a0a8585687fc86dd4033e0b37b..1579b3d9199dfe05e23eb701825e3eb3a230c97d 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -947,7 +947,8 @@ class config(object):
                "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_CALLER",
                "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES",
                "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
-               "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", "PORTAGE_GPG_DIR",
+               "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
+               "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR",
                "PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
                "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
                "PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
@@ -3048,12 +3049,42 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
        # every single available mirror is a waste of bandwidth
        # and time, so there needs to be a cap.
        checksum_failure_max_tries = 5
+       v = checksum_failure_max_tries
+       try:
+               v = int(mysettings.get("PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS",
+                       checksum_failure_max_tries))
+       except (ValueError, OverflowError):
+               writemsg("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS" + \
+                       " contains non-integer value: '%s'\n" % \
+                       mysettings["PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"], noiselevel=-1)
+               writemsg("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS " + \
+                       "default value: %s\n" % checksum_failure_max_tries,
+                       noiselevel=-1)
+               v = checksum_failure_max_tries
+       if v < 1:
+               writemsg("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS" + \
+                       " contains value less than 1: '%s'\n" % v, noiselevel=-1)
+               writemsg("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS " + \
+                       "default value: %s\n" % checksum_failure_max_tries,
+                       noiselevel=-1)
+               v = checksum_failure_max_tries
+       checksum_failure_max_tries = v
+       del v
+
        # Behave like the package has RESTRICT="primaryuri" after a
        # couple of checksum failures, to increase the probablility
        # of success before checksum_failure_max_tries is reached.
        checksum_failure_primaryuri = 2
        thirdpartymirrors = mysettings.thirdpartymirrors()
 
+       # In the background parallel-fetch process, it's safe to skip checksum
+       # verification of pre-existing files in $DISTDIR that have the correct
+       # file size. The parent process will verify their checksums prior to
+       # the unpack phase.
+
+       parallel_fetchonly = fetchonly and \
+               "PORTAGE_PARALLEL_FETCHONLY" in mysettings
+
        check_config_instance(mysettings)
 
        custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
@@ -3240,7 +3271,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 
                        if use_locks and can_fetch:
                                waiting_msg = None
-                               if "parallel-fetch" in features:
+                               if not parallel_fetchonly and "parallel-fetch" in features:
                                        waiting_msg = ("Downloading '%s'... " + \
                                                "see "+EPREFIX+"/var/log/emerge-fetch.log for details.") % myfile
                                if locks_in_subdir:
@@ -3299,6 +3330,15 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                if mystat.st_size < mydigests[myfile]["size"] and \
                                                        not restrict_fetch:
                                                        fetched = 1 # Try to resume this download.
+                                               elif parallel_fetchonly and \
+                                                       mystat.st_size == mydigests[myfile]["size"]:
+                                                       eout = portage.output.EOutput()
+                                                       eout.quiet = \
+                                                               mysettings.get("PORTAGE_QUIET") == "1"
+                                                       eout.ebegin(
+                                                               "%s size ;-)" % (myfile, ))
+                                                       eout.eend(0)
+                                                       continue
                                                else:
                                                        verified_ok, reason = portage.checksum.verify_all(
                                                                myfile_path, mydigests[myfile])
@@ -3484,10 +3524,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                                                        pass
                                                                fetched = 1
                                                                continue
-                                                       if not fetchonly:
-                                                               fetched=2
-                                                               break
-                                                       else:
+                                                       if True:
                                                                # File is the correct size--check the checksums for the fetched
                                                                # file NOW, for those users who don't have a stable/continuous
                                                                # net connection. This way we have a chance to try to download
@@ -4788,6 +4825,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                        fetchme = newuris[:]
                        checkme = alist[:]
 
+               if mydo == "fetch":
+                       # Files are already checked inside fetch(),
+                       # so do not check them again.
+                       checkme = []
+
                # Only try and fetch the files if we are going to need them ...
                # otherwise, if user has FEATURES=noauto and they run `ebuild clean
                # unpack compile install`, we will try and fetch 4 times :/