* Optimize parallel-fetch to avoid redundant checksum verification.

* Add parallel-fetch to the default FEATURES since it is more efficient now.

svn path=/main/trunk/; revision=9462

diff --git a/cnf/make.globals b/cnf/make.globals
index d36ede9a375c3fed86bc955185c815ad70dc701b..9023dab6b03547bedb5860a32b1fc4d8b8c5e47f 100644 (file)
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -33,7 +33,7 @@ FETCHCOMMAND="/usr/bin/wget -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\"
 RESUMECOMMAND="/usr/bin/wget -c -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
 
 # Default user options
-FEATURES="distlocks metadata-transfer sandbox sfperms strict unmerge-orphans userfetch"
+FEATURES="distlocks parallel-fetch metadata-transfer sandbox sfperms strict unmerge-orphans userfetch"
 
 # Enable for global testing
 FEATURES="${FEATURES} preserve-libs"

diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index 68bf0d9730ef6f5e393ce567952034c7dfd04db6..7a619271dce900b956bb5a03e2a0f4eb347b9b31 100644 (file)
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -4489,6 +4489,7 @@ class MergeTask(object):
                                fetch_env = os.environ.copy()
                                fetch_env["FEATURES"] = fetch_env.get("FEATURES", "") + " -cvs"
                                fetch_env["PORTAGE_NICENESS"] = "0"
+                               fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
                                fetch_args = [sys.argv[0], "--resume", "--fetchonly"]
                                resume_opts = self.myopts.copy()
                                # For automatic resume, we need to prevent
@@ -6920,7 +6921,7 @@ def action_build(settings, trees, mtimedb,
                if ("--resume" in myopts):
                        favorites=mtimedb["resume"]["favorites"]
                        mergetask = MergeTask(settings, trees, myopts)
-                       if "--fetchonly" in myopts:
+                       if "PORTAGE_PARALLEL_FETCHONLY" in settings:
                                """ parallel-fetch uses --resume --fetchonly and we don't want
                                it to write the mtimedb"""
                                mtimedb.filename = None

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 04f7acec016f272eb3ba083ccd46006d3465cb84..8e146c45bd693c1b3c3f7ec4aa96e2208bbb4209 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -3032,6 +3032,14 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
        checksum_failure_primaryuri = 2
        thirdpartymirrors = mysettings.thirdpartymirrors()
 
+       # In the background parallel-fetch process, it's safe to skip checksum
+       # verification of pre-existing files in $DISTDIR that have the correct
+       # file size. The parent process will verify their checksums prior to
+       # the unpack phase.
+
+       parallel_fetchonly = fetchonly and \
+               "PORTAGE_PARALLEL_FETCHONLY" in mysettings
+
        check_config_instance(mysettings)
 
        custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
@@ -3218,7 +3226,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 
                        if use_locks and can_fetch:
                                waiting_msg = None
-                               if "parallel-fetch" in features:
+                               if not parallel_fetchonly and "parallel-fetch" in features:
                                        waiting_msg = ("Downloading '%s'... " + \
                                                "see /var/log/emerge-fetch.log for details.") % myfile
                                if locks_in_subdir:
@@ -3277,6 +3285,15 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                if mystat.st_size < mydigests[myfile]["size"] and \
                                                        not restrict_fetch:
                                                        fetched = 1 # Try to resume this download.
+                                               elif parallel_fetchonly and \
+                                                       mystat.st_size == mydigests[myfile]["size"]:
+                                                       eout = portage.output.EOutput()
+                                                       eout.quiet = \
+                                                               mysettings.get("PORTAGE_QUIET") == "1"
+                                                       eout.ebegin(
+                                                               "%s size ;-)" % (myfile, ))
+                                                       eout.eend(0)
+                                                       continue
                                                else:
                                                        verified_ok, reason = portage.checksum.verify_all(
                                                                myfile_path, mydigests[myfile])
@@ -4737,6 +4754,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                        fetchme = newuris[:]
                        checkme = alist[:]
 
+               if mydo == "fetch":
+                       # Files are already checked inside fetch(),
+                       # so do not check them again.
+                       checkme = []
+
                # Only try and fetch the files if we are going to need them ...
                # otherwise, if user has FEATURES=noauto and they run `ebuild clean
                # unpack compile install`, we will try and fetch 4 times :/