Merged from trunk 5149:5157

Always verify the ebuild checksums before executing it.  Thanks to solar
for the suggestion and antarus for the initial patch.

For manifest and digest phases, use a global variable to temporarily
exempt the depend phase from manifest checks (in cache regeneration is
triggered).

Temporarily exempt the depend phase during the help phase also.

Avoid checking the same Manifest several times in a row during a regen
with an empty cache.

For consistent behavior in all timezones, make emerge-webrsync use UTC
time for decisions about which snapshots to download.

Only attempt to verify the Manifest if the ebuild is actually in a
portage tree.

Make sure that *all* of the ebuilds are listed in the Manifest.

Make sure that all depend phases triggered by the digest phase are
exempt from digest checks.

svn path=/main/branches/prefix/; revision=5158

diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index 8e9bb44c4c8c0916797d3099fc3a17b0fb4c5a00..b6d8e130aeb5d7832d7180fa4c10037ce4e8d15e 100755 (executable)
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -84,21 +84,20 @@ sync_local() {
 
 echo "Fetching most recent snapshot"
 
-declare -i attempts=-1
+declare -i attempts=0
 while (( $attempts <  40 )) ; do
        attempts=$(( attempts + 1 ))
 
-       #this too, sucks.  it works in the interim though.
-       if [ "$USERLAND" == "BSD" ] || [ "$USERLAND" == "Darwin" ] ; then
-               daysbefore=$(expr $(date +"%s") - 86400 \* $attempts)
-               day=$(date -r $daysbefore +"%d")
-               month=$(date -r $daysbefore +"%m")
-               year=$(date -r $daysbefore +"%Y")
-       else
-               day=$(date -d "-$attempts day" +"%d")
-               month=$(date -d "-$attempts day" +"%m")
-               year=$(date -d "-$attempts day" +"%Y")
-       fi
+       # The snapshot for a given day is generated at 01:45 UTC on the following
+       # day, so the current day's snapshot (going by UTC time) hasn't been
+       # generated yet.  Therefore, always start by looking for the previous day's
+       # snapshot (for attempts=1, subtract 1 day from the current UTC time).
+       daysbefore=$(expr $(date -u +"%s") - 86400 \* ${attempts})
+       DATE_ARGS="-d @${daysbefore}"
+       [ "${USERLAND}" != "GNU" ] && DATE_ARGS="-r ${daysbefore}"
+       day=$(date ${DATE_ARGS} -u +"%d")
+       month=$(date ${DATE_ARGS} -u +"%m")
+       year=$(date ${DATE_ARGS} -u +"%Y")
 
        FILE_ORIG="portage-${year}${month}${day}.tar.bz2"
 

diff --git a/pym/portage.py b/pym/portage.py
index 4f980a18b2cf118e0c867f941d1465dd4baa6ee7..45d5b3f7fbde5fdf3cab40c4743b222fb49ade75 100644 (file)
--- a/pym/portage.py
+++ b/pym/portage.py
@@ -2978,6 +2978,9 @@ def prepare_build_dirs(myroot, mysettings, cleanup):
                        (mysettings["CATEGORY"], mysettings["PF"], logid_time))
                del logid_path, logid_time
 
+_doebuild_manifest_exempt_depend = False
+_doebuild_manifest_checked = None
+
 def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
        fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None,
        mydbapi=None, vartree=None, prev_mtimes=None):
@@ -3014,34 +3017,78 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                        noiselevel=-1)
                return 1
 
-       doebuild_environment(myebuild, mydo, myroot, mysettings, debug,
-               use_cache, mydbapi)
-
-       # get possible slot information from the deps file
-       if mydo=="depend":
-               if mysettings.has_key("PORTAGE_DEBUG") and mysettings["PORTAGE_DEBUG"]=="1":
-                       # XXX: This needs to use a FD for saving the output into a file.
-                       # XXX: Set this up through spawn
-                       pass
-               writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2)
-               if dbkey:
-                       mysettings["dbkey"] = dbkey
-               else:
-                       mysettings["dbkey"] = mysettings.depcachedir+"/aux_db_key_temp"
-
-               retval = spawn(EBUILD_SH_BINARY+" depend",mysettings)
-               return retval
+       global _doebuild_manifest_exempt_depend
 
-       if not os.path.isdir(mysettings["PORTAGE_TMPDIR"]):
-               writemsg("The directory specified in your PORTAGE_TMPDIR variable, '%s',\n" % \
-                       mysettings["PORTAGE_TMPDIR"], noiselevel=-1)
-               writemsg("does not exist.  Please create this directory or correct your PORTAGE_TMPDIR setting.\n",
-                       noiselevel=-1)
-               return 1
+       if "strict" in features and \
+               tree == "porttree" and \
+               mydo not in ("digest", "manifest", "help") and \
+               not _doebuild_manifest_exempt_depend:
+               # Always verify the ebuild checksums before executing it.
+               pkgdir = os.path.dirname(myebuild)
+               manifest_path = os.path.join(pkgdir, "Manifest")
+               global _doebuild_manifest_checked
+               # Avoid checking the same Manifest several times in a row during a
+               # regen with an empty cache.
+               if _doebuild_manifest_checked != manifest_path:
+                       if not os.path.exists(manifest_path):
+                               writemsg("!!! Manifest file not found: '%s'\n" % manifest_path,
+                                       noiselevel=-1)
+                               return 1
+                       mf = Manifest(pkgdir, mysettings["DISTDIR"])
+                       try:
+                               mf.checkTypeHashes("EBUILD")
+                       except portage_exception.FileNotFound, e:
+                               writemsg("!!! A file listed in the Manifest " + \
+                                       "could not be found: %s\n" % str(e), noiselevel=-1)
+                               return 1
+                       except portage_exception.DigestException, e:
+                               writemsg("!!! Digest verification failed:\n", noiselevel=-1)
+                               writemsg("!!! %s\n" % e.value[0], noiselevel=-1)
+                               writemsg("!!! Reason: %s\n" % e.value[1], noiselevel=-1)
+                               writemsg("!!! Got: %s\n" % e.value[2], noiselevel=-1)
+                               writemsg("!!! Expected: %s\n" % e.value[3], noiselevel=-1)
+                               return 1
+                       # Make sure that all of the ebuilds are actually listed in the
+                       # Manifest.
+                       for f in os.listdir(pkgdir):
+                               if f.endswith(".ebuild") and not mf.hasFile("EBUILD", f):
+                                       writemsg("!!! A file is not listed in the " + \
+                                       "Manifest: '%s'\n" % os.path.join(pkgdir, f),
+                                       noiselevel=-1)
+                                       return 1
+                       _doebuild_manifest_checked = manifest_path
 
        logfile=None
        builddir_lock = None
        try:
+               if mydo in ("digest", "manifest", "help"):
+                       # Temporarily exempt the depend phase from manifest checks, in case
+                       # aux_get calls trigger cache generation.
+                       _doebuild_manifest_exempt_depend = True
+
+               doebuild_environment(myebuild, mydo, myroot, mysettings, debug,
+                       use_cache, mydbapi)
+
+               # get possible slot information from the deps file
+               if mydo == "depend":
+                       writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2)
+                       if dbkey:
+                               mysettings["dbkey"] = dbkey
+                       else:
+                               mysettings["dbkey"] = \
+                                       os.path.join(mysettings.depcachedir, "aux_db_key_temp")
+
+                       return spawn(EBUILD_SH_BINARY + " depend", mysettings)
+
+               if "PORTAGE_TMPDIR" not in mysettings or \
+                       not os.path.isdir(mysettings["PORTAGE_TMPDIR"]):
+                       writemsg("The directory specified in your " + \
+                               "PORTAGE_TMPDIR variable, '%s',\n" % \
+                               mysettings.get("PORTAGE_TMPDIR", ""), noiselevel=-1)
+                       writemsg("does not exist.  Please create this directory or " + \
+                               "correct your PORTAGE_TMPDIR setting.\n", noiselevel=-1)
+                       return 1
+
                # Build directory creation isn't required for any of these.
                if mydo not in ["fetch","digest","manifest"]:
                        mystatus = prepare_build_dirs(myroot, mysettings, cleanup)
@@ -3312,6 +3359,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                        except OSError:
                                pass
 
+               if mydo in ("digest", "manifest", "help"):
+                       # If necessary, depend phase has been triggered by aux_get calls
+                       # and the exemption is no longer needed.
+                       _doebuild_manifest_exempt_depend = False
+
 expandcache={}
 
 def movefile(src,dest,newmtime=None,sstat=None,mysettings=None):