Merged from trunk 7038:7052
authorFabian Groffen <grobian@gentoo.org>
Wed, 27 Jun 2007 07:57:53 +0000 (07:57 -0000)
committerFabian Groffen <grobian@gentoo.org>
Wed, 27 Jun 2007 07:57:53 +0000 (07:57 -0000)
   +------------------------------------------------------------------------+
   | 7039    | Add sanity checks in fetch() and digestgen() to              |
   | zmedico | automatically detect and handle invalid empty distfiles      |
   |         | since some users have reported difficulty when trying to     |
   |         | create digests.                                              |
   |---------+--------------------------------------------------------------|
   | 7041    | Add one more size == 0 check to digestgen().                 |
   | zmedico |                                                              |
   |---------+--------------------------------------------------------------|
   | 7042    | Short circuit the loop as soon as missing hashes are         |
   | zmedico | detected.                                                    |
   |---------+--------------------------------------------------------------|
   | 7044    | Short circuit the loop as soon as missing hashes are         |
   | zmedico | detected.                                                    |
   |---------+--------------------------------------------------------------|
   | 7046    | Preserve merge order in elog output.                         |
   | zmedico |                                                              |
   |---------+--------------------------------------------------------------|
   | 7047    | Send the ebuild die message through eerror and make sure     |
   | zmedico | that elog_process() gets called when an ebuild dies. This    |
   |         | makes the elog echo module display the die message last in   |
   |         | case other elog messages push the original die message off   |
   |         | of the top of the screen.                                    |
   |---------+--------------------------------------------------------------|
   | 7050    | Format the callstack indentation more like it used to be.    |
   | zmedico |                                                              |
   |---------+--------------------------------------------------------------|
   | 7052    | For bug #144333, add a --reinstall=changed-use option which  |
   | zmedico | behaves similar to how --newuse behaved in portage-2.0.x     |
   |         | days.                                                        |
   +------------------------------------------------------------------------+

svn path=/main/branches/prefix/; revision=7054

bin/isolated-functions.sh
pym/emerge/__init__.py
pym/portage/__init__.py
pym/portage/dbapi/vartree.py
pym/portage/elog/mod_echo.py
pym/portage/manifest.py

index ac10d087ff01c498a0f94e92e5c5ff29f762d716..594c4e8ec41160e6d9b01a9f4ec09ecc2a82b4d6 100755 (executable)
@@ -23,7 +23,7 @@ dump_trace() {
                 strip=$(( $1 ))
         fi
 
-        echo "Call stack:"
+        eerror "Call stack:"
         for (( n = ${#FUNCNAME[@]} - 1, p = ${#BASH_ARGV[@]} ; n > $strip ; n-- )) ; do
                 funcname=${FUNCNAME[${n} - 1]}
                 sourcefile=$(basename ${BASH_SOURCE[${n}]})
@@ -37,31 +37,32 @@ dump_trace() {
                         done
                         (( p -= ${BASH_ARGC[${n} - 1]} ))
                 fi
-                echo "  ${sourcefile}, line ${lineno}:   Called ${funcname}${args:+ ${args}}"
+                eerror "  ${sourcefile}, line ${lineno}:   Called ${funcname}${args:+ ${args}}"
         done
 }
 
 diefunc() {
         local funcname="$1" lineno="$2" exitcode="$3"
         shift 3
-        echo >&2
-        echo "!!! ERROR: $CATEGORY/$PF failed." >&2
-        dump_trace 2 1>&2
-        echo "  $(basename "${BASH_SOURCE[1]}"), line ${BASH_LINENO[0]}:   Called die" 1>&2
-        echo >&2
-        echo "!!! ${*:-(no error message)}" >&2
-        echo "!!! If you need support, post the topmost build error, and the call stack if relevant." >&2
+        eerror
+        eerror "ERROR: $CATEGORY/$PF failed."
+        dump_trace 2
+        eerror "  ${BASH_SOURCE[1]##*/}, line ${BASH_LINENO[0]}:   Called die"
+        eerror
+        eerror "${*:-(no error message)}"
+        eerror "If you need support, post the topmost build error, and the call stack if relevant."
         [ -n "${PORTAGE_LOG_FILE}" ] && \
-            echo "!!! A complete build log is located at '${PORTAGE_LOG_FILE}'." >&2
-        echo >&2
+            eerror "A complete build log is located at '${PORTAGE_LOG_FILE}'."
         if [ -n "${EBUILD_OVERLAY_ECLASSES}" ] ; then
-                echo "This ebuild used the following eclasses from overlays:" >&2
-                echo >&2
-                for x in ${EBUILD_OVERLAY_ECLASSES} ; do
-                        echo "  ${x}" >&2
-                done
-                echo >&2
+                eerror "This ebuild used eclasses from overlays: ${EBUILD_OVERLAY_ECLASSES}"
+        fi
+        if [ ${EBUILD#${PORTDIR}/} == ${EBUILD} ] ; then
+            local overlay=${EBUILD%/*}
+            overlay=${overlay%/*}
+            overlay=${overlay%/*}
+            eerror "This ebuild is from an overlay: '${overlay}/'"
         fi
+        eerror
 
         if [ "${EBUILD_PHASE/depend}" == "${EBUILD_PHASE}" ]; then
                 local x
index 66bc34d4aa30e4ad3f48c2640ea968557290d565..4e03f60ae81caea0db4b59dca63d26cc5baf66a9 100644 (file)
@@ -344,6 +344,7 @@ def create_depgraph_params(myopts, myaction):
        sub=[]
        if "--update" in myopts or \
                "--newuse" in myopts or \
+               "--reinstall" in myopts or \
                "--noreplace" in myopts or \
                myaction in ("system", "world"):
                add.extend(["selective"])
@@ -1042,6 +1043,21 @@ class depgraph(object):
                f.end_paragraph(1)
                f.writer.flush()
 
+       def _reinstall_for_flags(self, forced_flags,
+               orig_use, org_iuse, cur_use, cur_iuse):
+               if "--newuse" in self.myopts:
+                       if org_iuse.symmetric_difference(
+                               cur_iuse).difference(forced_flags):
+                               return True
+                       elif org_iuse.intersection(orig_use) != \
+                               cur_iuse.intersection(cur_use):
+                               return True
+               elif "changed-use" in self.myopts.get("--reinstall","").split(","):
+                       if org_iuse.intersection(orig_use) != \
+                               cur_iuse.intersection(cur_use):
+                               return True
+               return False
+
        def create(self, mybigkey, myparent=None, addme=1, myuse=None,
                priority=DepPriority(), rev_dep=False, arg=None):
                """
@@ -1115,7 +1131,9 @@ class depgraph(object):
                        """ If we aren't merging, perform the --newuse check.
                            If the package has new iuse flags or different use flags then if
                            --newuse is specified, we need to merge the package. """
-                       if merging==0 and "--newuse" in self.myopts and \
+                       if merging == 0 and \
+                               ("--newuse" in self.myopts or
+                               "--reinstall" in self.myopts) and \
                                vardbapi.cpv_exists(mykey):
                                pkgsettings.setcpv(mykey, mydb=mydbapi)
                                forced_flags = set()
@@ -1126,12 +1144,9 @@ class depgraph(object):
                                        mydbapi.aux_get(mykey, ["IUSE"])[0].split()))
                                old_iuse = set(filter_iuse_defaults(
                                        vardbapi.aux_get(mykey, ["IUSE"])[0].split()))
-                               if iuses.symmetric_difference(
-                                       old_iuse).difference(forced_flags):
+                               if self._reinstall_for_flags(
+                                       forced_flags, old_use, old_iuse, myuse, iuses):
                                        merging = 1
-                               elif old_iuse.intersection(old_use) != \
-                                       iuses.intersection(myuse):
-                                       merging=1
 
                if addme and merging == 1:
                        mybigkey.append("merge")
@@ -1597,7 +1612,9 @@ class depgraph(object):
                                        if myeb_pkg_matches:
                                                myeb_pkg = portage.best(myeb_pkg_matches)
 
-                               if myeb_pkg and "--newuse" in self.myopts:
+                               if myeb_pkg and \
+                                       ("--newuse" in self.myopts or \
+                                       "--reinstall" in self.myopts):
                                        iuses = set(filter_iuse_defaults(
                                                bindb.aux_get(myeb_pkg, ["IUSE"])[0].split()))
                                        old_use = bindb.aux_get(myeb_pkg, ["USE"])[0].split()
@@ -1616,11 +1633,8 @@ class depgraph(object):
                                        if "--usepkgonly" not in self.myopts and myeb:
                                                cur_iuse = set(filter_iuse_defaults(
                                                        portdb.aux_get(myeb, ["IUSE"])[0].split()))
-                                       if iuses.symmetric_difference(
-                                               cur_iuse).difference(forced_flags):
-                                               myeb_pkg = None
-                                       elif iuses.intersection(old_use) != \
-                                               cur_iuse.intersection(now_use):
+                                       if self._reinstall_for_flags(
+                                               forced_flags, old_use, iuses, now_use, cur_iuse):
                                                myeb_pkg = None
                                if myeb_pkg:
                                        binpkguseflags = \
@@ -3230,7 +3244,8 @@ class MergeTask(object):
                                                        pkgsettings, self.edebug, mydbapi=portdb,
                                                        tree="porttree")
                                                del pkgsettings["PORTAGE_BINPKG_TMPFILE"]
-                                               if "--buildpkgonly" in self.myopts:
+                                               if retval != os.EX_OK or \
+                                                       "--buildpkgonly" in self.myopts:
                                                        elog_process(pkg_key, pkgsettings)
                                                if retval != os.EX_OK:
                                                        return retval
@@ -5318,6 +5333,9 @@ def parse_opts(tmpcmdline, silent=False):
                        "help":"include unnecessary build time dependencies",
                        "type":"choice",
                        "choices":("y", "n")
+               },
+               "--reinstall": {
+                       "help":"specify conditions to trigger package reinstallation"
                }
        }
 
index 31dafff000966c17bddd826aea067b46bbf35e45..c5cb97f64aa256bea833f50829344500a2a038c4 100644 (file)
@@ -2709,7 +2709,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                if not os.access(myfile_path, os.R_OK):
                                                        writemsg("!!! Failed to adjust permissions:" + \
                                                                " %s\n" % str(e), noiselevel=-1)
-                                       if myfile not in mydigests:
+
+                                       # If the file is empty then it's obviously invalid. Remove
+                                       # the empty file and try to download if possible.
+                                       if mystat.st_size == 0:
+                                               if can_fetch:
+                                                       try:
+                                                               os.unlink(myfile_path)
+                                                       except EnvironmentError:
+                                                               pass
+                                       elif myfile not in mydigests:
                                                # We don't have a digest, but the file exists.  We must
                                                # assume that it is fully downloaded.
                                                continue
@@ -2833,6 +2842,17 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                                writemsg("!!! Failed to adjust permissions:" + \
                                                                        " %s\n" % str(e), noiselevel=-1)
 
+                                       # If the file is empty then it's obviously invalid.  Don't
+                                       # trust the return value from the fetcher.  Remove the
+                                       # empty file and try to download again.
+                                       try:
+                                               if os.stat(myfile_path).st_size == 0:
+                                                       os.unlink(myfile_path)
+                                                       fetched = 0
+                                                       continue
+                                       except EnvironmentError:
+                                               pass
+
                                        if mydigests!=None and mydigests.has_key(myfile):
                                                try:
                                                        mystat = os.stat(myfile_path)
@@ -2967,16 +2987,23 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None
                                continue
                        if required_hash_types.difference(myhashes):
                                missing_hashes.add(myfile)
+                               continue
+                       if myhashes["size"] == 0:
+                               missing_hashes.add(myfile)
                if missing_hashes:
                        missing_files = []
                        for myfile in missing_hashes:
                                try:
-                                       os.stat(os.path.join(mysettings["DISTDIR"], myfile))
+                                       st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
                                except OSError, e:
                                        if e.errno != errno.ENOENT:
                                                raise
                                        del e
                                        missing_files.append(myfile)
+                               else:
+                                       # If the file is empty then it's obviously invalid.
+                                       if st.st_size == 0:
+                                               missing_files.append(myfile)
                        if missing_files:
                                mytree = os.path.realpath(os.path.dirname(
                                        os.path.dirname(mysettings["O"])))
@@ -3238,8 +3265,9 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m
        mysettings["FILESDIR"] = os.path.join(pkg_dir, "files")
        mysettings["PF"]       = mypv
 
-       mysettings["ECLASSDIR"]   = os.path.join(mysettings["PORTDIR"], "eclass")
-       mysettings["SANDBOX_LOG"] = mycpv.replace(os.path.sep, "_-_")
+       mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
+       mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
+       mysettings["SANDBOX_LOG"] = mycpv.replace("/", "_-_")
 
        mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles)+"\n"+CUSTOM_PROFILE_PATH
        mysettings["P"]  = mysplit[0]+"-"+mysplit[1]
@@ -3985,19 +4013,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                        print "!!! Unknown mydo:",mydo
                        return 1
 
-               if retval != os.EX_OK and tree == "porttree":
-                       for i in xrange(len(mydbapi.porttrees)-1):
-                               t = mydbapi.porttrees[i+1]
-                               if myebuild.startswith(t):
-                                       # Display the non-cannonical path, in case it's different, to
-                                       # prevent confusion.
-                                       overlays = mysettings["PORTDIR_OVERLAY"].split()
-                                       try:
-                                               writemsg("!!! This ebuild is from an overlay: '%s'\n" % \
-                                                       overlays[i], noiselevel=-1)
-                                       except IndexError:
-                                               pass
-                                       break
                return retval
 
        finally:
index 9cc264cc40e3fddc198590eaddfd751928f3e6c7..ad8e464321c43853898113523364dcc2c014169b 100644 (file)
@@ -20,7 +20,7 @@ from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs,
        grabfile, grabdict, normalize_path, new_protect_filename
 from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp
 
-from portage import listdir, dep_expand, config, flatten, key_expand, \
+from portage import listdir, dep_expand, flatten, key_expand, \
        doebuild_environment, doebuild, env_update, \
        abssymlink, movefile, _movefile, bsd_chflags
 
@@ -650,6 +650,7 @@ class vartree(object):
                        self.root = clone.root[:]
                        self.dbapi = copy.deepcopy(clone.dbapi)
                        self.populated = 1
+                       from portage import config
                        self.settings = config(clone=clone.settings)
                else:
                        self.root = root[:]
@@ -1058,6 +1059,7 @@ class dblink(object):
                                uid=portage_uid, gid=portage_gid, mode=070, mask=0)
                builddir_lock = None
                catdir_lock = None
+               retval = -1
                try:
                        if myebuildpath:
                                catdir_lock = lockdir(catdir)
@@ -1090,23 +1092,27 @@ class dblink(object):
                                         self.settings, use_cache=0, tree="vartree",
                                         mydbapi=self.vartree.dbapi, vartree=self.vartree)
 
-                               # process logs created during pre/postrm
-                               elog_process(self.mycpv, self.settings)
-
                                # XXX: Decide how to handle failures here.
                                if retval != os.EX_OK:
                                        writemsg("!!! FAILED postrm: %s\n" % retval, noiselevel=-1)
                                        return retval
-                               doebuild(myebuildpath, "cleanrm", self.myroot, self.settings,
-                                       tree="vartree", mydbapi=self.vartree.dbapi,
-                                       vartree=self.vartree)
-                       
+
                        # regenerate reverse NEEDED map
                        self.vartree.dbapi.libmap.update()
 
                finally:
                        if builddir_lock:
-                               unlockdir(builddir_lock)
+                               try:
+                                       if myebuildpath:
+                                               # process logs created during pre/postrm
+                                               elog_process(self.mycpv, self.settings)
+                                               if retval == os.EX_OK:
+                                                       doebuild(myebuildpath, "cleanrm", self.myroot,
+                                                               self.settings, tree="vartree",
+                                                               mydbapi=self.vartree.dbapi,
+                                                               vartree=self.vartree)
+                               finally:
+                                       unlockdir(builddir_lock)
                        try:
                                if myebuildpath and not catdir_lock:
                                        # Lock catdir for removal if empty.
@@ -1609,9 +1615,12 @@ class dblink(object):
                        slot_matches.append(self.mycpv)
 
                others_in_slot = []
+               from portage import config
                for cur_cpv in slot_matches:
+                       # Clone the config in case one of these has to be unmerged since
+                       # we need it to have private ${T} etc... for things like elog.
                        others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
-                               self.vartree.root, self.settings,
+                               self.vartree.root, config(clone=self.settings),
                                vartree=self.vartree))
                retval = self._security_check(others_in_slot)
                if retval:
@@ -1806,12 +1815,6 @@ class dblink(object):
                        contents=contents, env=self.settings.environ())
 
                writemsg_stdout(">>> %s %s\n" % (self.mycpv,"merged."))
-
-               # Process ebuild logfiles
-               elog_process(self.mycpv, self.settings)
-               if "noclean" not in self.settings.features:
-                       doebuild(myebuild, "clean", destroot, self.settings,
-                               tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree)
                return os.EX_OK
 
        def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime):
@@ -2083,12 +2086,21 @@ class dblink(object):
 
        def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0,
                mydbapi=None, prev_mtimes=None):
+               retval = -1
+               self.lockdb()
                try:
-                       self.lockdb()
-                       return self.treewalk(mergeroot, myroot, inforoot, myebuild,
+                       retval = self.treewalk(mergeroot, myroot, inforoot, myebuild,
                                cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes)
+                       # Process ebuild logfiles
+                       elog_process(self.mycpv, self.settings)
+                       if retval == os.EX_OK and "noclean" not in self.settings.features:
+                               if myebuild is None:
+                                       myebuild = os.path.join(inforoot, self.pkg + ".ebuild")
+                               doebuild(myebuild, "clean", myroot, self.settings,
+                                       tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree)
                finally:
                        self.unlockdb()
+               return retval
 
        def getstring(self,name):
                "returns contents of a file with whitespace converted to spaces"
index a5749a93a9b327a1fc323a15e7c2a8c660e4b070..9ae6fcf6c4e4c390948e83c89833bb888b2d2cf5 100644 (file)
@@ -6,25 +6,18 @@
 from portage.output import EOutput
 from portage.const import EBUILD_PHASES
 
-_items = {}
+_items = []
 def process(mysettings, key, logentries, fulltext):
        global _items
-       config_root = mysettings["PORTAGE_CONFIGROOT"]
-       mysettings, items = _items.setdefault(config_root, (mysettings, {}))
-       items[key] = logentries
+       _items.append((mysettings, key, logentries))
 
 def finalize():
        global _items
-       for mysettings, items in _items.itervalues():
-               _finalize(mysettings, items)
-       _items.clear()
-
-def _finalize(mysettings, items):
        printer = EOutput()
-       root_msg = ""
-       if mysettings["ROOT"] != "/":
-               root_msg = " merged to %s" % mysettings["ROOT"]
-       for key, logentries in items.iteritems():
+       for mysettings, key, logentries in _items:
+               root_msg = ""
+               if mysettings["ROOT"] != "/":
+                       root_msg = " merged to %s" % mysettings["ROOT"]
                print
                printer.einfo("Messages for package %s%s:" % (key, root_msg))
                print
@@ -39,4 +32,5 @@ def _finalize(mysettings, items):
                                                "QA": printer.ewarn}
                                for line in msgcontent:
                                        fmap[msgtype](line.strip("\n"))
+       _items = []
        return
index 5add55c81ab5d62e987231c1e24f0f69f404545f..61af21d94f898b2b2bc1f21233592d9d91edece4 100644 (file)
@@ -487,12 +487,12 @@ class Manifest(object):
                        except OSError:
                                pass
                        if f in distfilehashes and \
+                               not required_hash_types.difference(distfilehashes[f]) and \
                                ((assumeDistHashesSometimes and mystat is None) or \
                                (assumeDistHashesAlways and mystat is None) or \
                                (assumeDistHashesAlways and mystat is not None and \
                                len(distfilehashes[f]) == len(self.hashes) and \
-                               distfilehashes[f]["size"] == mystat.st_size)) and \
-                               not required_hash_types.difference(distfilehashes[f]):
+                               distfilehashes[f]["size"] == mystat.st_size)):
                                self.fhashdict["DIST"][f] = distfilehashes[f]
                        else:
                                try: