Merged from trunk -r12790:12806
authorFabian Groffen <grobian@gentoo.org>
Mon, 9 Mar 2009 19:51:11 +0000 (19:51 -0000)
committerFabian Groffen <grobian@gentoo.org>
Mon, 9 Mar 2009 19:51:11 +0000 (19:51 -0000)
   | 12791    | Also support + character in eclass names.                   |
   | zmedico  |                                                             |

   | 12792    | Fix typo from previous commit.                              |
   | zmedico  |                                                             |

   | 12793    | for pointers cast to 32bit ints, we always want to abort on |
   | SpankMan | 64bit systems, not just when FEATURES=stricter              |

   | 12794    | Generate a QA Notice when EXPORT_FUNCTIONS is called before |
   | zmedico  | inherit, since it's incompatible with <=portage-2.1.6.7.    |

   | 12795    | Replace things like [[ $PORTAGE_BUILDDIR/.prepared -nt      |
   | zmedico  | $WORKDIR ]] with simply [[ -e $PORTAGE_BUILDDIR/.prepared   |
   |          | ]] since the timestamp of $WORKDIR is practically always    |
   |          | newer due to the other hidden files that are created there  |
   |          | as each phase is executed. Thanks to Alfredo Tupone         |
   |          | <tupone@g.o> for reporting.                                 |

   | 12796    | Make the --digest warning message show for FEATURES=digest. |
   | zmedico  |                                                             |

   | 12797    | Bug #261675 - When doebuild() is called by emerge, don't    |
   | zmedico  | call digestgen() for FEATURES=digest because it's redundant |
   |          | and it can cause interference with parallel-fetch and       |
   |          | parallel builds.                                            |

   | 12798    | Bug #261675 - Also don't call digestgen() from              |
   | zmedico  | parallel-fetch processes.                                   |

   | 12799    | Bug #261675 - Generate manifests in advance when            |
   | zmedico  | FEATURES=digest is enabled.                                 |

   | 12800    | Never call digestgen() inside doebuild() which only 'fetch' |
   | zmedico  | has been requested since it's not needed in this case.      |

   | 12801    | Reorganize code for --digests and FEATURES=digest.          |
   | zmedico  |                                                             |

   | 12802    | Make manifest mode with --force option cause existing       |
   | zmedico  | digests to be replaced for any files that exist in          |
   |          | ${DISTDIR}. This provides an alternative to ebuild --force  |
   |          | manifest for updating existing distfiles digests. Digests   |
   |          | are assumed to be corect for files that do not exist in     |
   |          | ${DISTDIR} since the user could simply remove the whole     |
   |          | Manifest if they wanted to regenerate digests for all files |
   |          | (and it's safer to regenerate as few as possible because    |
   |          | it's less probably that a valid digest will get replaced by |
   |          | an invalid one).                                            |

   | 12803    | Fix _setitem() to account for the change to                 |
   | zmedico  | serialize_eclasses = False.                                 |

   | 12804    | Inside dyn_unpack(), only call pre/post phase hooks when    |
   | zmedico  | src_unpack is called (when .unpacked does not already       |
   |          | exist).                                                     |

   | 12805    | Use the ebuild_phase helper function to calls phase hooks   |
   | zmedico  | and fix broken trap calls that try to unset multiple signal |
   |          | handlers at once without passing - as the first argument.   |

   | 12806    | When generating variable names to hold EXPORT_FUNCTIONS     |
   | zmedico  | argument in, use $ECLASS_DEPTH as a unique id since it's a  |
   |          | lot simpler then encoding the eclass name in the variable   |
   |          | name.                                                       |

svn path=/main/branches/prefix/; revision=12807

bin/ebuild.sh
bin/misc-functions.sh
bin/repoman
man/repoman.1
pym/_emerge/__init__.py
pym/portage/__init__.py
pym/portage/cache/metadata.py

index 5014f6519482038ceea5e4f2046b1cf7b62c30da..b8f5a6837bdad76debf9adb8a85ed97e654b60e6 100755 (executable)
@@ -649,7 +649,6 @@ dyn_setup() {
 }
 
 dyn_unpack() {
-       [ "$(type -t pre_src_unpack)" == "function" ] && qa_call pre_src_unpack
        local newstuff="no"
        if [ -e "${WORKDIR}" ]; then
                local x
@@ -682,7 +681,6 @@ dyn_unpack() {
        if [ -e "${WORKDIR}" ]; then
                if [ "$newstuff" == "no" ]; then
                        vecho ">>> WORKDIR is up-to-date, keeping..."
-                       [ "$(type -t post_src_unpack)" == "function" ] && qa_call post_src_unpack
                        return 0
                fi
        fi
@@ -691,13 +689,12 @@ dyn_unpack() {
                install -m${PORTAGE_WORKDIR_MODE:-0700} -d "${WORKDIR}" || die "Failed to create dir '${WORKDIR}'"
        fi
        cd "${WORKDIR}" || die "Directory change failed: \`cd '${WORKDIR}'\`"
+       ebuild_phase pre_src_unpack
        vecho ">>> Unpacking source..."
        ebuild_phase src_unpack
        touch "${PORTAGE_BUILDDIR}/.unpacked" || die "IO Failure -- Failed 'touch .unpacked' in ${PORTAGE_BUILDDIR}"
        vecho ">>> Source unpacked in ${WORKDIR}"
-       cd "${PORTAGE_BUILDDIR}"
-
-       [ "$(type -t post_src_unpack)" == "function" ] && qa_call post_src_unpack
+       ebuild_phase post_src_unpack
 }
 
 dyn_clean() {
@@ -824,7 +821,7 @@ abort_handler() {
        echo
        eval ${3}
        #unset signal handler
-       trap SIGINT SIGQUIT
+       trap - SIGINT SIGQUIT
 }
 
 abort_prepare() {
@@ -859,7 +856,7 @@ abort_install() {
 
 dyn_prepare() {
 
-       if [[ $PORTAGE_BUILDDIR/.prepared -nt $WORKDIR ]] ; then
+       if [[ -e $PORTAGE_BUILDDIR/.prepared ]] ; then
                vecho ">>> It appears that '$PF' is already prepared; skipping."
                vecho ">>> Remove '$PORTAGE_BUILDDIR/.prepared' to force prepare."
                return 0
@@ -882,12 +879,12 @@ dyn_prepare() {
        vecho ">>> Source prepared."
        ebuild_phase post_src_prepare
 
-       trap SIGINT SIGQUIT
+       trap - SIGINT SIGQUIT
 }
 
 dyn_configure() {
 
-       if [[ $PORTAGE_BUILDDIR/.configured -nt $WORKDIR ]] ; then
+       if [[ -e $PORTAGE_BUILDDIR/.configured ]] ; then
                vecho ">>> It appears that '$PF' is already configured; skipping."
                vecho ">>> Remove '$PORTAGE_BUILDDIR/.configured' to force configuration."
                return 0
@@ -895,23 +892,21 @@ dyn_configure() {
 
        trap abort_configure SIGINT SIGQUIT
 
-       [[ $(type -t pre_src_configure) = function ]] && \
-               qa_call pre_src_configure
+       ebuild_phase pre_src_configure
 
        vecho ">>> Configuring source in $srcdir ..."
        ebuild_phase src_configure
        touch "$PORTAGE_BUILDDIR"/.configured
        vecho ">>> Source configured."
 
-       [[ $(type -t post_src_configure) = function ]] && \
-               qa_call post_src_configure
+       ebuild_phase post_src_configure
 
-       trap SIGINT SIGQUIT
+       trap - SIGINT SIGQUIT
 }
 
 dyn_compile() {
 
-       if [[ $PORTAGE_BUILDDIR/.compiled -nt $WORKDIR ]] ; then
+       if [[ -e $PORTAGE_BUILDDIR/.compiled ]] ; then
                vecho ">>> It appears that '${PF}' is already compiled; skipping."
                vecho ">>> Remove '$PORTAGE_BUILDDIR/.compiled' to force compilation."
                return 0
@@ -919,17 +914,16 @@ dyn_compile() {
 
        trap abort_compile SIGINT SIGQUIT
 
-       [[ $(type -t pre_src_compile) = function ]] && \
-               qa_call pre_src_compile
+       ebuild_phase pre_src_compile
 
        vecho ">>> Compiling source in ${srcdir} ..."
        ebuild_phase src_compile
        touch "$PORTAGE_BUILDDIR"/.compiled
        vecho ">>> Source compiled."
 
-       [ "$(type -t post_src_compile)" == "function" ] && qa_call post_src_compile
+       ebuild_phase post_src_compile
 
-       trap SIGINT SIGQUIT
+       trap - SIGINT SIGQUIT
 }
 
 dyn_test() {
@@ -939,10 +933,9 @@ dyn_test() {
                # like it's supposed to here.
                ! hasq test ${USE} && export USE="${USE} test"
        fi
-       [ "$(type -t pre_src_test)" == "function" ] && qa_call pre_src_test
-       if [ "${PORTAGE_BUILDDIR}/.tested" -nt "${WORKDIR}" ]; then
+       ebuild_phase pre_src_test
+       if [[ -e $PORTAGE_BUILDDIR/.tested ]] ; then
                vecho ">>> It appears that ${PN} has already been tested; skipping."
-               [ "$(type -t post_src_test)" == "function" ] && qa_call post_src_test
                return
        fi
        trap "abort_test" SIGINT SIGQUIT
@@ -962,23 +955,23 @@ dyn_test() {
                SANDBOX_PREDICT="${SANDBOX_PREDICT%:/}"
        fi
 
-       cd "${PORTAGE_BUILDDIR}"
-       touch .tested || die "Failed to 'touch .tested' in ${PORTAGE_BUILDDIR}"
-       [ "$(type -t post_src_test)" == "function" ] && qa_call post_src_test
-       trap SIGINT SIGQUIT
+       touch "$PORTAGE_BUILDDIR/.tested" || \
+               die "Failed to 'touch .tested' in $PORTAGE_BUILDDIR"
+       ebuild_phase post_src_test
+       trap - SIGINT SIGQUIT
 }
 
 dyn_install() {
        [ -z "$PORTAGE_BUILDDIR" ] && die "${FUNCNAME}: PORTAGE_BUILDDIR is unset"
        if hasq noauto $FEATURES ; then
                rm -f "${PORTAGE_BUILDDIR}/.installed"
-       elif [[ ${PORTAGE_BUILDDIR}/.installed -nt ${WORKDIR} ]] ; then
+       elif [[ -e $PORTAGE_BUILDDIR/.installed ]] ; then
                vecho ">>> It appears that '${PF}' is already installed; skipping."
                vecho ">>> Remove '${PORTAGE_BUILDDIR}/.installed' to force install."
                return 0
        fi
        trap "abort_install" SIGINT SIGQUIT
-       [ "$(type -t pre_src_install)" == "function" ] && qa_call pre_src_install
+       ebuild_phase pre_src_install
        rm -rf "${PORTAGE_BUILDDIR}/image"
        mkdir "${PORTAGE_BUILDDIR}/image"
        if [ -d "${S}" ]; then
@@ -1007,8 +1000,7 @@ dyn_install() {
        touch "${PORTAGE_BUILDDIR}/.installed"
        vecho ">>> Completed installing ${PF} into ${ED}"
        vecho
-       cd ${PORTAGE_BUILDDIR}
-       [ "$(type -t post_src_install)" == "function" ] && qa_call post_src_install
+       ebuild_phase post_src_install
 
        cd "${PORTAGE_BUILDDIR}"/build-info
        set -f
@@ -1041,7 +1033,7 @@ dyn_install() {
        then
                touch DEBUGBUILD
        fi
-       trap SIGINT SIGQUIT
+       trap - SIGINT SIGQUIT
 }
 
 dyn_preinst() {
@@ -1166,6 +1158,13 @@ inherit() {
                debug-print "*** Multiple Inheritence (Level: ${ECLASS_DEPTH})"
        fi
 
+       if [[ -n $ECLASS && -n ${!__export_funcs_var} ]] ; then
+               echo "QA Notice: EXPORT_FUNCTIONS is called before inherit in" \
+                       "$ECLASS.eclass. For compatibility with <=portage-2.1.6.7," \
+                       "only call EXPORT_FUNCTIONS after inherit(s)." \
+                       | fmt -w 75 | while read ; do eqawarn "$REPLY" ; done
+       fi
+
        local location
        local olocation
        local x
@@ -1183,11 +1182,7 @@ inherit() {
                olocation=""
 
                export ECLASS="$1"
-               __export_funcs_var=__export_functions_$ECLASS
-               while [[ $__export_funcs_var =~ [-.] ]] ; do
-                       __export_funcs_var=${__export_funcs_var/-/__dash__}
-                       __export_funcs_var=${__export_funcs_var/./__dot__}
-               done
+               __export_funcs_var=__export_functions_$ECLASS_DEPTH
                unset $__export_funcs_var
 
                if [ "${EBUILD_PHASE}" != "depend" ] && \
index 59217fdce244761e5c8cae840ea61c9e2e190b1f..b6944ffa779e2d97b84636a7244d7d8f12dd7bab 100644 (file)
@@ -536,14 +536,18 @@ install_qa_check() {
                        fi
 
                fi
-               if [[ $abort = yes ]] && [[ $gentoo_bug != yes ]] ; then
-                       echo "Please do not file a Gentoo bug and instead" \
-                       "report the above QA issues directly to the upstream" \
-                       "developers of this software." | fmt -w 70 | \
-                       while read line ; do eqawarn "${line}" ; done
-                       eqawarn "Homepage: ${HOMEPAGE}"
+               if [[ ${abort} == "yes" ]] ; then
+                       if [[ ${gentoo_bug} == "yes" ]] ; then
+                               die "poor code kills airplanes"
+                       else
+                               echo "Please do not file a Gentoo bug and instead" \
+                               "report the above QA issues directly to the upstream" \
+                               "developers of this software." | fmt -w 70 | \
+                               while read line ; do eqawarn "${line}" ; done
+                               eqawarn "Homepage: ${HOMEPAGE}"
+                               hasq stricter ${FEATURES} && die "poor code kills airplanes"
+                       fi
                fi
-               [[ ${abort} == "yes" ]] && hasq stricter ${FEATURES} && die "poor code kills airplanes"
        fi
 
        # Compiled python objects do not belong in /usr/share (FHS violation)
index 166d3967ef12be7c973732f1593c6c3213fafbdc..7d134b5b85f2075f3af6d44686b980bc9097fd16 100755 (executable)
@@ -799,11 +799,50 @@ for x in scanlist:
 
        if options.mode == "manifest" or \
          options.mode in ('commit', 'fix') and not options.pretend:
+               auto_assumed = set()
+               fetchlist_dict = portage.FetchlistDict(checkdir,
+                       repoman_settings, portdb)
+               if options.mode == 'manifest' and options.force:
+                       portage._doebuild_manifest_exempt_depend += 1
+                       try:
+                               distdir = repoman_settings['DISTDIR']
+                               mf = portage.manifest.Manifest(checkdir, distdir,
+                                       fetchlist_dict=fetchlist_dict)
+                               mf.create(requiredDistfiles=None,
+                                       assumeDistHashesAlways=True)
+                               for distfiles in fetchlist_dict.itervalues():
+                                       for distfile in distfiles:
+                                               if os.path.isfile(os.path.join(distdir, distfile)):
+                                                       mf.fhashdict['DIST'].pop(distfile, None)
+                                               else:
+                                                       auto_assumed.add(distfile)
+                               mf.write()
+                       finally:
+                               portage._doebuild_manifest_exempt_depend -= 1
+
                repoman_settings["O"] = checkdir
                if not portage.digestgen([], repoman_settings, myportdb=portdb):
                        print "Unable to generate manifest."
                        dofail = 1
                if options.mode == "manifest":
+                       if not dofail and options.force and auto_assumed and \
+                               'assume-digests' in repoman_settings.features:
+                               # Show which digests were assumed despite the --force option
+                               # being given. This output will already have been shown by
+                               # digestgen() if assume-digests is not enabled, so only show
+                               # it here if assume-digests is enabled.
+                               pkgs = list(fetchlist_dict)
+                               pkgs.sort()
+                               portage.writemsg_stdout("  digest.assumed" + \
+                                       portage.output.colorize("WARN",
+                                       str(len(auto_assumed)).rjust(18)) + "\n")
+                               for cpv in pkgs:
+                                       fetchmap = fetchlist_dict[cpv]
+                                       pf = portage.catsplit(cpv)[1]
+                                       for distfile in sorted(fetchmap):
+                                               if distfile in auto_assumed:
+                                                       portage.writemsg_stdout(
+                                                               "   %s::%s\n" % (pf, distfile))
                        continue
                elif dofail:
                        sys.exit(1)
index aca5587c06f6a0f8ebf51c980df9f7a51237ff98..b14cb17c2224d5942dc951bfbb64bc01eca74b37 100644 (file)
@@ -18,6 +18,14 @@ Force commit to proceed, regardless of QA issues. For convenience, this option
 causes the most time consuming QA checks to be skipped. The commit message will
 include an indication that this option has been enabled, together with the
 usual portage version stamp.
+
+When used together with \fBmanifest\fR mode, \fB--force\fR causes existing
+digests to be replaced for any files that exist in ${DISTDIR}.
+Existing digests are assumed to be correct for files that would otherwise
+have to be downloaded in order to recompute digests. \fBWARNING:\fR When
+replacing existing digests, it is the user's responsibility to ensure that
+files contained in ${DISTDIR} have the correct identities. Especially beware
+of partially downloaded files.
 .TP
 \fB-q\fR, \fB--quiet\fR
 Be less verbose about extraneous info
@@ -69,7 +77,8 @@ Scan directory tree for QA issues (short listing)
 Fix simple QA issues (stray digests, missing digests)
 .TP
 .B manifest
-Generate a Manifest (fetches files if necessary)
+Generate a Manifest (fetches distfiles if necessary). See the \fB\-\-force\fR
+option if you would like to replace existing distfiles digests.
 .TP
 .B commit
 Scan directory tree for QA issues; if OK, commit via cvs
index 58dc1dba241392b1dbec0cf460d63799b7e6e33b..22f6bf4d1ab9382d297d40065459955cb7554e24 100644 (file)
@@ -10470,6 +10470,51 @@ class Scheduler(PollScheduler):
 
                return ebuild_phase.returncode
 
+       def _generate_digests(self):
+               """
+               Generate digests if necessary for --digests or FEATURES=digest.
+               In order to avoid interference, this must done before parallel
+               tasks are started.
+               """
+
+               if '--fetchonly' in self.myopts:
+                       return os.EX_OK
+
+               digest = '--digest' in self.myopts
+               if not digest:
+                       for pkgsettings in self.pkgsettings.itervalues():
+                               if 'digest' in pkgsettings.features:
+                                       digest = True
+                                       break
+
+               if not digest:
+                       return os.EX_OK
+
+               for x in self._mergelist:
+                       if not isinstance(x, Package) or \
+                               x.type_name != 'ebuild' or \
+                               x.operation != 'merge':
+                               continue
+                       pkgsettings = self.pkgsettings[x.root]
+                       if '--digest' not in self.myopts and \
+                               'digest' not in pkgsettings.features:
+                               continue
+                       portdb = x.root_config.trees['porttree'].dbapi
+                       ebuild_path = portdb.findname(x.cpv)
+                       if not ebuild_path:
+                               writemsg_level(
+                                       "!!! Could not locate ebuild for '%s'.\n" \
+                                       % x.cpv, level=logging.ERROR, noiselevel=-1)
+                               return 1
+                       pkgsettings['O'] = os.path.dirname(ebuild_path)
+                       if not portage.digestgen([], pkgsettings, myportdb=portdb):
+                               writemsg_level(
+                                       "!!! Unable to generate manifest for '%s'.\n" \
+                                       % x.cpv, level=logging.ERROR, noiselevel=-1)
+                               return 1
+
+               return os.EX_OK
+
        def _check_manifests(self):
                # Verify all the manifests now so that the user is notified of failure
                # as soon as possible.
@@ -10678,6 +10723,10 @@ class Scheduler(PollScheduler):
                        self.pkgsettings[root] = portage.config(
                                clone=root_config.settings)
 
+               rval = self._generate_digests()
+               if rval != os.EX_OK:
+                       return rval
+
                rval = self._check_manifests()
                if rval != os.EX_OK:
                        return rval
@@ -14350,8 +14399,13 @@ def action_build(settings, trees, mtimedb,
        if pretend or fetchonly:
                # make the mtimedb readonly
                mtimedb.filename = None
-       if "--digest" in myopts:
-               msg = "The --digest option can prevent corruption from being" + \
+       if '--digest' in myopts or 'digest' in settings.features:
+               if '--digest' in myopts:
+                       msg = "The --digest option"
+               else:
+                       msg = "The FEATURES=digest setting"
+
+               msg += " can prevent corruption from being" + \
                        " noticed. The `repoman manifest` command is the preferred" + \
                        " way to generate manifests and it is capable of doing an" + \
                        " entire repository or category at once."
@@ -14657,20 +14711,6 @@ def action_build(settings, trees, mtimedb,
                        # Convert Atom instances to plain str.
                        mtimedb["resume"]["favorites"] = [str(x) for x in favorites]
 
-                       if ("--digest" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts):
-                               for pkgline in mydepgraph.altlist():
-                                       if pkgline[0]=="ebuild" and pkgline[3]=="merge":
-                                               y = trees[pkgline[1]]["porttree"].dbapi.findname(pkgline[2])
-                                               tmpsettings = portage.config(clone=settings)
-                                               edebug = 0
-                                               if settings.get("PORTAGE_DEBUG", "") == "1":
-                                                       edebug = 1
-                                               retval = portage.doebuild(
-                                                       y, "digest", settings["ROOT"], tmpsettings, edebug,
-                                                       ("--pretend" in myopts),
-                                                       mydbapi=trees[pkgline[1]]["porttree"].dbapi,
-                                                       tree="porttree")
-
                        pkglist = mydepgraph.altlist()
                        mydepgraph.saveNomergeFavorites()
                        mydepgraph.break_refs(pkglist)
index 4ed25e280b5867a3542e2114edf910826ee0a395..28461d9c4fd904ecf94d580bdf51bd4ee8d0d17f 100644 (file)
@@ -5996,6 +5996,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"]))
 
                emerge_skip_distfiles = returnpid
+               emerge_skip_digest = returnpid
                # Only try and fetch the files if we are going to need them ...
                # otherwise, if user has FEATURES=noauto and they run `ebuild clean
                # unpack compile install`, we will try and fetch 4 times :/
@@ -6052,7 +6053,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                        elif mydo == "digest":
                                return not digestgen(aalist, mysettings, overwrite=1,
                                        myportdb=mydbapi)
-                       elif "digest" in mysettings.features:
+                       elif mydo != 'fetch' and not emerge_skip_digest and \
+                               "digest" in mysettings.features:
+                               # Don't do this when called by emerge or when called just
+                               # for fetch (especially parallel-fetch) since it's not needed
+                               # and it can interfere with parallel tasks.
                                digestgen(aalist, mysettings, overwrite=0, myportdb=mydbapi)
                except portage.exception.PermissionDenied, e:
                        writemsg("!!! Permission Denied: %s\n" % (e,), noiselevel=-1)
index 4f74b767d1f168170a6682c046c5eb4916d15bc1..5222223c6d0449dbeab98328d9180249667a1b57 100644 (file)
@@ -71,8 +71,7 @@ class database(flat_hash.database):
        def _setitem(self, cpv, values):
                if "_eclasses_" in values:
                        values = ProtectedDict(values)
-                       values["INHERITED"] = ' '.join(sorted(
-                               reconstruct_eclasses(cpv, values["_eclasses_"])))
+                       values["INHERITED"] = ' '.join(sorted(values["_eclasses_"]))
 
                s = cpv.rfind("/")
                fp = os.path.join(self.location,cpv[:s],