From: Fabian Groffen Date: Mon, 9 Mar 2009 19:51:11 +0000 (-0000) Subject: Merged from trunk -r12790:12806 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=936d9e7b00e81843e3035fa8c9bda785ac50df9c;p=portage.git Merged from trunk -r12790:12806 | 12791 | Also support + character in eclass names. | | zmedico | | | 12792 | Fix typo from previous commit. | | zmedico | | | 12793 | for pointers cast to 32bit ints, we always want to abort on | | SpankMan | 64bit systems, not just when FEATURES=stricter | | 12794 | Generate a QA Notice when EXPORT_FUNCTIONS is called before | | zmedico | inherit, since it's incompatible with <=portage-2.1.6.7. | | 12795 | Replace things like [[ $PORTAGE_BUILDDIR/.prepared -nt | | zmedico | $WORKDIR ]] with simply [[ -e $PORTAGE_BUILDDIR/.prepared | | | ]] since the timestamp of $WORKDIR is practically always | | | newer due to the other hidden files that are created there | | | as each phase is executed. Thanks to Alfredo Tupone | | | for reporting. | | 12796 | Make the --digest warning message show for FEATURES=digest. | | zmedico | | | 12797 | Bug #261675 - When doebuild() is called by emerge, don't | | zmedico | call digestgen() for FEATURES=digest because it's redundant | | | and it can cause interference with parallel-fetch and | | | parallel builds. | | 12798 | Bug #261675 - Also don't call digestgen() from | | zmedico | parallel-fetch processes. | | 12799 | Bug #261675 - Generate manifests in advance when | | zmedico | FEATURES=digest is enabled. | | 12800 | Never call digestgen() inside doebuild() which only 'fetch' | | zmedico | has been requested since it's not needed in this case. | | 12801 | Reorganize code for --digests and FEATURES=digest. | | zmedico | | | 12802 | Make manifest mode with --force option cause existing | | zmedico | digests to be replaced for any files that exist in | | | ${DISTDIR}. This provides an alternative to ebuild --force | | | manifest for updating existing distfiles digests. Digests | | | are assumed to be corect for files that do not exist in | | | ${DISTDIR} since the user could simply remove the whole | | | Manifest if they wanted to regenerate digests for all files | | | (and it's safer to regenerate as few as possible because | | | it's less probably that a valid digest will get replaced by | | | an invalid one). | | 12803 | Fix _setitem() to account for the change to | | zmedico | serialize_eclasses = False. | | 12804 | Inside dyn_unpack(), only call pre/post phase hooks when | | zmedico | src_unpack is called (when .unpacked does not already | | | exist). | | 12805 | Use the ebuild_phase helper function to calls phase hooks | | zmedico | and fix broken trap calls that try to unset multiple signal | | | handlers at once without passing - as the first argument. | | 12806 | When generating variable names to hold EXPORT_FUNCTIONS | | zmedico | argument in, use $ECLASS_DEPTH as a unique id since it's a | | | lot simpler then encoding the eclass name in the variable | | | name. | svn path=/main/branches/prefix/; revision=12807 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 5014f6519..b8f5a6837 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -649,7 +649,6 @@ dyn_setup() { } dyn_unpack() { - [ "$(type -t pre_src_unpack)" == "function" ] && qa_call pre_src_unpack local newstuff="no" if [ -e "${WORKDIR}" ]; then local x @@ -682,7 +681,6 @@ dyn_unpack() { if [ -e "${WORKDIR}" ]; then if [ "$newstuff" == "no" ]; then vecho ">>> WORKDIR is up-to-date, keeping..." - [ "$(type -t post_src_unpack)" == "function" ] && qa_call post_src_unpack return 0 fi fi @@ -691,13 +689,12 @@ dyn_unpack() { install -m${PORTAGE_WORKDIR_MODE:-0700} -d "${WORKDIR}" || die "Failed to create dir '${WORKDIR}'" fi cd "${WORKDIR}" || die "Directory change failed: \`cd '${WORKDIR}'\`" + ebuild_phase pre_src_unpack vecho ">>> Unpacking source..." ebuild_phase src_unpack touch "${PORTAGE_BUILDDIR}/.unpacked" || die "IO Failure -- Failed 'touch .unpacked' in ${PORTAGE_BUILDDIR}" vecho ">>> Source unpacked in ${WORKDIR}" - cd "${PORTAGE_BUILDDIR}" - - [ "$(type -t post_src_unpack)" == "function" ] && qa_call post_src_unpack + ebuild_phase post_src_unpack } dyn_clean() { @@ -824,7 +821,7 @@ abort_handler() { echo eval ${3} #unset signal handler - trap SIGINT SIGQUIT + trap - SIGINT SIGQUIT } abort_prepare() { @@ -859,7 +856,7 @@ abort_install() { dyn_prepare() { - if [[ $PORTAGE_BUILDDIR/.prepared -nt $WORKDIR ]] ; then + if [[ -e $PORTAGE_BUILDDIR/.prepared ]] ; then vecho ">>> It appears that '$PF' is already prepared; skipping." vecho ">>> Remove '$PORTAGE_BUILDDIR/.prepared' to force prepare." return 0 @@ -882,12 +879,12 @@ dyn_prepare() { vecho ">>> Source prepared." ebuild_phase post_src_prepare - trap SIGINT SIGQUIT + trap - SIGINT SIGQUIT } dyn_configure() { - if [[ $PORTAGE_BUILDDIR/.configured -nt $WORKDIR ]] ; then + if [[ -e $PORTAGE_BUILDDIR/.configured ]] ; then vecho ">>> It appears that '$PF' is already configured; skipping." vecho ">>> Remove '$PORTAGE_BUILDDIR/.configured' to force configuration." return 0 @@ -895,23 +892,21 @@ dyn_configure() { trap abort_configure SIGINT SIGQUIT - [[ $(type -t pre_src_configure) = function ]] && \ - qa_call pre_src_configure + ebuild_phase pre_src_configure vecho ">>> Configuring source in $srcdir ..." ebuild_phase src_configure touch "$PORTAGE_BUILDDIR"/.configured vecho ">>> Source configured." - [[ $(type -t post_src_configure) = function ]] && \ - qa_call post_src_configure + ebuild_phase post_src_configure - trap SIGINT SIGQUIT + trap - SIGINT SIGQUIT } dyn_compile() { - if [[ $PORTAGE_BUILDDIR/.compiled -nt $WORKDIR ]] ; then + if [[ -e $PORTAGE_BUILDDIR/.compiled ]] ; then vecho ">>> It appears that '${PF}' is already compiled; skipping." vecho ">>> Remove '$PORTAGE_BUILDDIR/.compiled' to force compilation." return 0 @@ -919,17 +914,16 @@ dyn_compile() { trap abort_compile SIGINT SIGQUIT - [[ $(type -t pre_src_compile) = function ]] && \ - qa_call pre_src_compile + ebuild_phase pre_src_compile vecho ">>> Compiling source in ${srcdir} ..." ebuild_phase src_compile touch "$PORTAGE_BUILDDIR"/.compiled vecho ">>> Source compiled." - [ "$(type -t post_src_compile)" == "function" ] && qa_call post_src_compile + ebuild_phase post_src_compile - trap SIGINT SIGQUIT + trap - SIGINT SIGQUIT } dyn_test() { @@ -939,10 +933,9 @@ dyn_test() { # like it's supposed to here. ! hasq test ${USE} && export USE="${USE} test" fi - [ "$(type -t pre_src_test)" == "function" ] && qa_call pre_src_test - if [ "${PORTAGE_BUILDDIR}/.tested" -nt "${WORKDIR}" ]; then + ebuild_phase pre_src_test + if [[ -e $PORTAGE_BUILDDIR/.tested ]] ; then vecho ">>> It appears that ${PN} has already been tested; skipping." - [ "$(type -t post_src_test)" == "function" ] && qa_call post_src_test return fi trap "abort_test" SIGINT SIGQUIT @@ -962,23 +955,23 @@ dyn_test() { SANDBOX_PREDICT="${SANDBOX_PREDICT%:/}" fi - cd "${PORTAGE_BUILDDIR}" - touch .tested || die "Failed to 'touch .tested' in ${PORTAGE_BUILDDIR}" - [ "$(type -t post_src_test)" == "function" ] && qa_call post_src_test - trap SIGINT SIGQUIT + touch "$PORTAGE_BUILDDIR/.tested" || \ + die "Failed to 'touch .tested' in $PORTAGE_BUILDDIR" + ebuild_phase post_src_test + trap - SIGINT SIGQUIT } dyn_install() { [ -z "$PORTAGE_BUILDDIR" ] && die "${FUNCNAME}: PORTAGE_BUILDDIR is unset" if hasq noauto $FEATURES ; then rm -f "${PORTAGE_BUILDDIR}/.installed" - elif [[ ${PORTAGE_BUILDDIR}/.installed -nt ${WORKDIR} ]] ; then + elif [[ -e $PORTAGE_BUILDDIR/.installed ]] ; then vecho ">>> It appears that '${PF}' is already installed; skipping." vecho ">>> Remove '${PORTAGE_BUILDDIR}/.installed' to force install." return 0 fi trap "abort_install" SIGINT SIGQUIT - [ "$(type -t pre_src_install)" == "function" ] && qa_call pre_src_install + ebuild_phase pre_src_install rm -rf "${PORTAGE_BUILDDIR}/image" mkdir "${PORTAGE_BUILDDIR}/image" if [ -d "${S}" ]; then @@ -1007,8 +1000,7 @@ dyn_install() { touch "${PORTAGE_BUILDDIR}/.installed" vecho ">>> Completed installing ${PF} into ${ED}" vecho - cd ${PORTAGE_BUILDDIR} - [ "$(type -t post_src_install)" == "function" ] && qa_call post_src_install + ebuild_phase post_src_install cd "${PORTAGE_BUILDDIR}"/build-info set -f @@ -1041,7 +1033,7 @@ dyn_install() { then touch DEBUGBUILD fi - trap SIGINT SIGQUIT + trap - SIGINT SIGQUIT } dyn_preinst() { @@ -1166,6 +1158,13 @@ inherit() { debug-print "*** Multiple Inheritence (Level: ${ECLASS_DEPTH})" fi + if [[ -n $ECLASS && -n ${!__export_funcs_var} ]] ; then + echo "QA Notice: EXPORT_FUNCTIONS is called before inherit in" \ + "$ECLASS.eclass. For compatibility with <=portage-2.1.6.7," \ + "only call EXPORT_FUNCTIONS after inherit(s)." \ + | fmt -w 75 | while read ; do eqawarn "$REPLY" ; done + fi + local location local olocation local x @@ -1183,11 +1182,7 @@ inherit() { olocation="" export ECLASS="$1" - __export_funcs_var=__export_functions_$ECLASS - while [[ $__export_funcs_var =~ [-.] ]] ; do - __export_funcs_var=${__export_funcs_var/-/__dash__} - __export_funcs_var=${__export_funcs_var/./__dot__} - done + __export_funcs_var=__export_functions_$ECLASS_DEPTH unset $__export_funcs_var if [ "${EBUILD_PHASE}" != "depend" ] && \ diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 59217fdce..b6944ffa7 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -536,14 +536,18 @@ install_qa_check() { fi fi - if [[ $abort = yes ]] && [[ $gentoo_bug != yes ]] ; then - echo "Please do not file a Gentoo bug and instead" \ - "report the above QA issues directly to the upstream" \ - "developers of this software." | fmt -w 70 | \ - while read line ; do eqawarn "${line}" ; done - eqawarn "Homepage: ${HOMEPAGE}" + if [[ ${abort} == "yes" ]] ; then + if [[ ${gentoo_bug} == "yes" ]] ; then + die "poor code kills airplanes" + else + echo "Please do not file a Gentoo bug and instead" \ + "report the above QA issues directly to the upstream" \ + "developers of this software." | fmt -w 70 | \ + while read line ; do eqawarn "${line}" ; done + eqawarn "Homepage: ${HOMEPAGE}" + hasq stricter ${FEATURES} && die "poor code kills airplanes" + fi fi - [[ ${abort} == "yes" ]] && hasq stricter ${FEATURES} && die "poor code kills airplanes" fi # Compiled python objects do not belong in /usr/share (FHS violation) diff --git a/bin/repoman b/bin/repoman index 166d3967e..7d134b5b8 100755 --- a/bin/repoman +++ b/bin/repoman @@ -799,11 +799,50 @@ for x in scanlist: if options.mode == "manifest" or \ options.mode in ('commit', 'fix') and not options.pretend: + auto_assumed = set() + fetchlist_dict = portage.FetchlistDict(checkdir, + repoman_settings, portdb) + if options.mode == 'manifest' and options.force: + portage._doebuild_manifest_exempt_depend += 1 + try: + distdir = repoman_settings['DISTDIR'] + mf = portage.manifest.Manifest(checkdir, distdir, + fetchlist_dict=fetchlist_dict) + mf.create(requiredDistfiles=None, + assumeDistHashesAlways=True) + for distfiles in fetchlist_dict.itervalues(): + for distfile in distfiles: + if os.path.isfile(os.path.join(distdir, distfile)): + mf.fhashdict['DIST'].pop(distfile, None) + else: + auto_assumed.add(distfile) + mf.write() + finally: + portage._doebuild_manifest_exempt_depend -= 1 + repoman_settings["O"] = checkdir if not portage.digestgen([], repoman_settings, myportdb=portdb): print "Unable to generate manifest." dofail = 1 if options.mode == "manifest": + if not dofail and options.force and auto_assumed and \ + 'assume-digests' in repoman_settings.features: + # Show which digests were assumed despite the --force option + # being given. This output will already have been shown by + # digestgen() if assume-digests is not enabled, so only show + # it here if assume-digests is enabled. + pkgs = list(fetchlist_dict) + pkgs.sort() + portage.writemsg_stdout(" digest.assumed" + \ + portage.output.colorize("WARN", + str(len(auto_assumed)).rjust(18)) + "\n") + for cpv in pkgs: + fetchmap = fetchlist_dict[cpv] + pf = portage.catsplit(cpv)[1] + for distfile in sorted(fetchmap): + if distfile in auto_assumed: + portage.writemsg_stdout( + " %s::%s\n" % (pf, distfile)) continue elif dofail: sys.exit(1) diff --git a/man/repoman.1 b/man/repoman.1 index aca5587c0..b14cb17c2 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -18,6 +18,14 @@ Force commit to proceed, regardless of QA issues. For convenience, this option causes the most time consuming QA checks to be skipped. The commit message will include an indication that this option has been enabled, together with the usual portage version stamp. + +When used together with \fBmanifest\fR mode, \fB--force\fR causes existing +digests to be replaced for any files that exist in ${DISTDIR}. +Existing digests are assumed to be correct for files that would otherwise +have to be downloaded in order to recompute digests. \fBWARNING:\fR When +replacing existing digests, it is the user's responsibility to ensure that +files contained in ${DISTDIR} have the correct identities. Especially beware +of partially downloaded files. .TP \fB-q\fR, \fB--quiet\fR Be less verbose about extraneous info @@ -69,7 +77,8 @@ Scan directory tree for QA issues (short listing) Fix simple QA issues (stray digests, missing digests) .TP .B manifest -Generate a Manifest (fetches files if necessary) +Generate a Manifest (fetches distfiles if necessary). See the \fB\-\-force\fR +option if you would like to replace existing distfiles digests. .TP .B commit Scan directory tree for QA issues; if OK, commit via cvs diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 58dc1dba2..22f6bf4d1 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -10470,6 +10470,51 @@ class Scheduler(PollScheduler): return ebuild_phase.returncode + def _generate_digests(self): + """ + Generate digests if necessary for --digests or FEATURES=digest. + In order to avoid interference, this must done before parallel + tasks are started. + """ + + if '--fetchonly' in self.myopts: + return os.EX_OK + + digest = '--digest' in self.myopts + if not digest: + for pkgsettings in self.pkgsettings.itervalues(): + if 'digest' in pkgsettings.features: + digest = True + break + + if not digest: + return os.EX_OK + + for x in self._mergelist: + if not isinstance(x, Package) or \ + x.type_name != 'ebuild' or \ + x.operation != 'merge': + continue + pkgsettings = self.pkgsettings[x.root] + if '--digest' not in self.myopts and \ + 'digest' not in pkgsettings.features: + continue + portdb = x.root_config.trees['porttree'].dbapi + ebuild_path = portdb.findname(x.cpv) + if not ebuild_path: + writemsg_level( + "!!! Could not locate ebuild for '%s'.\n" \ + % x.cpv, level=logging.ERROR, noiselevel=-1) + return 1 + pkgsettings['O'] = os.path.dirname(ebuild_path) + if not portage.digestgen([], pkgsettings, myportdb=portdb): + writemsg_level( + "!!! Unable to generate manifest for '%s'.\n" \ + % x.cpv, level=logging.ERROR, noiselevel=-1) + return 1 + + return os.EX_OK + def _check_manifests(self): # Verify all the manifests now so that the user is notified of failure # as soon as possible. @@ -10678,6 +10723,10 @@ class Scheduler(PollScheduler): self.pkgsettings[root] = portage.config( clone=root_config.settings) + rval = self._generate_digests() + if rval != os.EX_OK: + return rval + rval = self._check_manifests() if rval != os.EX_OK: return rval @@ -14350,8 +14399,13 @@ def action_build(settings, trees, mtimedb, if pretend or fetchonly: # make the mtimedb readonly mtimedb.filename = None - if "--digest" in myopts: - msg = "The --digest option can prevent corruption from being" + \ + if '--digest' in myopts or 'digest' in settings.features: + if '--digest' in myopts: + msg = "The --digest option" + else: + msg = "The FEATURES=digest setting" + + msg += " can prevent corruption from being" + \ " noticed. The `repoman manifest` command is the preferred" + \ " way to generate manifests and it is capable of doing an" + \ " entire repository or category at once." @@ -14657,20 +14711,6 @@ def action_build(settings, trees, mtimedb, # Convert Atom instances to plain str. mtimedb["resume"]["favorites"] = [str(x) for x in favorites] - if ("--digest" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts): - for pkgline in mydepgraph.altlist(): - if pkgline[0]=="ebuild" and pkgline[3]=="merge": - y = trees[pkgline[1]]["porttree"].dbapi.findname(pkgline[2]) - tmpsettings = portage.config(clone=settings) - edebug = 0 - if settings.get("PORTAGE_DEBUG", "") == "1": - edebug = 1 - retval = portage.doebuild( - y, "digest", settings["ROOT"], tmpsettings, edebug, - ("--pretend" in myopts), - mydbapi=trees[pkgline[1]]["porttree"].dbapi, - tree="porttree") - pkglist = mydepgraph.altlist() mydepgraph.saveNomergeFavorites() mydepgraph.break_refs(pkglist) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 4ed25e280..28461d9c4 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -5996,6 +5996,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"])) emerge_skip_distfiles = returnpid + emerge_skip_digest = returnpid # Only try and fetch the files if we are going to need them ... # otherwise, if user has FEATURES=noauto and they run `ebuild clean # unpack compile install`, we will try and fetch 4 times :/ @@ -6052,7 +6053,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, elif mydo == "digest": return not digestgen(aalist, mysettings, overwrite=1, myportdb=mydbapi) - elif "digest" in mysettings.features: + elif mydo != 'fetch' and not emerge_skip_digest and \ + "digest" in mysettings.features: + # Don't do this when called by emerge or when called just + # for fetch (especially parallel-fetch) since it's not needed + # and it can interfere with parallel tasks. digestgen(aalist, mysettings, overwrite=0, myportdb=mydbapi) except portage.exception.PermissionDenied, e: writemsg("!!! Permission Denied: %s\n" % (e,), noiselevel=-1) diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py index 4f74b767d..5222223c6 100644 --- a/pym/portage/cache/metadata.py +++ b/pym/portage/cache/metadata.py @@ -71,8 +71,7 @@ class database(flat_hash.database): def _setitem(self, cpv, values): if "_eclasses_" in values: values = ProtectedDict(values) - values["INHERITED"] = ' '.join(sorted( - reconstruct_eclasses(cpv, values["_eclasses_"]))) + values["INHERITED"] = ' '.join(sorted(values["_eclasses_"])) s = cpv.rfind("/") fp = os.path.join(self.location,cpv[:s],