From 4da29fea55338c06f1b9a8143c8629bea9e457f8 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 9 Feb 2007 22:28:52 +0000 Subject: [PATCH] Merged from trunk 5905:5931 Update modules doc for the new portage namespace. For backward compatibility with api consumers, _dep_check_strict behavior will be explicitly enabled as necessary. Thanks to Brian for reporting issues with porthole and equery. For bug #165783, handle an InvalidDependString exception triggered by an invalid SRC_URI. move die into isolated-functions.sh (WarnerBro) Only echo PORTAGE_LOG_FILE if it's actually set (may not be if sesandbox is enabled). For bug #165780, make config-protect code consistently ignore '.*~' and '.*.bak'. For bug #165783, handle a potential InvalidDependString exception inside digestgen(). Enable strict dep checking for repoman. For bug #165783, handle a potential InvalidDependString exception inside doebuild(). For bug #165783, handle a potential InvalidDependString exception in repoman and report a SRC_URI.syntax failure. Synchronize required hash code in all relevant places. Turn on strict dep checking to fix the isvalidatom test. Make the "lfull" output more succinct by omitting categories for which there are no violations. die requires diefunc and dump_trace too; duh; also move extdebug into isolated-functions.sh (WarnerBro) Create namespace for emerge (genone) svn path=/main/branches/prefix/; revision=5933 --- bin/dispatch-conf | 2 +- bin/ebuild | 2 + bin/ebuild.sh | 80 +------------------ bin/emerge | 23 ++++-- bin/isolated-functions.sh | 75 ++++++++++++++++- bin/repoman | 41 ++++++---- man/portage.5 | 2 +- pym/{portage/emergehelp.py => emerge/help.py} | 0 pym/portage/__init__.py | 36 ++++++--- pym/portage/dep.py | 5 +- pym/portage/manifest.py | 5 +- pym/portage/tests/dep/test_isvalidatom.py | 2 + 12 files changed, 157 insertions(+), 116 deletions(-) rename pym/{portage/emergehelp.py => emerge/help.py} (100%) diff --git a/bin/dispatch-conf b/bin/dispatch-conf index 996105945..10dc33830 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -27,7 +27,7 @@ except ImportError: import dispatch_conf, portage_const from portage.process import find_binary -FIND_EXTANT_CONFIGS = "find '%s' %s -iname '._cfg????_%s'" +FIND_EXTANT_CONFIGS = "find '%s' %s -iname '._cfg????_%s' ! -iname '.*~' ! -iname '.*.bak'" DIFF_CONTENTS = 'diff -Nu %s %s' DIFF_CVS_INTERP = 'diff -Nu %s %s | grep "^[+-][^+-]" | grep -v "# .Header:.*"' DIFF_WSCOMMENTS = 'diff -Nu %s %s | grep "^[+-][^+-]" | grep -v "^[-+]#" | grep -v "^[-+][:space:]*$"' diff --git a/bin/ebuild b/bin/ebuild index 6f90d110d..08c382083 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -28,6 +28,8 @@ except ImportError: import portage import portage.util, portage.const +import portage.dep +portage.dep._dep_check_strict = True # do this _after_ 'import portage' to prevent unnecessary tracing if debug and "python-trace" in portage.features: diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 8138f99e8..e8361538b 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -64,22 +64,14 @@ unalias -a # Unset some variables that break things. unset GZIP BZIP BZIP2 CDPATH GREP_OPTIONS GREP_COLOR GLOBIGNORE -# We need this next line for "die" and "assert". It expands -# It _must_ preceed all the calls to die and assert. -shopt -s expand_aliases -alias die='diefunc "$FUNCNAME" "$LINENO" "$?"' -alias assert='_pipestatus="${PIPESTATUS[*]}"; [[ "${_pipestatus// /}" -eq 0 ]] || diefunc "$FUNCNAME" "$LINENO" "$_pipestatus"' -alias save_IFS='[ "${IFS:-unset}" != "unset" ] && old_IFS="${IFS}"' -alias restore_IFS='if [ "${old_IFS:-unset}" != "unset" ]; then IFS="${old_IFS}"; unset old_IFS; else unset IFS; fi' - -OCC="$CC" -OCXX="$CXX" - export PATH="${DEFAULT_PATH}:${PORTAGE_BIN_PATH}:${ROOTPATH}" [ ! -z "$PREROOTPATH" ] && export PATH="${PREROOTPATH%%:}:$PATH" source "@PORTAGE_BASE@"/bin/isolated-functions.sh &>/dev/null +OCC="$CC" +OCXX="$CXX" + [[ $PORTAGE_QUIET != "" ]] && export PORTAGE_QUIET # the sandbox is disabled by default except when overridden in the relevant stages @@ -425,72 +417,6 @@ register_die_hook() { export EBUILD_DEATH_HOOKS="${EBUILD_DEATH_HOOKS} $*" } -diefunc() { - local funcname="$1" lineno="$2" exitcode="$3" - shift 3 - echo >&2 - echo "!!! ERROR: $CATEGORY/$PF failed." >&2 - dump_trace 2 1>&2 - echo " $(basename "${BASH_SOURCE[1]}"), line ${BASH_LINENO[0]}: Called die" 1>&2 - echo >&2 - echo "!!! ${*:-(no error message)}" >&2 - echo "!!! If you need support, post the topmost build error, and the call stack if relevant." >&2 - echo "!!! A complete build log is located at '${PORTAGE_LOG_FILE}'." >&2 - echo >&2 - if [ -n "${EBUILD_OVERLAY_ECLASSES}" ] ; then - echo "This ebuild used the following eclasses from overlays:" >&2 - echo >&2 - for x in ${EBUILD_OVERLAY_ECLASSES} ; do - echo " ${x}" >&2 - done - echo >&2 - fi - - if [ "${EBUILD_PHASE/depend}" == "${EBUILD_PHASE}" ]; then - local x - for x in $EBUILD_DEATH_HOOKS; do - ${x} "$@" >&2 1>&2 - done - fi - - # subshell die support - kill -s SIGTERM ${EBUILD_MASTER_PID} - exit 1 -} - -shopt -s extdebug &> /dev/null - -# usage- first arg is the number of funcs on the stack to ignore. -# defaults to 1 (ignoring dump_trace) -dump_trace() { - local funcname="" sourcefile="" lineno="" n e s="yes" - - declare -i strip=1 - - if [[ -n $1 ]]; then - strip=$(( $1 )) - fi - - echo "Call stack:" - for (( n = ${#FUNCNAME[@]} - 1, p = ${#BASH_ARGV[@]} ; n > $strip ; n-- )) ; do - funcname=${FUNCNAME[${n} - 1]} - sourcefile=$(basename ${BASH_SOURCE[${n}]}) - lineno=${BASH_LINENO[${n} - 1]} - # Display function arguments - args= - if [[ -n "${BASH_ARGV[@]}" ]]; then - for (( j = 1 ; j <= ${BASH_ARGC[${n} - 1]} ; ++j )); do - newarg=${BASH_ARGV[$(( p - j - 1 ))]} - args="${args:+${args} }'${newarg}'" - done - (( p -= ${BASH_ARGC[${n} - 1]} )) - fi - echo " ${sourcefile}, line ${lineno}: Called ${funcname}${args:+ ${args}}" - done -} - - - #if no perms are specified, dirs/files will have decent defaults #(not secretive, but not stupid) umask 022 diff --git a/bin/emerge b/bin/emerge index bcbdb9323..47b269a93 100755 --- a/bin/emerge +++ b/bin/emerge @@ -32,7 +32,8 @@ del os.environ["PORTAGE_LEGACY_GLOBALS"] from portage import digraph, portdbapi from portage.const import NEWS_LIB_PATH -import portage.emergehelp, portage.xpak, commands, errno, re, socket, time, types +import emerge.help +import portage.xpak, commands, errno, re, socket, time, types import portage.output from portage.output import blue, bold, colorize, darkblue, darkgreen, darkred, green, \ havecolor, nc_len, nocolor, red, teal, turquoise, white, xtermTitle, \ @@ -42,6 +43,7 @@ good = create_color_func("GOOD") bad = create_color_func("BAD") import portage.dep +portage.dep._dep_check_strict = True import portage.util import portage.locks import portage.exception @@ -2487,9 +2489,15 @@ class depgraph: # size verbose mysize=0 if x[0] == "ebuild" and x[-1]!="nomerge": - myfilesdict = portdb.getfetchsizes( - pkg_key, useflags=self.useFlags[myroot][pkg_key], - debug=self.edebug) + try: + myfilesdict = portdb.getfetchsizes(pkg_key, + useflags=self.useFlags[myroot][pkg_key], + debug=self.edebug) + except portage.exception.InvalidDependString, e: + src_uri = portdb.aux_get(pkg_key, ["SRC_URI"])[0] + show_invalid_depstring_notice(x, src_uri, str(e)) + del e + sys.exit(1) if myfilesdict is None: myfilesdict="[empty/missing/bad digest]" else: @@ -3604,7 +3612,8 @@ def chk_updated_cfg_files(target_root, config_protect): else: mycommand = "cd '%s'; find . -maxdepth 1 -iname '._cfg????_%s'" % \ os.path.split(x.rstrip(os.path.sep)) - a = commands.getstatusoutput(mycommand) + a = commands.getstatusoutput(mycommand + \ + " ! -iname '.*~' ! -iname '.*.bak'") if a[0] != 0: print >> sys.stderr, " " + bad("*")+ " error scanning '%s'" % x else: @@ -5250,7 +5259,7 @@ def emerge_main(): trees[settings["ROOT"]]["vartree"].dbapi) sys.exit(0) elif "--help" in myopts: - portage.emergehelp.help(myaction, myopts, portage.output.havecolor) + emerge.help.help(myaction, myopts, portage.output.havecolor) sys.exit(0) if "--debug" in myopts: @@ -5258,7 +5267,7 @@ def emerge_main(): print "myopts", myopts if not myaction and not myfiles and "--resume" not in myopts: - portage.emergehelp.help(myaction, myopts, portage.output.havecolor) + emerge.help.help(myaction, myopts, portage.output.havecolor) sys.exit(1) # check if root user is the current user for the actions where emerge needs this diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 1862d87b0..d994333fe 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -1,6 +1,79 @@ # Copyright 1999-2006 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Header$ +# $Id$ + +# We need this next line for "die" and "assert". It expands +# It _must_ preceed all the calls to die and assert. +shopt -s expand_aliases +alias die='diefunc "$FUNCNAME" "$LINENO" "$?"' +alias assert='_pipestatus="${PIPESTATUS[*]}"; [[ "${_pipestatus// /}" -eq 0 ]] || diefunc "$FUNCNAME" "$LINENO" "$_pipestatus"' +alias save_IFS='[ "${IFS:-unset}" != "unset" ] && old_IFS="${IFS}"' +alias restore_IFS='if [ "${old_IFS:-unset}" != "unset" ]; then IFS="${old_IFS}"; unset old_IFS; else unset IFS; fi' + +shopt -s extdebug + +# usage- first arg is the number of funcs on the stack to ignore. +# defaults to 1 (ignoring dump_trace) +dump_trace() { + local funcname="" sourcefile="" lineno="" n e s="yes" + + declare -i strip=1 + + if [[ -n $1 ]]; then + strip=$(( $1 )) + fi + + echo "Call stack:" + for (( n = ${#FUNCNAME[@]} - 1, p = ${#BASH_ARGV[@]} ; n > $strip ; n-- )) ; do + funcname=${FUNCNAME[${n} - 1]} + sourcefile=$(basename ${BASH_SOURCE[${n}]}) + lineno=${BASH_LINENO[${n} - 1]} + # Display function arguments + args= + if [[ -n "${BASH_ARGV[@]}" ]]; then + for (( j = 1 ; j <= ${BASH_ARGC[${n} - 1]} ; ++j )); do + newarg=${BASH_ARGV[$(( p - j - 1 ))]} + args="${args:+${args} }'${newarg}'" + done + (( p -= ${BASH_ARGC[${n} - 1]} )) + fi + echo " ${sourcefile}, line ${lineno}: Called ${funcname}${args:+ ${args}}" + done +} + +diefunc() { + local funcname="$1" lineno="$2" exitcode="$3" + shift 3 + echo >&2 + echo "!!! ERROR: $CATEGORY/$PF failed." >&2 + dump_trace 2 1>&2 + echo " $(basename "${BASH_SOURCE[1]}"), line ${BASH_LINENO[0]}: Called die" 1>&2 + echo >&2 + echo "!!! ${*:-(no error message)}" >&2 + echo "!!! If you need support, post the topmost build error, and the call stack if relevant." >&2 + [ -n "${PORTAGE_LOG_FILE}" ] && \ + echo "!!! A complete build log is located at '${PORTAGE_LOG_FILE}'." >&2 + echo >&2 + if [ -n "${EBUILD_OVERLAY_ECLASSES}" ] ; then + echo "This ebuild used the following eclasses from overlays:" >&2 + echo >&2 + for x in ${EBUILD_OVERLAY_ECLASSES} ; do + echo " ${x}" >&2 + done + echo >&2 + fi + + if [ "${EBUILD_PHASE/depend}" == "${EBUILD_PHASE}" ]; then + local x + for x in $EBUILD_DEATH_HOOKS; do + ${x} "$@" >&2 1>&2 + done + fi + + # subshell die support + kill -s SIGTERM ${EBUILD_MASTER_PID} + exit 1 +} quiet_mode() { [[ ${PORTAGE_QUIET} -eq 1 ]] diff --git a/bin/repoman b/bin/repoman index 29e209569..f60ced470 100755 --- a/bin/repoman +++ b/bin/repoman @@ -33,6 +33,8 @@ del os.environ["PORTAGE_LEGACY_GLOBALS"] import portage.checksum import portage.const import portage.dep +import portage.dep +portage.dep._dep_check_strict = True import portage.exception import portage.cvstree import time @@ -159,6 +161,7 @@ qahelp={ "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)", "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)", "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)", + "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)", "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error", "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.", "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.", @@ -303,8 +306,7 @@ def last(): if x not in qawarnings: dofail=1 else: - if mymode!="lfull": - continue + continue print " "+ x.ljust(20), if stats[x]==0: print green(`stats[x]`) @@ -919,6 +921,22 @@ for x in scanlist: mf = Manifest(checkdir, repoman_settings["DISTDIR"]) mydigests=mf.getTypeDigests("DIST") + fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb) + myfiles_all = [] + for mykey in fetchlist_dict: + try: + myfiles_all.extend(fetchlist_dict[mykey]) + except portage.exception.InvalidDependString, e: + stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1 + fails["SRC_URI.syntax"].append(mykey+".ebuild SRC_URI: "+str(e)) + del fetchlist_dict + myfiles_all = set(myfiles_all) + for entry in mydigests: + if entry not in myfiles_all: + stats["digestentry.unused"] += 1 + fails["digestentry.unused"].append(checkdir+"::"+entry) + del myfiles_all + if os.path.exists(checkdir+"/files"): filesdirlist=os.listdir(checkdir+"/files") for y in filesdirlist: @@ -952,7 +970,11 @@ for x in scanlist: fails["digest.stray"].append(x+"/files/"+y) else: # We have an ebuild - myuris, myfiles = portdb.getfetchlist(mykey, all=True) + try: + myuris, myfiles = portdb.getfetchlist(mykey, all=True) + except portage.exception.InvalidDependString, e: + # Already handled above. + continue uri_dict = {} for myu in myuris: @@ -1017,18 +1039,7 @@ for x in scanlist: fails["file.name"].append("%s/files/%s: char '%s'" % (checkdir, y, c)) break - fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb) - myfiles_all = [] - for myfiles in fetchlist_dict.itervalues(): - myfiles_all.extend(myfiles) - del myfiles - del fetchlist_dict - myfiles_all = set(myfiles_all) - for entry in mydigests: - if entry not in myfiles_all: - stats["digestentry.unused"] += 1 - fails["digestentry.unused"].append(checkdir+"::"+entry) - del mydigests, myfiles_all + del mydigests if "ChangeLog" not in checkdirlist: stats["changelog.missing"]+=1 diff --git a/man/portage.5 b/man/portage.5 index 4b19dc909..b59eb0a0e 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -366,7 +366,7 @@ override. .I Example: .nf -portdbapi.auxdbmodule = cache.metadata_overlay.database +portdbapi.auxdbmodule = portage.cache.metadata_overlay.database .fi The metadata_overlay cache module makes it possible to disable diff --git a/pym/portage/emergehelp.py b/pym/emerge/help.py similarity index 100% rename from pym/portage/emergehelp.py rename to pym/emerge/help.py diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index b7fa6ae47..0188e2076 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2702,19 +2702,24 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None _doebuild_manifest_exempt_depend += 1 distfiles_map = {} fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb) - for cpv, fetchlist in fetchlist_dict.iteritems(): - for myfile in fetchlist: - distfiles_map.setdefault(myfile, []).append(cpv) + for cpv in fetchlist_dict: + try: + for myfile in fetchlist_dict[cpv]: + distfiles_map.setdefault(myfile, []).append(cpv) + except portage.exception.InvalidDependString, e: + writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg("!!! Invalid SRC_URI for '%s'.\n" % cpv, noiselevel=-1) + del e + return 0 mf = Manifest(mysettings["O"], mysettings["DISTDIR"], fetchlist_dict=fetchlist_dict) # Don't require all hashes since that can trigger excessive # fetches when sufficient digests already exist. To ease transition # while Manifest 1 is being removed, only require hashes that will # exist before and after the transition. - required_hash_types = set(portage.const.MANIFEST1_HASH_FUNCTIONS - ).intersection(portage.const.MANIFEST2_HASH_FUNCTIONS) - required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) + required_hash_types = set() required_hash_types.add("size") + required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) dist_hashes = mf.fhashdict.get("DIST", {}) missing_hashes = set() for myfile in distfiles_map: @@ -3562,10 +3567,16 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Make sure we get the correct tree in case there are overlays. mytree = os.path.realpath( os.path.dirname(os.path.dirname(mysettings["O"]))) - newuris, alist = mydbapi.getfetchlist( - mycpv, mytree=mytree, mysettings=mysettings) - alluris, aalist = mydbapi.getfetchlist( - mycpv, mytree=mytree, all=True, mysettings=mysettings) + try: + newuris, alist = mydbapi.getfetchlist( + mycpv, mytree=mytree, mysettings=mysettings) + alluris, aalist = mydbapi.getfetchlist( + mycpv, mytree=mytree, all=True, mysettings=mysettings) + except portage.exception.InvalidDependString, e: + writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg("!!! Invalid SRC_URI for '%s'.\n" % mycpv, noiselevel=-1) + del e + return 1 mysettings["A"] = " ".join(alist) mysettings["AA"] = " ".join(aalist) if ("mirror" in features) or fetchall: @@ -3577,8 +3588,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Skip files that we already have digests for. mf = Manifest(mysettings["O"], mysettings["DISTDIR"]) mydigests = mf.getTypeDigests("DIST") + required_hash_types = set() + required_hash_types.add("size") + required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) for filename, hashes in mydigests.iteritems(): - if len(hashes) == len(mf.hashes): + if not required_hash_types.difference(hashes): checkme = [i for i in checkme if i != filename] fetchme = [i for i in fetchme \ if os.path.basename(i) != filename] diff --git a/pym/portage/dep.py b/pym/portage/dep.py index bf8c0ae05..fb11248ec 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -118,8 +118,9 @@ def paren_enclose(mylist): # This is just for use by emerge so that it can enable a backward compatibility # mode in order to gracefully deal with installed packages that have invalid -# atoms or dep syntax. -_dep_check_strict = True +# atoms or dep syntax. For backward compatibility with api consumers, strict +# behavior will be explicitly enabled as necessary. +_dep_check_strict = False def use_reduce(deparray, uselist=[], masklist=[], matchall=0, excludeall=[]): """ diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 08ac13932..732d403f8 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -473,6 +473,9 @@ class Manifest(object): # repoman passes in an empty list, which implies that all distfiles # are required. requiredDistfiles = distlist.copy() + required_hash_types = set() + required_hash_types.add("size") + required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) for f in distlist: fname = os.path.join(self.distdir, f) mystat = None @@ -486,7 +489,7 @@ class Manifest(object): (assumeDistHashesAlways and mystat is not None and \ len(distfilehashes[f]) == len(self.hashes) and \ distfilehashes[f]["size"] == mystat.st_size)) and \ - portage.const.MANIFEST2_REQUIRED_HASH in distfilehashes[f].keys(): + not required_hash_types.difference(distfilehashes[f]): self.fhashdict["DIST"][f] = distfilehashes[f] else: try: diff --git a/pym/portage/tests/dep/test_isvalidatom.py b/pym/portage/tests/dep/test_isvalidatom.py index 2ba91b7cc..054205eb9 100644 --- a/pym/portage/tests/dep/test_isvalidatom.py +++ b/pym/portage/tests/dep/test_isvalidatom.py @@ -5,6 +5,8 @@ from unittest import TestCase from portage.dep import isvalidatom +import portage.dep +portage.dep._dep_check_strict = True class IsValidAtom(TestCase): """ A simple testcase for isvalidatom -- 2.26.2