From 44e072f9a7c9c60166c4cedbdf746fac171fcbed Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 27 Jun 2007 07:57:53 +0000 Subject: [PATCH] Merged from trunk 7038:7052 +------------------------------------------------------------------------+ | 7039 | Add sanity checks in fetch() and digestgen() to | | zmedico | automatically detect and handle invalid empty distfiles | | | since some users have reported difficulty when trying to | | | create digests. | |---------+--------------------------------------------------------------| | 7041 | Add one more size == 0 check to digestgen(). | | zmedico | | |---------+--------------------------------------------------------------| | 7042 | Short circuit the loop as soon as missing hashes are | | zmedico | detected. | |---------+--------------------------------------------------------------| | 7044 | Short circuit the loop as soon as missing hashes are | | zmedico | detected. | |---------+--------------------------------------------------------------| | 7046 | Preserve merge order in elog output. | | zmedico | | |---------+--------------------------------------------------------------| | 7047 | Send the ebuild die message through eerror and make sure | | zmedico | that elog_process() gets called when an ebuild dies. This | | | makes the elog echo module display the die message last in | | | case other elog messages push the original die message off | | | of the top of the screen. | |---------+--------------------------------------------------------------| | 7050 | Format the callstack indentation more like it used to be. | | zmedico | | |---------+--------------------------------------------------------------| | 7052 | For bug #144333, add a --reinstall=changed-use option which | | zmedico | behaves similar to how --newuse behaved in portage-2.0.x | | | days. | +------------------------------------------------------------------------+ svn path=/main/branches/prefix/; revision=7054 --- bin/isolated-functions.sh | 35 +++++++++++++------------- pym/emerge/__init__.py | 44 ++++++++++++++++++++++---------- pym/portage/__init__.py | 49 +++++++++++++++++++++++------------- pym/portage/dbapi/vartree.py | 48 ++++++++++++++++++++++------------- pym/portage/elog/mod_echo.py | 20 ++++++--------- pym/portage/manifest.py | 4 +-- 6 files changed, 120 insertions(+), 80 deletions(-) diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index ac10d087f..594c4e8ec 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -23,7 +23,7 @@ dump_trace() { strip=$(( $1 )) fi - echo "Call stack:" + eerror "Call stack:" for (( n = ${#FUNCNAME[@]} - 1, p = ${#BASH_ARGV[@]} ; n > $strip ; n-- )) ; do funcname=${FUNCNAME[${n} - 1]} sourcefile=$(basename ${BASH_SOURCE[${n}]}) @@ -37,31 +37,32 @@ dump_trace() { done (( p -= ${BASH_ARGC[${n} - 1]} )) fi - echo " ${sourcefile}, line ${lineno}: Called ${funcname}${args:+ ${args}}" + eerror " ${sourcefile}, line ${lineno}: Called ${funcname}${args:+ ${args}}" done } diefunc() { local funcname="$1" lineno="$2" exitcode="$3" shift 3 - echo >&2 - echo "!!! ERROR: $CATEGORY/$PF failed." >&2 - dump_trace 2 1>&2 - echo " $(basename "${BASH_SOURCE[1]}"), line ${BASH_LINENO[0]}: Called die" 1>&2 - echo >&2 - echo "!!! ${*:-(no error message)}" >&2 - echo "!!! If you need support, post the topmost build error, and the call stack if relevant." >&2 + eerror + eerror "ERROR: $CATEGORY/$PF failed." + dump_trace 2 + eerror " ${BASH_SOURCE[1]##*/}, line ${BASH_LINENO[0]}: Called die" + eerror + eerror "${*:-(no error message)}" + eerror "If you need support, post the topmost build error, and the call stack if relevant." [ -n "${PORTAGE_LOG_FILE}" ] && \ - echo "!!! A complete build log is located at '${PORTAGE_LOG_FILE}'." >&2 - echo >&2 + eerror "A complete build log is located at '${PORTAGE_LOG_FILE}'." if [ -n "${EBUILD_OVERLAY_ECLASSES}" ] ; then - echo "This ebuild used the following eclasses from overlays:" >&2 - echo >&2 - for x in ${EBUILD_OVERLAY_ECLASSES} ; do - echo " ${x}" >&2 - done - echo >&2 + eerror "This ebuild used eclasses from overlays: ${EBUILD_OVERLAY_ECLASSES}" + fi + if [ ${EBUILD#${PORTDIR}/} == ${EBUILD} ] ; then + local overlay=${EBUILD%/*} + overlay=${overlay%/*} + overlay=${overlay%/*} + eerror "This ebuild is from an overlay: '${overlay}/'" fi + eerror if [ "${EBUILD_PHASE/depend}" == "${EBUILD_PHASE}" ]; then local x diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 66bc34d4a..4e03f60ae 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -344,6 +344,7 @@ def create_depgraph_params(myopts, myaction): sub=[] if "--update" in myopts or \ "--newuse" in myopts or \ + "--reinstall" in myopts or \ "--noreplace" in myopts or \ myaction in ("system", "world"): add.extend(["selective"]) @@ -1042,6 +1043,21 @@ class depgraph(object): f.end_paragraph(1) f.writer.flush() + def _reinstall_for_flags(self, forced_flags, + orig_use, org_iuse, cur_use, cur_iuse): + if "--newuse" in self.myopts: + if org_iuse.symmetric_difference( + cur_iuse).difference(forced_flags): + return True + elif org_iuse.intersection(orig_use) != \ + cur_iuse.intersection(cur_use): + return True + elif "changed-use" in self.myopts.get("--reinstall","").split(","): + if org_iuse.intersection(orig_use) != \ + cur_iuse.intersection(cur_use): + return True + return False + def create(self, mybigkey, myparent=None, addme=1, myuse=None, priority=DepPriority(), rev_dep=False, arg=None): """ @@ -1115,7 +1131,9 @@ class depgraph(object): """ If we aren't merging, perform the --newuse check. If the package has new iuse flags or different use flags then if --newuse is specified, we need to merge the package. """ - if merging==0 and "--newuse" in self.myopts and \ + if merging == 0 and \ + ("--newuse" in self.myopts or + "--reinstall" in self.myopts) and \ vardbapi.cpv_exists(mykey): pkgsettings.setcpv(mykey, mydb=mydbapi) forced_flags = set() @@ -1126,12 +1144,9 @@ class depgraph(object): mydbapi.aux_get(mykey, ["IUSE"])[0].split())) old_iuse = set(filter_iuse_defaults( vardbapi.aux_get(mykey, ["IUSE"])[0].split())) - if iuses.symmetric_difference( - old_iuse).difference(forced_flags): + if self._reinstall_for_flags( + forced_flags, old_use, old_iuse, myuse, iuses): merging = 1 - elif old_iuse.intersection(old_use) != \ - iuses.intersection(myuse): - merging=1 if addme and merging == 1: mybigkey.append("merge") @@ -1597,7 +1612,9 @@ class depgraph(object): if myeb_pkg_matches: myeb_pkg = portage.best(myeb_pkg_matches) - if myeb_pkg and "--newuse" in self.myopts: + if myeb_pkg and \ + ("--newuse" in self.myopts or \ + "--reinstall" in self.myopts): iuses = set(filter_iuse_defaults( bindb.aux_get(myeb_pkg, ["IUSE"])[0].split())) old_use = bindb.aux_get(myeb_pkg, ["USE"])[0].split() @@ -1616,11 +1633,8 @@ class depgraph(object): if "--usepkgonly" not in self.myopts and myeb: cur_iuse = set(filter_iuse_defaults( portdb.aux_get(myeb, ["IUSE"])[0].split())) - if iuses.symmetric_difference( - cur_iuse).difference(forced_flags): - myeb_pkg = None - elif iuses.intersection(old_use) != \ - cur_iuse.intersection(now_use): + if self._reinstall_for_flags( + forced_flags, old_use, iuses, now_use, cur_iuse): myeb_pkg = None if myeb_pkg: binpkguseflags = \ @@ -3230,7 +3244,8 @@ class MergeTask(object): pkgsettings, self.edebug, mydbapi=portdb, tree="porttree") del pkgsettings["PORTAGE_BINPKG_TMPFILE"] - if "--buildpkgonly" in self.myopts: + if retval != os.EX_OK or \ + "--buildpkgonly" in self.myopts: elog_process(pkg_key, pkgsettings) if retval != os.EX_OK: return retval @@ -5318,6 +5333,9 @@ def parse_opts(tmpcmdline, silent=False): "help":"include unnecessary build time dependencies", "type":"choice", "choices":("y", "n") + }, + "--reinstall": { + "help":"specify conditions to trigger package reinstallation" } } diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 31dafff00..c5cb97f64 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2709,7 +2709,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if not os.access(myfile_path, os.R_OK): writemsg("!!! Failed to adjust permissions:" + \ " %s\n" % str(e), noiselevel=-1) - if myfile not in mydigests: + + # If the file is empty then it's obviously invalid. Remove + # the empty file and try to download if possible. + if mystat.st_size == 0: + if can_fetch: + try: + os.unlink(myfile_path) + except EnvironmentError: + pass + elif myfile not in mydigests: # We don't have a digest, but the file exists. We must # assume that it is fully downloaded. continue @@ -2833,6 +2842,17 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", writemsg("!!! Failed to adjust permissions:" + \ " %s\n" % str(e), noiselevel=-1) + # If the file is empty then it's obviously invalid. Don't + # trust the return value from the fetcher. Remove the + # empty file and try to download again. + try: + if os.stat(myfile_path).st_size == 0: + os.unlink(myfile_path) + fetched = 0 + continue + except EnvironmentError: + pass + if mydigests!=None and mydigests.has_key(myfile): try: mystat = os.stat(myfile_path) @@ -2967,16 +2987,23 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None continue if required_hash_types.difference(myhashes): missing_hashes.add(myfile) + continue + if myhashes["size"] == 0: + missing_hashes.add(myfile) if missing_hashes: missing_files = [] for myfile in missing_hashes: try: - os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) except OSError, e: if e.errno != errno.ENOENT: raise del e missing_files.append(myfile) + else: + # If the file is empty then it's obviously invalid. + if st.st_size == 0: + missing_files.append(myfile) if missing_files: mytree = os.path.realpath(os.path.dirname( os.path.dirname(mysettings["O"]))) @@ -3238,8 +3265,9 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m mysettings["FILESDIR"] = os.path.join(pkg_dir, "files") mysettings["PF"] = mypv - mysettings["ECLASSDIR"] = os.path.join(mysettings["PORTDIR"], "eclass") - mysettings["SANDBOX_LOG"] = mycpv.replace(os.path.sep, "_-_") + mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) + mysettings["ECLASSDIR"] = mysettings["PORTDIR"]+"/eclass" + mysettings["SANDBOX_LOG"] = mycpv.replace("/", "_-_") mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles)+"\n"+CUSTOM_PROFILE_PATH mysettings["P"] = mysplit[0]+"-"+mysplit[1] @@ -3985,19 +4013,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, print "!!! Unknown mydo:",mydo return 1 - if retval != os.EX_OK and tree == "porttree": - for i in xrange(len(mydbapi.porttrees)-1): - t = mydbapi.porttrees[i+1] - if myebuild.startswith(t): - # Display the non-cannonical path, in case it's different, to - # prevent confusion. - overlays = mysettings["PORTDIR_OVERLAY"].split() - try: - writemsg("!!! This ebuild is from an overlay: '%s'\n" % \ - overlays[i], noiselevel=-1) - except IndexError: - pass - break return retval finally: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 9cc264cc4..ad8e46432 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -20,7 +20,7 @@ from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, grabfile, grabdict, normalize_path, new_protect_filename from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp -from portage import listdir, dep_expand, config, flatten, key_expand, \ +from portage import listdir, dep_expand, flatten, key_expand, \ doebuild_environment, doebuild, env_update, \ abssymlink, movefile, _movefile, bsd_chflags @@ -650,6 +650,7 @@ class vartree(object): self.root = clone.root[:] self.dbapi = copy.deepcopy(clone.dbapi) self.populated = 1 + from portage import config self.settings = config(clone=clone.settings) else: self.root = root[:] @@ -1058,6 +1059,7 @@ class dblink(object): uid=portage_uid, gid=portage_gid, mode=070, mask=0) builddir_lock = None catdir_lock = None + retval = -1 try: if myebuildpath: catdir_lock = lockdir(catdir) @@ -1090,23 +1092,27 @@ class dblink(object): self.settings, use_cache=0, tree="vartree", mydbapi=self.vartree.dbapi, vartree=self.vartree) - # process logs created during pre/postrm - elog_process(self.mycpv, self.settings) - # XXX: Decide how to handle failures here. if retval != os.EX_OK: writemsg("!!! FAILED postrm: %s\n" % retval, noiselevel=-1) return retval - doebuild(myebuildpath, "cleanrm", self.myroot, self.settings, - tree="vartree", mydbapi=self.vartree.dbapi, - vartree=self.vartree) - + # regenerate reverse NEEDED map self.vartree.dbapi.libmap.update() finally: if builddir_lock: - unlockdir(builddir_lock) + try: + if myebuildpath: + # process logs created during pre/postrm + elog_process(self.mycpv, self.settings) + if retval == os.EX_OK: + doebuild(myebuildpath, "cleanrm", self.myroot, + self.settings, tree="vartree", + mydbapi=self.vartree.dbapi, + vartree=self.vartree) + finally: + unlockdir(builddir_lock) try: if myebuildpath and not catdir_lock: # Lock catdir for removal if empty. @@ -1609,9 +1615,12 @@ class dblink(object): slot_matches.append(self.mycpv) others_in_slot = [] + from portage import config for cur_cpv in slot_matches: + # Clone the config in case one of these has to be unmerged since + # we need it to have private ${T} etc... for things like elog. others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], - self.vartree.root, self.settings, + self.vartree.root, config(clone=self.settings), vartree=self.vartree)) retval = self._security_check(others_in_slot) if retval: @@ -1806,12 +1815,6 @@ class dblink(object): contents=contents, env=self.settings.environ()) writemsg_stdout(">>> %s %s\n" % (self.mycpv,"merged.")) - - # Process ebuild logfiles - elog_process(self.mycpv, self.settings) - if "noclean" not in self.settings.features: - doebuild(myebuild, "clean", destroot, self.settings, - tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) return os.EX_OK def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime): @@ -2083,12 +2086,21 @@ class dblink(object): def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, mydbapi=None, prev_mtimes=None): + retval = -1 + self.lockdb() try: - self.lockdb() - return self.treewalk(mergeroot, myroot, inforoot, myebuild, + retval = self.treewalk(mergeroot, myroot, inforoot, myebuild, cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) + # Process ebuild logfiles + elog_process(self.mycpv, self.settings) + if retval == os.EX_OK and "noclean" not in self.settings.features: + if myebuild is None: + myebuild = os.path.join(inforoot, self.pkg + ".ebuild") + doebuild(myebuild, "clean", myroot, self.settings, + tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) finally: self.unlockdb() + return retval def getstring(self,name): "returns contents of a file with whitespace converted to spaces" diff --git a/pym/portage/elog/mod_echo.py b/pym/portage/elog/mod_echo.py index a5749a93a..9ae6fcf6c 100644 --- a/pym/portage/elog/mod_echo.py +++ b/pym/portage/elog/mod_echo.py @@ -6,25 +6,18 @@ from portage.output import EOutput from portage.const import EBUILD_PHASES -_items = {} +_items = [] def process(mysettings, key, logentries, fulltext): global _items - config_root = mysettings["PORTAGE_CONFIGROOT"] - mysettings, items = _items.setdefault(config_root, (mysettings, {})) - items[key] = logentries + _items.append((mysettings, key, logentries)) def finalize(): global _items - for mysettings, items in _items.itervalues(): - _finalize(mysettings, items) - _items.clear() - -def _finalize(mysettings, items): printer = EOutput() - root_msg = "" - if mysettings["ROOT"] != "/": - root_msg = " merged to %s" % mysettings["ROOT"] - for key, logentries in items.iteritems(): + for mysettings, key, logentries in _items: + root_msg = "" + if mysettings["ROOT"] != "/": + root_msg = " merged to %s" % mysettings["ROOT"] print printer.einfo("Messages for package %s%s:" % (key, root_msg)) print @@ -39,4 +32,5 @@ def _finalize(mysettings, items): "QA": printer.ewarn} for line in msgcontent: fmap[msgtype](line.strip("\n")) + _items = [] return diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 5add55c81..61af21d94 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -487,12 +487,12 @@ class Manifest(object): except OSError: pass if f in distfilehashes and \ + not required_hash_types.difference(distfilehashes[f]) and \ ((assumeDistHashesSometimes and mystat is None) or \ (assumeDistHashesAlways and mystat is None) or \ (assumeDistHashesAlways and mystat is not None and \ len(distfilehashes[f]) == len(self.hashes) and \ - distfilehashes[f]["size"] == mystat.st_size)) and \ - not required_hash_types.difference(distfilehashes[f]): + distfilehashes[f]["size"] == mystat.st_size)): self.fhashdict["DIST"][f] = distfilehashes[f] else: try: -- 2.26.2