From 4ed74042167400a32f0ccd95c467cb5e1140a3b8 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 4 Jan 2007 20:43:41 +0000 Subject: [PATCH] Merged from trunk 5447:5459 Try to create DISTDIR before disabling fetch due to it's nonexistence. Thanks to astinus for reporting. add support for scanning of build logs for common issues #111436 (SpankMan) Allow - in elog module names (genone) undo (genone) When using the spawn's returnpid parameter, manually remove the returned pids from portage_exec.spawned_pids. Use os._exit instead of sys.exit in a forked child process. Make sure stderr is flushed since os._exit doesn't flush stdio buffers. Make sure stderr is flushed since os._exit doesn't flush stdio buffers. Implement distlocks for --getbinpkg so that it works properly when parallel-fetch is enabled. Thanks to Brent Baude for reporting. Export EBUILD_PHASE="" when running misc-function.sh in order to prevent bogus phase values coming from ${T}/environment. Thanks to solar for reporting. When fetching metadata from a binhost, reconnect if the http server closes the connection before sending a valid response. Thanks to ranger for reporting. Make sure that myid is defined for the previous commit. svn path=/main/branches/prefix/; revision=5460 --- bin/check-implicit-pointer-usage.py | 53 +++++++++++++++++++++++++++++ bin/emerge | 28 ++++++++++----- bin/misc-functions.sh | 50 +++++++++++++++++++++++---- pym/getbinpkg.py | 22 +++++++++--- pym/portage.py | 23 ++++++++----- pym/portage_exec.py | 1 + 6 files changed, 150 insertions(+), 27 deletions(-) create mode 100755 bin/check-implicit-pointer-usage.py diff --git a/bin/check-implicit-pointer-usage.py b/bin/check-implicit-pointer-usage.py new file mode 100755 index 000000000..4afa8f24b --- /dev/null +++ b/bin/check-implicit-pointer-usage.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Ripped from HP and updated from Debian + +# +# Copyright (c) 2004 Hewlett-Packard Development Company, L.P. +# David Mosberger +# +# Scan standard input for GCC warning messages that are likely to +# source of real 64-bit problems. In particular, see whether there +# are any implicitly declared functions whose return values are later +# interpreted as pointers. Those are almost guaranteed to cause +# crashes. +# +import re +import sys + +implicit_pattern = re.compile("([^:]*):(\d+): warning: implicit declaration " + + "of function [`']([^']*)'") +pointer_pattern = re.compile( + "([^:]*):(\d+): warning: " + + "(" + + "(assignment" + + "|initialization" + + "|return" + + "|passing arg \d+ of `[^']*'" + + "|passing arg \d+ of pointer to function" + + ") makes pointer from integer without a cast" + + "|" + + "cast to pointer from integer of different size)") +last_implicit_filename = "" +last_implicit_linenum = -1 +last_implicit_func = "" + +while True: + line = sys.stdin.readline() + if line == '': + break + m = implicit_pattern.match(line) + if m: + last_implicit_filename = m.group(1) + last_implicit_linenum = int(m.group(2)) + last_implicit_func = m.group(3) + else: + m = pointer_pattern.match(line) + if m: + pointer_filename = m.group(1) + pointer_linenum = int(m.group(2)) + if (last_implicit_filename == pointer_filename + and last_implicit_linenum == pointer_linenum): + print "Function `%s' implicitly converted to pointer at " \ + "%s:%d" % (last_implicit_func, last_implicit_filename, + last_implicit_linenum) diff --git a/bin/emerge b/bin/emerge index a4f0318c5..a314fb441 100755 --- a/bin/emerge +++ b/bin/emerge @@ -2995,14 +2995,26 @@ class MergeTask(object): elif x[0]=="binary": #merge the tbz2 mytbz2 = self.trees[myroot]["bintree"].getname(pkg_key) - if self.trees[myroot]["bintree"].isremote(pkg_key): - short_msg = "emerge: ("+str(mergecount)+" of "+str(len(mymergelist))+") "+x[pkgindex]+" Fetch" - emergelog(xterm_titles, " --- ("+str(mergecount)+\ - " of "+str(len(mymergelist))+\ - ") Fetching Binary ("+x[pkgindex]+\ - "::"+mytbz2+")", short_msg=short_msg) - if not self.trees[myroot]["bintree"].gettbz2(pkg_key): - sys.exit(1) + if "--getbinpkg" in self.myopts: + tbz2_lock = None + try: + if "distlocks" in pkgsettings.features and \ + os.access(pkgsettings["PKGDIR"], os.W_OK): + portage_util.ensure_dirs(os.path.dirname(mytbz2)) + tbz2_lock = portage_locks.lockfile(mytbz2, + wantnewlockfile=1) + if self.trees[myroot]["bintree"].isremote(pkg_key): + msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ + (mergecount, len(mymergelist), pkg_key, mytbz2) + short_msg = "emerge: (%s of %s) %s Fetch" % \ + (mergecount, len(mymergelist), pkg_key) + emergelog(xterm_titles, msg, short_msg=short_msg) + if not self.trees[myroot]["bintree"].gettbz2( + pkg_key): + return 1 + finally: + if tbz2_lock: + portage_locks.unlockfile(tbz2_lock) if "--fetchonly" in self.myopts or \ "--fetch-all-uri" in self.myopts: diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 99cea0826..1975e72e0 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -241,7 +241,7 @@ install_qa_check() { if [[ -L ${j} ]] ; then linkdest=$(readlink "${j}") if [[ ${linkdest} == /* ]] ; then - vecho -e "\a\n" + vecho -ne '\a\n' vecho "QA Notice: Found an absolute symlink in a library directory:" vecho " ${j#${D}} -> ${linkdest}" vecho " It should be a relative symlink if in the same directory" @@ -273,7 +273,7 @@ install_qa_check() { if [[ ! -e ${s} ]] ; then s=${s%usr/*}${s##*/usr/} if [[ -e ${s} ]] ; then - vecho -e "\a\n" + vecho -ne '\a\n' vecho "QA Notice: missing gen_usr_ldscript for ${s##*/}\a" abort="yes" fi @@ -284,10 +284,10 @@ install_qa_check() { # Make sure people don't store libtool files or static libs in /lib f=$(ls "${ED}"lib*/*.{a,la} 2>/dev/null) if [[ -n ${f} ]] ; then - vecho -e "\a\n" + vecho -ne '\a\n' vecho "QA Notice: excessive files found in the / partition\a" vecho "${f}" - vecho -e "\a\n" + vecho -ne '\a\n' die "static archives (*.a) and libtool library files (*.la) do not belong in /" fi @@ -296,16 +296,52 @@ install_qa_check() { for a in "${ED}"usr/lib*/*.la ; do s=${a##*/} if grep -qs "${ED}" "${a}" ; then - vecho -e "\a\n" + vecho -ne '\a\n' vecho "QA Notice: ${s} appears to contain PORTAGE_TMPDIR paths" abort="yes" fi done [[ ${abort} == "yes" ]] && die "soiled libtool library files found" + # Evaluate misc gcc warnings + if [[ -n ${PORTAGE_LOG_FILE} && -r ${PORTAGE_LOG_FILE} ]] ; then + local m msgs=( + ": warning: dereferencing type-punned pointer will break strict-aliasing rules$" + ": warning: implicit declaration of function " + ": warning: incompatible implicit declaration of built-in function " + ": warning: is used uninitialized in this function$" # we'll ignore "may" and "might" + ": warning: comparisons like X<=Y<=Z do not have their mathematical meaning$" + ": warning: null argument where non-null required " + ) + abort="no" + i=0 + while [[ -n ${msgs[${i}]} ]] ; do + m=${msgs[$((i++))]} + f=$(grep "${m}" "${PORTAGE_LOG_FILE}") + if [[ -n ${f} ]] ; then + vecho -ne '\a\n' + vecho "QA Notice: Package has poor programming practices which may compile" + vecho " fine but exhibit random runtime failures." + vecho "${f}" + vecho -ne '\a\n' + abort="yes" + fi + done + f=$(cat "${PORTAGE_LOG_FILE}" | check-implicit-pointer-usage.py) + if [[ -n ${f} ]] ; then + vecho -ne '\a\n' + vecho "QA Notice: Package has poor programming practices which may compile" + vecho " but will almost certainly crash on 64bit architectures." + vecho "${f}" + vecho -ne '\a\n' + abort="yes" + fi + [[ ${abort} == "yes" ]] && hasq stricter ${FEATURES} && die "poor code kills airplanes" + fi + # Portage regenerates this on the installed system. - if [ -f "${ED}/usr/share/info/dir.gz" ]; then - rm -f "${ED}/usr/share/info/dir.gz" + if [[ -f ${ED}/usr/share/info/dir.gz ]] ; then + rm -f "${ED}"/usr/share/info/dir.gz fi if hasq multilib-strict ${FEATURES} && \ diff --git a/pym/getbinpkg.py b/pym/getbinpkg.py index 8d4a26fc6..074f7fa28 100644 --- a/pym/getbinpkg.py +++ b/pym/getbinpkg.py @@ -375,7 +375,8 @@ def file_get(baseurl,dest,conn=None,fcmd=None): except OSError: pass sys.stderr.write("!!! Failed to spawn fetcher.\n") - sys.exit(1) + sys.stderr.flush() + os._exit(1) retval=os.waitpid(mypid,0)[1] if (retval & 0xff) == 0: retval = retval >> 8 @@ -522,9 +523,22 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= (x not in metadata[baseurl]["data"].keys())): sys.stderr.write(yellow("x")) metadata[baseurl]["modified"] = 1 - myid = file_get_metadata(baseurl+"/"+x, conn, chunk_size) - - if myid[0]: + myid = None + for retry in xrange(3): + try: + myid = file_get_metadata( + "/".join((baseurl.rstrip("/"), x.lstrip("/"))), + conn, chunk_size) + break + except httplib.BadStatusLine: + # Sometimes this error is thrown from conn.getresponse() in + # make_http_request(). The docstring for this error in + # httplib.py says "Presumably, the server closed the + # connection before sending a valid response". + conn, protocol, address, params, headers = create_conn( + baseurl) + + if myid and myid[0]: metadata[baseurl]["data"][x] = make_metadata_dict(myid) elif verbose: sys.stderr.write(red("!!! Failed to retrieve metadata on: ")+str(x)+"\n") diff --git a/pym/portage.py b/pym/portage.py index b08d03a80..17ead0138 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -501,6 +501,8 @@ def elog_process(cpv, mysettings): # pass the processing to the individual modules logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split() for s in logsystems: + # - is nicer than _ for module names, so allow people to use it. + s = s.replace("-", "_") try: # FIXME: ugly ad.hoc import code # TODO: implement a common portage module loader @@ -2294,13 +2296,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if not mysettings.get(var_name, None): can_fetch = False - if can_fetch and \ - not fetch_to_ro and \ - not os.access(mysettings["DISTDIR"], os.W_OK): - writemsg("!!! No write access to '%s'\n" % mysettings["DISTDIR"], - noiselevel=-1) - can_fetch = False - if can_fetch: dirmode = 02070 filemode = 060 @@ -2328,6 +2323,13 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", writemsg("!!! Directory Not Found: DISTDIR='%s'\n" % mysettings["DISTDIR"], noiselevel=-1) writemsg("!!! Fetching will fail!\n", noiselevel=-1) + if can_fetch and \ + not fetch_to_ro and \ + not os.access(mysettings["DISTDIR"], os.W_OK): + writemsg("!!! No write access to '%s'\n" % mysettings["DISTDIR"], + noiselevel=-1) + can_fetch = False + if can_fetch and use_locks and locks_in_subdir: distlocks_subdir = os.path.join(mysettings["DISTDIR"], locks_in_subdir) if not os.access(distlocks_subdir, os.W_OK): @@ -2783,7 +2785,7 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None): kwargs = actionmap[mydo]["args"] mysettings["EBUILD_PHASE"] = mydo phase_retval = spawn(actionmap[mydo]["cmd"] % mydo, mysettings, debug=debug, logfile=logfile, **kwargs) - del mysettings["EBUILD_PHASE"] + mysettings["EBUILD_PHASE"] = "" if not kwargs["droppriv"] and secpass >= 2: """ Privileged phases may have left files that need to be made @@ -3129,6 +3131,8 @@ def prepare_build_dirs(myroot, mysettings, cleanup): mysettings["PORT_LOGDIR"], "%s:%s:%s.log" % \ (mysettings["CATEGORY"], mysettings["PF"], logid_time)) del logid_path, logid_time + else: + mysettings["PORTAGE_LOG_FILE"] = os.path.join(mysettings["T"], "build.log") _doebuild_manifest_exempt_depend = 0 _doebuild_manifest_checked = None @@ -3306,6 +3310,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, for k, v in izip(auxdbkeys, mybytes.splitlines()): dbkey[k] = v retval = os.waitpid(mypids[0], 0)[1] + portage_exec.spawned_pids.remove(mypids[0]) # If it got a signal, return the signal that was sent, but # shift in order to distinguish it from a return value. (just # like portage_exec.spawn() would do). @@ -3375,6 +3380,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, myargs = [MISC_SH_BINARY, "preinst_bsdflags", "preinst_mask", "preinst_sfperms", "preinst_selinux_labels", "preinst_suid_scan"] + mysettings["EBUILD_PHASE"] = "" phase_retval = spawn(" ".join(myargs), mysettings, debug=debug, free=1, logfile=logfile) if phase_retval != os.EX_OK: @@ -3390,6 +3396,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Post phase logic and tasks that have been factored out of # ebuild.sh. myargs = [MISC_SH_BINARY, "postinst_bsdflags"] + mysettings["EBUILD_PHASE"] = "" phase_retval = spawn(" ".join(myargs), mysettings, debug=debug, free=1, logfile=logfile) if phase_retval != os.EX_OK: diff --git a/pym/portage_exec.py b/pym/portage_exec.py index 988865ad6..77f3da66e 100644 --- a/pym/portage_exec.py +++ b/pym/portage_exec.py @@ -200,6 +200,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # propogate out of this function and cause exiting # with anything other than os._exit() sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand))) + sys.stderr.flush() os._exit(1) # Add the pid to our local and the global pid lists. -- 2.26.2