From: Fabian Groffen Date: Tue, 2 Jan 2007 19:59:18 +0000 (-0000) Subject: Merged from trunk 5427:5444 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=344c07356bac9fb41c6f7d709c1e469d173a027d;p=portage.git Merged from trunk 5427:5444 For bug #151326, prevent invalid comments from being returned by portage.getmaskingreason(). Correct the package.mask lookup order in getmaskingreason(). For bug #141361, use an alarm signal to implement a 1 minute timeout for processing inside an elog module. Use a try/finally block to ensure that the alarm will always be cancelled appropriately. Use return values instead of direct sys.exit(123) calls when a merge fails. Send errors in elog_process() to stderr via writemsg. For bug #159518, increase the "database is locked" timeout. Use slot atom matching to simplify detection of downgrades relative to installed packages. For bug #159224, use separate xmatch caches for each value of ACCEPT_KEYWORDS. For bug #159638, fix the UnboundLocalError in vartree.getprovide(). Fix the writemsg noiselevel from the previous commit. merge all of the QA related checks together (SpankMan) (needed manual diff) Don't bother to list the contents of directories before they are unmerged since it triggers unnecessary stat calls. Thanks to James Cloos for reporting. Optimize dep_zapdeps to work without a vartree when called by repoman. svn path=/main/branches/prefix/; revision=5445 --- diff --git a/bin/emerge b/bin/emerge index 511da0a98..a4f0318c5 100755 --- a/bin/emerge +++ b/bin/emerge @@ -3016,8 +3016,8 @@ class MergeTask(object): mydbapi=bindb, vartree=self.trees[myroot]["vartree"], prev_mtimes=ldpath_mtimes) - if retval is None: - sys.exit(1) + if retval != os.EX_OK: + return retval #need to check for errors if "--buildpkgonly" not in self.myopts: self.trees[x[1]]["vartree"].inject(x[2]) diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 6044ad433..c84fd0bda 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -14,6 +14,7 @@ # XXX hack: clear the args so ebuild.sh doesn't see them MISC_FUNCTIONS_ARGS="$@" shift $# + source @PORTAGE_BASE@/bin/ebuild.sh install_qa_check() { @@ -164,6 +165,27 @@ install_qa_check() { die "Aborting due to QA concerns: ${die_msg}" fi + # Run some sanity checks on shared libraries + for d in "${D}"lib* "${D}"usr/lib* ; do + f=$(scanelf -ByF '%S %p' "${d}"/lib*.so* | gawk '$2 == "" { print }') + if [[ -n ${f} ]] ; then + vecho -ne '\a\n' + vecho "QA Notice: the following shared libraries lack a SONAME" + vecho "${f}" + vecho -ne '\a\n' + sleep 1 + fi + + f=$(scanelf -ByF '%n %p' "${d}"/lib*.so* | gawk '$2 == "" { print }') + if [[ -n ${f} ]] ; then + vecho -ne '\a\n' + vecho "QA Notice: the following shared libraries lack NEEDED entries" + vecho "${f}" + vecho -ne '\a\n' + sleep 1 + fi + done + PORTAGE_QUIET=${tmp_quiet} fi @@ -217,6 +239,81 @@ install_qa_check() { fi rm -f "${find_log}" + # this should help to ensure that all (most?) shared libraries are executable + # and that all libtool scripts / static libraries are not executable + for i in "${ED}"opt/*/lib{,32,64} \ + "${ED}"lib{,32,64} \ + "${ED}"usr/lib{,32,64} \ + "${ED}"usr/X11R6/lib{,32,64} ; do + [[ ! -d ${i} ]] && continue + + for j in "${i}"/*.so.* "${i}"/*.so ; do + [[ ! -e ${j} ]] && continue + if [[ -L ${j} ]] ; then + linkdest=$(readlink "${j}") + if [[ ${linkdest} == /* ]] ; then + vecho -e "\a\n" + vecho "QA Notice: Found an absolute symlink in a library directory:" + vecho " ${j#${D}} -> ${linkdest}" + vecho " It should be a relative symlink if in the same directory" + vecho " or a linker script if it crosses the /usr boundary." + fi + continue + fi + [[ -x ${j} ]] && continue + vecho "making executable: ${j#${D}}" + chmod +x "${j}" + done + + for j in "${i}"/*.a "${i}"/*.la ; do + [[ ! -e ${j} ]] && continue + [[ -L ${j} ]] && continue + [[ ! -x ${j} ]] && continue + vecho "removing executable bit: ${j#${D}}" + chmod -x "${j}" + done + done + + # When installing static libraries into /usr/lib and shared libraries into + # /lib, we have to make sure we have a linker script in /usr/lib along side + # the static library, or gcc will utilize the static lib when linking :(. + # http://bugs.gentoo.org/4411 + abort="no" + for a in "${ED}"usr/lib*/*.a ; do + s=${a%.a}.so + if [[ ! -e ${s} ]] ; then + s=${s%usr/*}${s##*/usr/} + if [[ -e ${s} ]] ; then + vecho -e "\a\n" + vecho "QA Notice: missing gen_usr_ldscript for ${s##*/}\a" + abort="yes" + fi + fi + done + [[ ${abort} == "yes" ]] && die "add those ldscripts" + + # Make sure people don't store libtool files or static libs in /lib + f=$(ls "${ED}"lib*/*.{a,la} 2>/dev/null) + if [[ -n ${f} ]] ; then + vecho -e "\a\n" + vecho "QA Notice: excessive files found in the / partition\a" + vecho "${f}" + vecho -e "\a\n" + die "static archives (*.a) and libtool library files (*.la) do not belong in /" + fi + + # Verify that the libtool files don't contain bogus $D entries. + abort="no" + for a in "${ED}"usr/lib*/*.la ; do + s=${a##*/} + if grep -qs "${ED}" "${a}" ; then + vecho -e "\a\n" + vecho "QA Notice: ${s} appears to contain PORTAGE_TMPDIR paths" + abort="yes" + fi + done + [[ ${abort} == "yes" ]] && die "soiled libtool library files found" + # Portage regenerates this on the installed system. if [ -f "${ED}/usr/share/info/dir.gz" ]; then rm -f "${ED}/usr/share/info/dir.gz" @@ -243,7 +340,6 @@ install_qa_check() { done [[ ${abort} == yes ]] && die "multilib-strict check failed!" fi - } diff --git a/bin/prepall b/bin/prepall index 9cb84a5cc..1f756f0d3 100755 --- a/bin/prepall +++ b/bin/prepall @@ -15,96 +15,6 @@ prepallman prepallinfo prepallstrip -# this should help to ensure that all (most?) shared libraries are executable -# and that all libtool scripts / static libraries are not executable -for i in "${ED}"/opt/*/lib{,32,64} \ - "${ED}"/lib{,32,64} \ - "${ED}"/usr/lib{,32,64} \ - "${ED}"/usr/X11R6/lib{,32,64} ; do - [[ ! -d ${i} ]] && continue - - for j in "${i}"/*.so.* "${i}"/*.so ; do - [[ ! -e ${j} ]] && continue - if [[ -L ${j} ]] ; then - linkdest=$(readlink "${j}") - if [[ ${linkdest} == /* ]] ; then - vecho -e "\a" - vecho "QA Notice: Found an absolute symlink in a library directory:" - vecho " ${j#${D}} -> ${linkdest}" - vecho " It should be a relative symlink if in the same directory" - vecho " or a linker script if it crosses the /usr boundary." - fi - continue - fi - [[ -x ${j} ]] && continue - vecho "making executable: ${j#${D}}" - chmod +x "${j}" - done - - for j in "${i}"/*.a "${i}"/*.la ; do - [[ ! -e ${j} ]] && continue - [[ -L ${j} ]] && continue - [[ ! -x ${j} ]] && continue - vecho "removing executable bit: ${j#${D}}" - chmod -x "${j}" - done -done - -# When installing static libraries into /usr/lib and shared libraries into -# /lib, we have to make sure we have a linker script in /usr/lib along side -# the static library, or gcc will utilize the static lib when linking :(. -# http://bugs.gentoo.org/4411 -for a in "${ED}"/usr/lib*/*.a ; do - s=${a%.a}.so - if [[ ! -e ${s} ]] ; then - s=${s%usr/*}${s##*/usr/} - if [[ -e ${s} ]] ; then - vecho -e "\aQA Notice: missing gen_usr_ldscript for ${s##*/}\a" - sleep 1 - fi - fi -done - -# Make sure people don't store libtool files or static libs in /lib -f=$(ls "${ED}"/lib*/*.{a,la} 2>/dev/null) -if [[ -n ${f} ]] ; then - vecho -e "\n\aQA Notice: excessive files found in the / partition\a" - vecho "${f}" - sleep 1 -fi - -# Verify that the libtool files don't contain bogus $D entries. -for a in "${ED}"/usr/lib*/*.la ; do - s=${a##*/} - if grep -qs "${ED}" "${a}" ; then - vecho -e "\n\aQA Notice: ${s} appears to contain PORTAGE_TMPDIR paths\a" - sleep 1 - fi -done - -if type -p scanelf > /dev/null ; then - -# Run some sanity checks on shared libraries -for d in "${ED}"/lib* "${ED}"/usr/lib* ; do - f=$(scanelf -ByF '%S %p' "${d}"/lib*.so* | gawk '$2 == "" { print }') - if [[ -n ${f} ]] ; then - vecho -e "\n\aQA Notice: the following shared libraries lack a SONAME\a" - vecho "${f}" - vecho - sleep 1 - fi - - f=$(scanelf -ByF '%n %p' "${d}"/lib*.so* | gawk '$2 == "" { print }') - if [[ -n ${f} ]] ; then - vecho -e "\n\aQA Notice: the following shared libraries lack NEEDED entries\a" - vecho "${f}" - vecho - sleep 1 - fi -done - -fi - if type -p chflags &>/dev/null && type -p mtree &>/dev/null; then # Restore all the file flags that were saved at the beginning of prepall. mtree -U -e -p "${ED}" -k flags < "${T}/bsdflags.mtree" &> /dev/null diff --git a/bin/repoman b/bin/repoman index 00d1254bc..9c95e06cc 100755 --- a/bin/repoman +++ b/bin/repoman @@ -543,7 +543,7 @@ trees["/"]["porttree"].settings = repoman_settings portdb = trees["/"]["porttree"].dbapi portdb.mysettings = repoman_settings # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman. -trees["/"]["vartree"].dbapi = portage.fakedbapi(settings=repoman_settings) +del trees["/"]["vartree"] if not myreporoot: myreporoot = os.path.basename(portdir_overlay) @@ -796,6 +796,7 @@ def x11_deprecation_check(depstr): arch_caches={} +arch_xmatch_caches = {} for x in scanlist: #ebuilds and digests added to cvs respectively. if verbose: @@ -1255,16 +1256,13 @@ for x in scanlist: profdir = portdir+"/profiles/"+prof[0] if prof[0] in arch_caches: - dep_settings, xcache = arch_caches[prof[0]] + dep_settings = arch_caches[prof[0]] else: dep_settings = portage.config( config_profile_path=profdir, config_incrementals=portage_const.INCREMENTALS, local_config=False) - portdb.melt() - portdb.freeze() - xcache = portdb.xcache - arch_caches[prof[0]] = (dep_settings, xcache) + arch_caches[prof[0]] = dep_settings while True: try: # Protect ACCEPT_KEYWORDS from config.regenerate() @@ -1273,6 +1271,14 @@ for x in scanlist: except ValueError: break + xmatch_cache_key = (prof[0], tuple(groups)) + xcache = arch_xmatch_caches.get(xmatch_cache_key) + if xcache is None: + portdb.melt() + portdb.freeze() + xcache = portdb.xcache + arch_xmatch_caches[xmatch_cache_key] = xcache + trees["/"]["porttree"].settings = dep_settings portdb.mysettings = dep_settings portdb.xcache = xcache diff --git a/pym/cache/sqlite.py b/pym/cache/sqlite.py index 6b1a3d323..5c1bfa266 100644 --- a/pym/cache/sqlite.py +++ b/pym/cache/sqlite.py @@ -38,6 +38,9 @@ class database(fs_template.FsBased): config.setdefault("autocommit", self.autocommits) config.setdefault("cache_bytes", self.cache_bytes) config.setdefault("synchronous", self.synchronous) + # Timeout for throwing a "database is locked" exception (pysqlite + # default is 5.0 seconds). + config.setdefault("timeout", 15) self._db_init_connection(config) self._db_init_structures() @@ -49,9 +52,12 @@ class database(fs_template.FsBased): self._dbpath = self.location + ".sqlite" #if os.path.exists(self._dbpath): # os.unlink(self._dbpath) + connection_kwargs = {} + connection_kwargs["timeout"] = config["timeout"] try: self._ensure_dirs() - self._db_connection = self._db_module.connect(database=self._dbpath) + self._db_connection = self._db_module.connect( + database=self._dbpath, **connection_kwargs) self._db_cursor = self._db_connection.cursor() self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8")) if not apply_secpass_permissions(self._dbpath, gid=portage_gid, mode=070, mask=02): diff --git a/pym/portage.py b/pym/portage.py index 5d0054b0c..b08d03a80 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -475,7 +475,8 @@ def elog_process(cpv, mysettings): and not msgtype.lower() in mysettings["PORTAGE_ELOG_CLASSES"].split(): continue if msgfunction not in portage_const.EBUILD_PHASES: - print "!!! can't process invalid log file: %s" % f + writemsg("!!! can't process invalid log file: %s\n" % f, + noiselevel=-1) continue if not msgfunction in mylogentries: mylogentries[msgfunction] = [] @@ -505,15 +506,27 @@ def elog_process(cpv, mysettings): # TODO: implement a common portage module loader logmodule = __import__("elog_modules.mod_"+s) m = getattr(logmodule, "mod_"+s) - m.process(mysettings, cpv, mylogentries, fulllog) + def timeout_handler(signum, frame): + raise portage_exception.PortageException( + "Timeout in elog_process for system '%s'" % s) + import signal + signal.signal(signal.SIGALRM, timeout_handler) + # Timeout after one minute (in case something like the mail + # module gets hung). + signal.alarm(60) + try: + m.process(mysettings, cpv, mylogentries, fulllog) + finally: + signal.alarm(0) if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers: _elog_atexit_handlers.append(m.finalize) atexit_register(m.finalize, mysettings) except (ImportError, AttributeError), e: - print "!!! Error while importing logging modules while loading \"mod_%s\":" % s - print e + writemsg("!!! Error while importing logging modules " + \ + "while loading \"mod_%s\":\n" % str(s)) + writemsg("%s\n" % str(e), noiselevel=-1) except portage_exception.PortageException, e: - print e + writemsg("%s\n" % str(e), noiselevel=-1) # clean logfiles to avoid repetitions for f in mylogfiles: @@ -3956,7 +3969,9 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): other = [] # Alias the trees we'll be checking availability against - vardb = trees[myroot]["vartree"].dbapi + vardb = None + if "vartree" in trees[myroot]: + vardb = trees[myroot]["vartree"].dbapi if use_binaries: mydbapi = trees[myroot]["bintree"].dbapi else: @@ -3971,6 +3986,17 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): else: atoms = [dep] + all_available = True + for atom in atoms: + if not mydbapi.match(atom): + all_available = False + break + + if not vardb: + # called by repoman + preferred.append((atoms, None, all_available)) + continue + """ The package names rather than the exact atoms are used for an initial rough match against installed packages. More specific preference selection is handled later via slot and version comparison.""" @@ -3981,16 +4007,10 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): all_installed = False break - all_available = True - for atom in atoms: - if not mydbapi.match(atom): - all_available = False - break - # Check if the set of atoms will result in a downgrade of # an installed package. If they will then don't prefer them # over other atoms. - is_downgrade = False + has_downgrade = False versions = {} if all_installed or all_available: for atom in atoms: @@ -3998,23 +4018,21 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg = best(mydbapi.match(atom)) if not avail_pkg: continue - avail_slot = mydbapi.aux_get(avail_pkg, ["SLOT"])[0] - versions["%s:%s" % (mykey, avail_slot)] = avail_pkg - avail_split = catpkgsplit(avail_pkg)[1:] - inst_pkgs = vardb.match(mykey) - if not inst_pkgs: + avail_slot = "%s:%s" % (mykey, + mydbapi.aux_get(avail_pkg, ["SLOT"])[0]) + versions[avail_slot] = avail_pkg + inst_pkg = vardb.match(avail_slot) + if not inst_pkg: continue - for pkg in inst_pkgs: - if avail_slot != vardb.aux_get(pkg, ["SLOT"])[0]: - continue - if pkgcmp(avail_split, catpkgsplit(pkg)[1:]) < 0: - is_downgrade = True - break - if is_downgrade: + # emerge guarantees 1 package per slot here (highest counter) + inst_pkg = inst_pkg[0] + if avail_pkg != inst_pkg and \ + avail_pkg != best([avail_pkg, inst_pkg]): + has_downgrade = True break this_choice = (atoms, versions, all_available) - if not is_downgrade: + if not has_downgrade: if all_installed: preferred.append(this_choice) continue @@ -4315,14 +4333,14 @@ def getmaskingreason(mycpv, settings=None, portdb=None): mycp=mysplit[0]+"/"+mysplit[1] # XXX- This is a temporary duplicate of code from the config constructor. - locations = settings.profiles[:] - locations.append(os.path.join(settings["PORTDIR"], "profiles")) - locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], - USER_CONFIG_PATH.lstrip(os.path.sep))) + locations = [os.path.join(settings["PORTDIR"], "profiles")] + locations.extend(settings.profiles) for ov in settings["PORTDIR_OVERLAY"].split(): profdir = os.path.join(normalize_path(ov), "profiles") if os.path.isdir(profdir): locations.append(profdir) + locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], + USER_CONFIG_PATH.lstrip(os.path.sep))) locations.reverse() pmasklists = [grablines(os.path.join(x, "package.mask"), recursive=1) for x in locations] pmasklines = [] @@ -4335,16 +4353,23 @@ def getmaskingreason(mycpv, settings=None, portdb=None): if mycpv in portdb.xmatch("match-all", x): comment = "" l = "\n" - i = 0 - while i < len(pmasklines): + comment_valid = -1 + for i in xrange(len(pmasklines)): l = pmasklines[i].strip() if l == "": comment = "" + comment_valid = -1 elif l[0] == "#": comment += (l+"\n") + comment_valid = i + 1 elif l == x: + if comment_valid != i: + comment = "" return comment - i = i + 1 + elif comment_valid != -1: + # Apparently this comment applies to muliple masks, so + # it remains valid until a blank line is encountered. + comment_valid += 1 return None def getmaskingstatus(mycpv, settings=None, portdb=None): @@ -5253,6 +5278,7 @@ class vartree(object): def get_provide(self,mycpv): myprovides=[] + mylines = None try: mylines, myuse = self.dbapi.aux_get(mycpv, ["PROVIDE","USE"]) if mylines: @@ -5267,11 +5293,13 @@ class vartree(object): except SystemExit, e: raise except Exception, e: - print - print "Check " + self.root+VDB_PATH+"/"+mycpv+"/PROVIDE and USE." - print "Possibly Invalid: " + str(mylines) - print "Exception: "+str(e) - print + mydir = os.path.join(self.root, VDB_PATH, mycpv) + writemsg("\nParse Error reading PROVIDE and USE in '%s'\n" % mydir, + noiselevel=-1) + if mylines: + writemsg("Possibly Invalid: '%s'\n" % str(mylines), + noiselevel=-1) + writemsg("Exception: %s\n\n" % str(e), noiselevel=-1) return [] def get_all_provides(self): @@ -6839,22 +6867,13 @@ class dblink: mydirs.sort() mydirs.reverse() - last_non_empty = "" for obj in mydirs: - if not last_non_empty.startswith(obj) and not listdir(obj): - try: - os.rmdir(obj) - writemsg_stdout("<<< %s %s\n" % ("dir",obj)) - last_non_empty = "" - continue - except (OSError,IOError),e: - #immutable? - pass - - writemsg_stdout("--- !empty dir %s\n" % obj) - last_non_empty = obj - continue + try: + os.rmdir(obj) + writemsg_stdout("<<< %s %s\n" % ("dir",obj)) + except (OSError, IOError): + writemsg_stdout("--- !empty dir %s\n" % obj) #remove self from vartree database so that our own virtual gets zapped if we're the last node self.vartree.zap(self.mycpv) @@ -7115,9 +7134,9 @@ class dblink: vartree=self.vartree) # XXX: Decide how to handle failures here. - if a != 0: + if a != os.EX_OK: writemsg("!!! FAILED preinst: "+str(a)+"\n", noiselevel=-1) - sys.exit(123) + return a # copy "info" files (like SLOT, CFLAGS, etc.) into the database for x in listdir(inforoot): @@ -7215,9 +7234,9 @@ class dblink: tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) # XXX: Decide how to handle failures here. - if a != 0: + if a != os.EX_OK: writemsg("!!! FAILED postinst: "+str(a)+"\n", noiselevel=-1) - sys.exit(123) + return a downgrade = False for v in otherversions: @@ -7239,7 +7258,7 @@ class dblink: if "noclean" not in self.settings.features: doebuild(myebuild, "clean", destroot, self.settings, tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) - return 0 + return os.EX_OK def mergeme(self,srcroot,destroot,outfile,secondhand,stufftomerge,cfgfiledict,thismtime): """ @@ -7641,7 +7660,7 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes vartree = db[myroot]["vartree"] if mytbz2[-5:]!=".tbz2": print "!!! Not a .tbz2 file" - return None + return 1 tbz2_lock = None builddir_lock = None @@ -7657,7 +7676,7 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes if not mycat: writemsg("!!! CATEGORY info missing from info chunk, aborting...\n", noiselevel=-1) - return None + return 1 mycat = mycat.strip() # These are the same directories that would be used at build time. @@ -7702,7 +7721,7 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes tree="bintree", mydbapi=mydbapi, vartree=vartree) if retval != os.EX_OK: writemsg("!!! Setup failed: %s\n" % retval, noiselevel=-1) - return None + return retval writemsg_stdout(">>> Extracting %s\n" % mypkg) retval = portage_exec.spawn_bash( @@ -7710,23 +7729,15 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes env=mysettings.environ()) if retval != os.EX_OK: writemsg("!!! Error Extracting '%s'\n" % mytbz2, noiselevel=-1) - return None + return retval #portage_locks.unlockfile(tbz2_lock) #tbz2_lock = None mylink = dblink(mycat, mypkg, myroot, mysettings, vartree=vartree, treetype="bintree") - mylink.merge(pkgloc, infloc, myroot, myebuild, cleanup=0, + retval = mylink.merge(pkgloc, infloc, myroot, myebuild, cleanup=0, mydbapi=mydbapi, prev_mtimes=prev_mtimes) - - try: - f = open(os.path.join(infloc, "RDEPEND", "r")) - try: - return " ".join(f.read().split()) - finally: - f.close() - except (IOError, OSError): - return "" + return retval finally: if tbz2_lock: portage_locks.unlockfile(tbz2_lock)