From 40f30dd492457a525cac71e75708c182904171b0 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Tue, 28 Oct 2008 18:47:54 +0000 Subject: [PATCH] Merged from trunk -r11701:11713 | 11702 | Remove the "found_available_arg" variable from | | zmedico | depgraph._select_pkg() and use the "matched_packages" | | | variable instead. This solve a problem where the | | | "found_available_arg" flag is unreliable due to the slot | | | filtering behavior of depgraph._iter_atoms_for_pkg(). For | | | example, a higher version in a different slot than the | | | available ebuild might be masked or unavailable, and | | | depgraph._iter_atoms_for_pkg() will not identify the | | | available ebuild as a match due to this installed package | | | (which is not available for reinstall). | | 11703 | Simplify logic from the previous commit. | | zmedico | | | 11704 | Revert previous 2 commits since they broke "selective" | | zmedico | behavior. | | 11705 | Handle the case that was intended to be fixed by r11702 | | zmedico | (which got reverted). | | 11706 | Prefer thirdpartymirrors over normal mirrors in cases when | | zmedico | the file does not yet exist on the normal mirrors. Thanks to | | | Maurice van der Pot for reporting. | | 11707 | Remove redundant duplicate uri elimination code. It's | | zmedico | already handled in the fetch loop. | | 11708 | Add PORTAGE_QUIET to the environment whitelist and filter it | | zmedico | from the saved ebuild environment. | | 11709 | Bug #186842 - Suppress file merge display, as previously | | zmedico | done in --quiet mode, whenever --verbose is not enabled. | | | Also, export PORTAGE_VERBOSE to the ebuild environment and | | | use it to control tar verbosity when creating binary | | | packages. | | 11710 | Use normal return statements instead of calling sys.exit() | | zmedico | inside dblink.mergeme(). | | 11711 | Remove needlessly complex error handling code from | | zmedico | vardbapi.cpv_counter(). | | 11712 | Remove unused imports found by pyflakes. | | zmedico | | | 11713 | Validate PROPERTIES inside doebuild(). | | zmedico | | svn path=/main/branches/prefix/; revision=11734 --- bin/isolated-functions.sh | 1 + bin/misc-functions.sh | 2 +- pym/_emerge/__init__.py | 20 ++++++++ pym/portage/__init__.py | 25 +++++---- pym/portage/dbapi/vartree.py | 99 ++++++++++++------------------------ 5 files changed, 70 insertions(+), 77 deletions(-) diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 22714fc01..7b8a5c935 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -535,6 +535,7 @@ save_ebuild_env() { PORTAGE_COLORMAP PORTAGE_CONFIGROOT PORTAGE_DEBUG \ PORTAGE_DEPCACHEDIR PORTAGE_GID PORTAGE_INST_GID \ PORTAGE_INST_UID PORTAGE_LOG_FILE PORTAGE_MASTER_PID \ + PORTAGE_QUIET \ PORTAGE_REPO_NAME PORTAGE_RESTRICT PORTAGE_UPDATE_ENV \ PORTAGE_WORKDIR_MODE PORTDIR \ PORTDIR_OVERLAY ${!PORTAGE_SANDBOX_*} PREROOTPATH \ diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index cdb3f44bb..7a17af14d 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -739,7 +739,7 @@ dyn_package() { cd "${T}" install_mask "${PORTAGE_BUILDDIR}/image" "${PKG_INSTALL_MASK}" local tar_options="" - [ "${PORTAGE_QUIET}" == "1" ] || tar_options="${tar_options} -v" + [[ $PORTAGE_VERBOSE = 1 ]] && tar_options+=" -v" # Sandbox is disabled in case the user wants to use a symlink # for $PKGDIR and/or $PKGDIR/All. export SANDBOX_ON="0" diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index f79cdb096..6837cf0fa 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -5708,6 +5708,22 @@ class depgraph(object): if pkg.cp == cp] break + # If the installed version is in a different slot and it is higher than + # the highest available visible package, _iter_atoms_for_pkg() may fail + # to properly match the available package with a corresponding argument + # atom. Detect this case and correct it here. + if not selective and len(matched_packages) > 1 and \ + matched_packages[-1].installed and \ + matched_packages[-1].slot_atom != \ + matched_packages[-2].slot_atom and \ + matched_packages[-1] > matched_packages[-2]: + pkg = matched_packages[-2] + if pkg.root == self.target_root and \ + self._set_atoms.findAtomForPackage(pkg): + # Select the available package instead + # of the installed package. + matched_packages.pop() + if len(matched_packages) > 1: bestmatch = portage.best( [pkg.cpv for pkg in matched_packages]) @@ -13513,6 +13529,10 @@ def adjust_config(myopts, settings): settings["PORTAGE_QUIET"]="1" settings.backup_changes("PORTAGE_QUIET") + if "--verbose" in myopts: + settings["PORTAGE_VERBOSE"] = "1" + settings.backup_changes("PORTAGE_VERBOSE") + # Set so that configs will be merged regardless of remembered status if ("--noconfmem" in myopts): settings["NOCONFMEM"]="1" diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 4435f5532..2f6712d21 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -938,8 +938,10 @@ class config(object): "PORTAGE_GID", "PORTAGE_INST_GID", "PORTAGE_INST_UID", "PORTAGE_IUSE", "PORTAGE_LOG_FILE", "PORTAGE_MASTER_PID", - "PORTAGE_PYM_PATH", "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT", - "PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV", "PORTAGE_WORKDIR_MODE", + "PORTAGE_PYM_PATH", "PORTAGE_QUIET", + "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT", + "PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV", + "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", "PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PROFILE_PATHS", "ROOT", "ROOTPATH", "STARTDIR", "T", "TMP", "TMPDIR", "USE_EXPAND", "USE_ORDER", "WORKDIR", @@ -3514,6 +3516,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", filedict={} primaryuri_indexes={} primaryuri_dict = {} + thirdpartymirror_uris = {} for myfile, myuri in file_uri_tuples: if myfile not in filedict: filedict[myfile]=[] @@ -3530,16 +3533,15 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", for cmirr in custommirrors[mirrorname]: filedict[myfile].append( cmirr.rstrip("/") + "/" + path) - # remove the mirrors we tried from the list of official mirrors - if cmirr.strip() in thirdpartymirrors[mirrorname]: - thirdpartymirrors[mirrorname].remove(cmirr) + # now try the official mirrors if mirrorname in thirdpartymirrors: shuffle(thirdpartymirrors[mirrorname]) - for locmirr in thirdpartymirrors[mirrorname]: - filedict[myfile].append( - locmirr.rstrip("/") + "/" + path) + uris = [locmirr.rstrip("/") + "/" + path \ + for locmirr in thirdpartymirrors[mirrorname]] + filedict[myfile].extend(uris) + thirdpartymirror_uris.setdefault(myfile, []).extend(uris) if not filedict[myfile]: writemsg("No known mirror by the name: %s\n" % (mirrorname)) @@ -3565,6 +3567,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", primaryuri_dict[myfile] = primaryuris primaryuris.append(myuri) + # Prefer thirdpartymirrors over normal mirrors in cases when + # the file does not yet exist on the normal mirrors. + for myfile, uris in thirdpartymirror_uris.iteritems(): + primaryuri_dict.setdefault(myfile, []).extend(uris) + can_fetch=True if listonly: @@ -5425,7 +5432,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, set(["clean", "cleanrm", "help", "prerm", "postrm"]) mycpv = mysettings["CATEGORY"] + "/" + mysettings["PF"] dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] - misc_keys = ["LICENSE", "PROVIDE", "RESTRICT", "SRC_URI"] + misc_keys = ["LICENSE", "PROPERTIES", "PROVIDE", "RESTRICT", "SRC_URI"] other_keys = ["SLOT"] all_keys = dep_keys + misc_keys + other_keys metadata = dict(izip(all_keys, mydbapi.aux_get(mycpv, all_keys))) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index b86a31bb5..078503e7d 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -7,20 +7,19 @@ __all__ = ["PreservedLibsRegistry", "LinkageMap", ["write_contents", "tar_contents"] from portage.checksum import perform_md5 -from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, PORTAGE_BIN_PATH, \ +from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ PRIVATE_PATH, VDB_PATH, EPREFIX, EPREFIX_LSTRIP from portage.data import portage_gid, portage_uid, secpass, ostype from portage.dbapi import dbapi -from portage.dep import dep_getslot, use_reduce, paren_reduce, isvalidatom, \ - isjustname, dep_getkey, match_from_list -from portage.exception import InvalidAtom, InvalidData, InvalidPackageName, \ +from portage.dep import use_reduce, paren_reduce, isvalidatom, \ + isjustname, dep_getkey +from portage.exception import InvalidData, InvalidPackageName, \ FileNotFound, PermissionDenied, UnsupportedAPIException from portage.locks import lockdir, unlockdir from portage.output import bold, red, green from portage.update import fixdbentries from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, \ - writemsg, writemsg_stdout, writemsg_level, \ - write_atomic, atomic_ofstream, writedict, \ + writemsg, writemsg_level, write_atomic, atomic_ofstream, writedict, \ grabfile, grabdict, normalize_path, new_protect_filename, getlibpaths from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp @@ -31,7 +30,7 @@ from portage import listdir, dep_expand, flatten, key_expand, \ from portage.elog import elog_process from portage.elog.filtering import filter_mergephases, filter_unmergephases -import os, re, sys, stat, errno, commands, copy, time, subprocess +import os, re, stat, errno, copy, subprocess import logging import shlex from itertools import izip @@ -1110,53 +1109,10 @@ class vardbapi(dbapi): return long(self.aux_get(mycpv, ["COUNTER"])[0]) except (KeyError, ValueError): pass - cdir = self.getpath(mycpv) - cpath = self.getpath(mycpv, filename="COUNTER") - - # We write our new counter value to a new file that gets moved into - # place to avoid filesystem corruption on XFS (unexpected reboot.) - corrupted = 0 - if os.path.exists(cpath): - cfile = open(cpath, "r") - try: - counter = long(cfile.readline()) - except ValueError: - print "portage: COUNTER for", mycpv, "was corrupted; resetting to value of 0" - counter = long(0) - corrupted = 1 - cfile.close() - elif os.path.exists(cdir): - mys = pkgsplit(mycpv) - myl = self.match(mys[0], use_cache=0) - print mys, myl - if len(myl) == 1: - try: - # Only one package... Counter doesn't matter. - write_atomic(cpath, "1") - counter = 1 - except SystemExit, e: - raise - except Exception, e: - writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n", - noiselevel=-1) - writemsg("!!! Please run %s/fix-db.py or\n" % PORTAGE_BIN_PATH, - noiselevel=-1) - writemsg("!!! unmerge this exact version.\n", noiselevel=-1) - writemsg("!!! %s\n" % e, noiselevel=-1) - sys.exit(1) - else: - writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n", - noiselevel=-1) - writemsg("!!! Please run %s/fix-db.py or\n" % PORTAGE_BIN_PATH, - noiselevel=-1) - writemsg("!!! remerge the package.\n", noiselevel=-1) - sys.exit(1) - else: - counter = long(0) - if corrupted: - # update new global counter file - write_atomic(cpath, str(counter)) - return counter + writemsg_level(("portage: COUNTER for %s was corrupted; " + \ + "resetting to value of 0\n") % (mycpv,), + level=logging.ERROR, noiselevel=-1) + return 0 def cpv_inject(self, mycpv): "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist" @@ -2109,8 +2065,7 @@ class dblink(object): self._lock_vdb = None self.settings = mysettings - if self.settings == 1: - raise ValueError + self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1" self.myroot=myroot protect_obj = ConfigProtect(myroot, @@ -2544,6 +2499,8 @@ class dblink(object): return os.EX_OK def _display_merge(self, msg, level=0, noiselevel=0): + if not self._verbose and noiselevel >= 0 and level < logging.WARN: + return if self._scheduler is not None: self._scheduler.dblinkDisplayMerge(self, msg, level=level, noiselevel=noiselevel) @@ -3534,7 +3491,9 @@ class dblink(object): # couldn't get merged will be added to thirdhand. thirdhand = [] - self.mergeme(srcroot, destroot, outfile, thirdhand, secondhand, cfgfiledict, mymtime) + if self.mergeme(srcroot, destroot, outfile, thirdhand, + secondhand, cfgfiledict, mymtime): + return 1 #swap hands lastlen = len(secondhand) @@ -3546,7 +3505,9 @@ class dblink(object): if len(secondhand): # force merge of remaining symlinks (broken or circular; oh well) - self.mergeme(srcroot, destroot, outfile, None, secondhand, cfgfiledict, mymtime) + if self.mergeme(srcroot, destroot, outfile, None, + secondhand, cfgfiledict, mymtime): + return 1 #restore umask os.umask(prevmask) @@ -3689,6 +3650,7 @@ class dblink(object): """ showMessage = self._display_merge + writemsg = self._display_merge scheduler = self._scheduler from os.path import sep, join @@ -3725,7 +3687,7 @@ class dblink(object): writemsg(red("!!! and ensure your filesystem is in a sane state. ")+bold("'shutdown -Fr now'\n")) writemsg(red("!!! File: ")+str(mysrc)+"\n", noiselevel=-1) writemsg(red("!!! Error: ")+str(e)+"\n", noiselevel=-1) - sys.exit(1) + return 1 except Exception, e: writemsg("\n") writemsg(red("!!! ERROR: An unknown error has occurred during the merge process.\n")) @@ -3734,7 +3696,7 @@ class dblink(object): writemsg( "!!! this as a portage bug at bugs.gentoo.org. Append 'emerge info'.\n") writemsg( "!!! File: "+str(mysrc)+"\n", noiselevel=-1) writemsg( "!!! Error: "+str(e)+"\n", noiselevel=-1) - sys.exit(1) + return 1 mymode = mystat[stat.ST_MODE] @@ -3800,9 +3762,11 @@ class dblink(object): showMessage(">>> %s -> %s\n" % (mydest, myto)) outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") else: - print "!!! Failed to move file." - print "!!!", mydest, "->", myto - sys.exit(1) + showMessage("!!! Failed to move file.\n", + level=logging.ERROR, noiselevel=-1) + showMessage("!!! %s -> %s\n" % (mydest, myto), + level=logging.ERROR, noiselevel=-1) + return 1 elif stat.S_ISDIR(mymode): # we are merging a directory if mydmode != None: @@ -3831,8 +3795,9 @@ class dblink(object): else: # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. if movefile(mydest, mydest+".backup", mysettings=self.settings) is None: - sys.exit(1) - print "bak", mydest, mydest+".backup" + return 1 + showMessage("bak %s %s.backup\n" % (mydest, mydest), + level=logging.ERROR, noiselevel=-1) #now create our directory if self.settings.selinux_enabled(): import selinux @@ -3916,7 +3881,7 @@ class dblink(object): if moveme: mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) if mymtime is None: - sys.exit(1) + return 1 zing = ">>>" if mymtime != None: @@ -3930,7 +3895,7 @@ class dblink(object): if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) != None: zing = ">>>" else: - sys.exit(1) + return 1 if stat.S_ISFIFO(mymode): outfile.write("fif %s\n" % myrealdest) else: -- 2.26.2