From 7ae3083fcde6665886ea4df61c17f05c7f698617 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sun, 30 Dec 2007 17:08:29 +0000 Subject: [PATCH] Merged from trunk 9063:9094 | 9064 | Fixes for logic related to FEATURES=test to USE=test | | zmedico | mapping: * Add EBUILD_FORCE_TEST to the environment | | | whitelist and filter it from the saved environment in | | | save_ebuild_env(). * Tweak logic inside the ebuild command | | | and config.regenerate() so that EBUILD_FORCE_TEST works even | | | in odd cases like when USE=test is masked. * Only make | | | FEATURES=test map to USE=test when "test" is actually in | | | IUSE. * Remove USE=test from the set of implicit IUSE so | | | that useq() calls in ebuild.sh properly generate a QA Notice | | | when "test" is missing from IUSE. | | 9065 | Show the EBUILD_FORCE_TEST "Forcing test" message even when | | zmedico | "test" is not in IUSE. | | 9066 | In action_regen(), use writemsg() to send error messages to | | zmedico | stderr. | | 9078 | Just ignore it when the global counter file does not match | | zmedico | the highest installed COUNTER since it's normal for them to | | | differ in some cases. For example, the global counter file | | | will contain a higher value if the user merges a package and | | | then immediately unmerges it (unmerge does not update the | | | global counter file, but merge does). | | 9080 | Make dep_expand() stop relying on having a categories list: | | zmedico | * Create and use a dbapi.categories property that is | | | automatically generated from dbapi.cp_all(). * Make mutable | | | dbapi instances delete the invalidate the cached categories | | | when they need to be regenerated. | | 9081 | Make key_expand() use dbapi.categories. | | zmedico | | | 9083 | Make the FEATURES=sfperms loops safe for whitespace in paths | | zmedico | by using find -print0 | while read -d $'\0'. | | 9085 | Bug #202697 - Add / to the default initial SANDBOX_READ in | | zmedico | order to avoid a situation where attempts to read arbitrary | | | files trigger sandbox violations. | | 9087 | Bug #121584 - Make the automatic USERLAND detection code | | zmedico | recognize GNU/kFreeBSD. | | 9089 | Make the ARCH and USERLAND sanity check use the autodetected | | zmedico | USERLAND from the portage.data module if necessary. | | 9090 | Make doebuild() check XARGS together with ARCH and USERLAND. | | zmedico | | | 9092 | * Initialize dbapi._categories = None so that we can compare | | zmedico | with None instead of using hasattr. * Remove unwanted | | | self._categories initialization from the portdbapi | | | constructor. | | 9094 | Add support for showing binary package file sizes in emerge | | zmedico | --search output. | svn path=/main/branches/prefix/; revision=9123 --- bin/ebuild | 14 +++++---- bin/ebuild.sh | 5 +--- bin/isolated-functions.sh | 2 +- bin/misc-functions.sh | 8 +++--- pym/_emerge/__init__.py | 33 +++++++++++++--------- pym/portage/__init__.py | 53 ++++++++++++++++++++++++----------- pym/portage/data.py | 2 +- pym/portage/dbapi/__init__.py | 18 ++++++++++++ pym/portage/dbapi/porttree.py | 1 - pym/portage/dbapi/vartree.py | 18 ++++++------ pym/portage/dbapi/virtual.py | 2 ++ 11 files changed, 102 insertions(+), 54 deletions(-) diff --git a/bin/ebuild b/bin/ebuild index 4db0f462e..eb5976315 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -133,13 +133,17 @@ def discard_digests(myebuild, mysettings, mydbapi): portage._doebuild_manifest_exempt_depend -= 1 tmpsettings = portage.config(clone=portage.settings) -if "test" in pargs and "test" not in tmpsettings.features: - print "Forcing test." +if "test" in pargs: + # This variable is a signal to config.regenerate() to + # indicate that the test phase should be enabled regardless + # of problems such as masked "test" USE flag. tmpsettings["EBUILD_FORCE_TEST"] = "1" tmpsettings.backup_changes("EBUILD_FORCE_TEST") - tmpsettings.features.append("test") - tmpsettings["FEATURES"] = " ".join(tmpsettings.features) - tmpsettings.backup_changes("FEATURES") + if "test" not in tmpsettings.features: + tmpsettings.features.append("test") + tmpsettings.features.sort() + tmpsettings["FEATURES"] = " ".join(tmpsettings.features) + tmpsettings.backup_changes("FEATURES") build_dir_phases = set(["setup", "unpack", "compile", "test", "install", "package", "rpm"]) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index b38db01dd..0d596a8a8 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -9,7 +9,7 @@ PORTAGE_PYM_PATH="${PORTAGE_PYM_PATH:-@PORTAGE_BASE@/pym}" SANDBOX_PREDICT="${SANDBOX_PREDICT}:/proc/self/maps:/dev/console:/dev/random" export SANDBOX_PREDICT="${SANDBOX_PREDICT}:${PORTAGE_PYM_PATH}:${PORTAGE_DEPCACHEDIR}" export SANDBOX_WRITE="${SANDBOX_WRITE}:/dev/shm:/dev/stdout:/dev/stderr:${PORTAGE_TMPDIR}" -export SANDBOX_READ="${SANDBOX_READ}:/dev/shm:/dev/stdin:${PORTAGE_TMPDIR}" +export SANDBOX_READ="${SANDBOX_READ}:/:/dev/shm:/dev/stdin:${PORTAGE_TMPDIR}" # Don't use sandbox's BASH_ENV for new shells because it does # 'source /etc/profile' which can interfere with the build # environment by modifying our PATH. @@ -939,9 +939,6 @@ dyn_test() { fi if ! hasq test $FEATURES && [ "${EBUILD_FORCE_TEST}" != "1" ]; then vecho ">>> Test phase [not enabled]: ${CATEGORY}/${PF}" - elif ! hasq test ${USE} && [ "${EBUILD_FORCE_TEST}" != "1" ]; then - ewarn "Skipping make test/check since USE=test is masked." - vecho ">>> Test phase [explicitly disabled]: ${CATEGORY}/${PF}" elif hasq test $RESTRICT; then ewarn "Skipping make test/check due to ebuild restriction." vecho ">>> Test phase [explicitly disabled]: ${CATEGORY}/${PF}" diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 60bbabb71..90fc0f59a 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -436,7 +436,7 @@ save_ebuild_env() { # portage config variables and variables set directly by portage unset BAD BRACKET BUILD_PREFIX COLS \ DISTCC_DIR DISTDIR DOC_SYMLINKS_DIR \ - EBUILD_EXIT_STATUS_FILE EBUILD_MASTER_PID \ + EBUILD_EXIT_STATUS_FILE EBUILD_FORCE_TEST EBUILD_MASTER_PID \ ECLASSDIR ECLASS_DEPTH ENDCOL FAKEROOTKEY \ GOOD HILITE HOME IMAGE \ LAST_E_CMD LAST_E_LEN LD_PRELOAD MISC_FUNCTIONS_ARGS MOPREFIX \ diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 74387bc93..062fe9d88 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -512,8 +512,8 @@ preinst_sfperms() { # Smart FileSystem Permissions if hasq sfperms $FEATURES; then local i -#note not space-safe - for i in $(find "${ED}" -type f -perm -4000); do + find "${ED}" -type f -perm -4000 -print0 | \ + while read -d $'\0' i ; do if [ -n "$(find "$i" -perm -2000)" ] ; then ebegin ">>> SetUID and SetGID: [chmod o-r] /${i#${D}}" chmod o-r "$i" @@ -524,8 +524,8 @@ preinst_sfperms() { eend $? fi done -#note not space-safe - for i in $(find "${ED}" -type f -perm -2000); do + find "${ED}" -type f -perm -2000 -print0 | \ + while read -d $'\0' i ; do if [ -n "$(find "$i" -perm -4000)" ] ; then # This case is already handled # by the SetUID check above. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index c5e59834d..e705bd3e0 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -678,12 +678,6 @@ class search(object): mysettings=self.settings, all=True)[1] try: mysum[0] = mf.getDistfilesSize(fetchlist) - mystr = str(mysum[0] / 1024) - mycount = len(mystr) - while (mycount > 3): - mycount -= 3 - mystr = mystr[:mycount] + "," + mystr[mycount:] - mysum[0] = mystr + " kB" except KeyError, e: mysum[0] = "Unknown (missing digest for %s)" % \ str(e) @@ -693,8 +687,22 @@ class search(object): if db is not vardb and \ db.cpv_exists(mycpv): available = True + if not myebuild and hasattr(db, "bintree"): + myebuild = db.bintree.getname(mycpv) + try: + mysum[0] = os.stat(myebuild).st_size + except OSError: + myebuild = None break + if myebuild: + mystr = str(mysum[0] / 1024) + mycount = len(mystr) + while (mycount > 3): + mycount -= 3 + mystr = mystr[:mycount] + "," + mystr[mycount:] + mysum[0] = mystr + " kB" + if self.verbose: if available: print " ", darkgreen("Latest version available:"),myversion @@ -5925,8 +5933,8 @@ def action_regen(settings, portdb): try: dead_nodes[mytree] = set(portdb.auxdb[mytree].iterkeys()) except CacheError, e: - print "Error listing cache entries for " + \ - "'%s': %s, continuing..." % (mytree, e) + portage.writemsg("Error listing cache entries for " + \ + "'%s': %s, continuing...\n" % (mytree, e), noiselevel=-1) del e dead_nodes = None break @@ -5936,11 +5944,10 @@ def action_regen(settings, portdb): for y in mymatches: try: foo = portdb.aux_get(y,["DEPEND"]) - except SystemExit, e: - # sys.exit is an exception... And consequently, we can't catch it. - raise - except Exception, e: - print "Error processing %(cpv)s, continuing... (%(e)s)" % {"cpv":y,"e":str(e)} + except (KeyError, portage.exception.PortageException), e: + portage.writemsg( + "Error processing %(cpv)s, continuing... (%(e)s)\n" % \ + {"cpv":y,"e":str(e)}, noiselevel=-1) if dead_nodes: for mytree in portdb.porttrees: if portdb.findname2(y, mytree=mytree)[0]: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index b9f20ca08..160ba4d25 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -875,8 +875,9 @@ class config(object): # in it's bashrc (causing major leakage). _environ_whitelist += [ "BASH_ENV", "BUILD_PREFIX", "D", - "DISTDIR", "DOC_SYMLINKS_DIR", "EBUILD_EXIT_STATUS_FILE", - "EBUILD", "EBUILD_PHASE", "ECLASSDIR", "ECLASS_DEPTH", "EMERGE_FROM", + "DISTDIR", "DOC_SYMLINKS_DIR", "EBUILD", + "EBUILD_EXIT_STATUS_FILE", "EBUILD_FORCE_TEST", + "EBUILD_PHASE", "ECLASSDIR", "ECLASS_DEPTH", "EMERGE_FROM", "FEATURES", "FILESDIR", "HOME", "PATH", "PKGUSE", "PKG_LOGDIR", "PKG_TMPDIR", "PORTAGE_ACTUAL_DISTDIR", "PORTAGE_ARCHLIST", @@ -1550,8 +1551,8 @@ class config(object): self[var] = "0" self.backup_changes(var) + # initialize self.features self.regenerate() - self.features = portage.util.unique_array(self["FEATURES"].split()) if local_config: self._accept_license = \ @@ -2335,19 +2336,37 @@ class config(object): myflags.update(self.useforce) + iuse = self.configdict["pkg"].get("IUSE","").split() + iuse = [ x.lstrip("+-") for x in iuse ] # FEATURES=test should imply USE=test - if "test" in self.configlist[-1].get("FEATURES","").split(): - myflags.add("test") - if self.get("EBUILD_FORCE_TEST") == "1": - self.usemask.discard("test") + if not hasattr(self, "features"): + self.features = list(sorted(set( + self.configlist[-1].get("FEATURES","").split()))) + self["FEATURES"] = " ".join(self.features) + ebuild_force_test = self.get("EBUILD_FORCE_TEST") == "1" + if ebuild_force_test and \ + self.get("EBUILD_PHASE") == "test" and \ + not hasattr(self, "_ebuild_force_test_msg_shown"): + self._ebuild_force_test_msg_shown = True + writemsg("Forcing test.\n", noiselevel=-1) + if "test" in self.features and "test" in iuse: + if "test" in self.usemask and not ebuild_force_test: + # "test" is in IUSE and USE=test is masked, so execution + # of src_test() probably is not reliable. Therefore, + # temporarily disable FEATURES=test just for this package. + self["FEATURES"] = " ".join(x for x in self.features \ + if x != "test") + myflags.discard("test") + else: + myflags.add("test") + if ebuild_force_test: + self.usemask.discard("test") usesplit = [ x for x in myflags if \ x not in self.usemask] # Use the calculated USE flags to regenerate the USE_EXPAND flags so # that they are consistent. - iuse = self.configdict["pkg"].get("IUSE","").split() - iuse = [ x.lstrip("+-") for x in iuse ] for var in use_expand: prefix = var.lower() + "_" prefix_len = len(prefix) @@ -2415,7 +2434,6 @@ class config(object): # * Masked flags, such as those from {,package}use.mask # * Forced flags, such as those from {,package}use.force # * build and bootstrap flags used by bootstrap.sh - # * The "test" flag that's enabled by FEATURES=test # Do this even when there's no package since setcpv() can # optimize away regenerate() calls. @@ -2444,7 +2462,6 @@ class config(object): # build and bootstrap flags used by bootstrap.sh iuse_implicit.add("build") iuse_implicit.add("bootstrap") - iuse_implicit.add("test") # prefix flag is used in Prefix iuse_implicit.add("prefix") @@ -4571,8 +4588,12 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if env_stat: mysettings._filter_calling_env = True else: - for var in "ARCH", "USERLAND": - if mysettings.get(var): + for var in "ARCH", "USERLAND", "XARGS": + value = mysettings.get(var) + if value and value.strip(): + continue + if var == "USERLAND" and userland: + mysettings["USERLAND"] = userland continue msg = ("%s is not set... " % var) + \ ("Are you missing the '%setc/make.profile' symlink? " % \ @@ -5545,7 +5566,7 @@ def key_expand(mykey, mydb=None, use_cache=1, settings=None): virts_p = settings.get_virts_p("/") if len(mysplit)==1: if hasattr(mydb, "cp_list"): - for x in settings.categories: + for x in mydb.categories: if mydb.cp_list(x+"/"+mykey,use_cache=use_cache): return x+"/"+mykey if virts_p.has_key(mykey): @@ -5601,8 +5622,8 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None): myp=mycpv mykey=None matches=[] - if mydb: - for x in settings.categories: + if mydb and hasattr(mydb, "categories"): + for x in mydb.categories: if mydb.cp_list(x+"/"+myp,use_cache=use_cache): matches.append(x+"/"+myp) if len(matches) > 1: diff --git a/pym/portage/data.py b/pym/portage/data.py index dc4400fd9..88735d5e6 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -14,7 +14,7 @@ ostype=os.uname()[0] userland = None lchown = getattr(os, "lchown", None) -os.environ.setdefault("XARGS", "xargs") +os.environ.setdefault("XARGS", "xargs -r") os.environ["XARGS"]="xargs -r" # "fix" for lchown on Darwin diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index 0e473ddd1..cd3208d50 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -15,9 +15,27 @@ from portage.versions import catpkgsplit, catsplit, pkgcmp class dbapi(object): _category_re = re.compile(r'^\w[-.+\w]*$') _pkg_dir_name_re = re.compile(r'^\w[-+\w]*$') + _categories = None def __init__(self): pass + @property + def categories(self): + """ + Use self.cp_all() to generate a category list. Mutable instances + can delete the self._categories attribute in cases when the cached + categories become invalid and need to be regenerated. + """ + if self._categories is not None: + return self._categories + categories = set() + cat_pattern = re.compile(r'(.*)/.*') + for cp in self.cp_all(): + categories.add(cat_pattern.match(cp).group(1)) + self._categories = list(categories) + self._categories.sort() + return self._categories + def close_caches(self): pass diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 7ce65b998..61ce3c47b 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -39,7 +39,6 @@ class portdbapi(dbapi): else: from portage import settings self.mysettings = config(clone=settings) - self._categories = set(self.mysettings.categories) # This is strictly for use in aux_get() doebuild calls when metadata # is generated by the depend phase. It's safest to use a clone for # this purpose because doebuild makes many changes to the config diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 59eb67a37..5efdcc40e 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -166,6 +166,11 @@ class LibraryPackageMap(object): class vardbapi(dbapi): def __init__(self, root, categories=None, settings=None, vartree=None): + """ + The categories parameter is unused since the dbapi class + now has a categories property that is generated from the + available packages. + """ self.root = root[:] #cache for category directory mtimes @@ -181,9 +186,6 @@ class vardbapi(dbapi): if settings is None: from portage import settings self.settings = settings - # The categories list is now automatically generated - # from a regular expression. - self.categories = None if vartree is None: from portage import db vartree = db[root]["vartree"] @@ -647,11 +649,6 @@ class vardbapi(dbapi): writemsg("!!! Initializing COUNTER to " + \ "value of %d\n" % counter, noiselevel=-1) - if counter != real_counter: - changed = True - writemsg("!!! Initializing COUNTER to " + \ - "value of %d\n" % real_counter, noiselevel=-1) - # Never trust the counter file, since having a # corrupt value that is too low there can trigger # incorrect AUTOCLEAN behavior due to newly installed @@ -1076,7 +1073,8 @@ class dblink(object): The caller must ensure that lockdb() and unlockdb() are called before and after this method. """ - + if self.vartree.dbapi._categories is not None: + self.vartree.dbapi._categories = None # When others_in_slot is supplied, the security check has already been # done for this slot, so it shouldn't be repeated until the next # replacement or unmerge operation. @@ -2430,6 +2428,8 @@ class dblink(object): we won't be able to later if they get unmerged (happens when namespace changes). """ + if self.vartree.dbapi._categories is not None: + self.vartree.dbapi._categories = None if self.myroot == "/" and \ "sys-apps" == self.cat and \ "portage" == pkgsplit(self.pkg)[0] and \ diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py index fe9356212..444b5363a 100644 --- a/pym/portage/dbapi/virtual.py +++ b/pym/portage/dbapi/virtual.py @@ -26,6 +26,8 @@ class fakedbapi(dbapi): self._match_cache = {} def _clear_cache(self): + if self._categories is not None: + self._categories = None if self._match_cache: self._match_cache = {} -- 2.26.2