From 1fa3fac5a4f5a7158475526d48bb8dc118fac08c Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 12 Mar 2009 19:18:52 +0000 Subject: [PATCH] Merged from trunk -r12807:12824 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit | 12808 | Add more QA warnings for econf calls during unexpected | | zmedico | phases. Thanks to Diego Elio Pettenò for | | | the suggestion. | | 12809 | Only execute src_test pre/post hooks when src_test is | | zmedico | actually executed. | | 12810 | Bug #261670 - Generate a QA Notice if CHOST (from | | zmedico | dyn_install) has a value which differs from the initial | | | setting. | | 12811 | Inside create_trees(), when cleansing the environment for | | zmedico | the ROOT=/ config instance, call config.reset() so that | | | configdict['env'] is cleared as was originally intended | | | here. This prevents thinks like CHOST overrides from the | | | calling environment from inappropriately being applied to | | | ROOT=/. | | 12812 | Simplify environment cleansing that's done for the ROOT=/ | | zmedico | config instance when ROOT!=/. A new "env" config constructor | | | parameter is used to allow the caller to avoid having the | | | config instance polluted by os.environ. | | 12813 | When the config_incrementals parameter is unspecified in the | | zmedico | config constructor, don't warn because it's ridiculous to | | | require this parameter when it's always the same anyway. | | 12814 | Bug #261852 - If an unexpected exception is raise from | | zmedico | emerge_main() then show the traceback after the mod_echo | | | output to that the traceback isn't obscured by mod_echo | | | flooding. | | 12815 | Optimize dbapi._cpv_sort_ascending() by calling vercmp | | zmedico | directly instead of calling pkgcmp. | | 12816 | Initialize UserDict.data in the constructor. | | zmedico | | | 12817 | Simplify cmp_sort_key._cmp_key.__lt__(). | | zmedico | | | 12818 | Fix incorrect slice inside _src_uri_validate() and add new | | zmedico | test case. | | 12819 | Remove redundant conditional inside _src_uri_validate(). | | zmedico | | | 12820 | Make _src_uri_validate() identify the || operator as invalid | | zmedico | and add a corresponding test case. | | 12821 | Use protocol=2 for pickle.dump() calls, so that pickles | | zmedico | written by py3k will be backward compatible. | | 12822 | Handle ValueError and UnpicklingError inside | | zmedico | PreservedLibsRegistry.load(). | | 12823 | Use slices instead of startswith inside | | zmedico | manifest2AuxfileFilter(). | | 12824 | Don't default env=os.environ directly in the config | | zmedico | constructor because that makes it show environment variables | | | in api docs generated by epydoc. | svn path=/main/branches/prefix/; revision=13093 --- bin/ebuild.sh | 23 +++++++---- bin/emerge | 16 ++++++++ pym/_emerge/__init__.py | 2 +- pym/portage/__init__.py | 39 +++++++------------ pym/portage/cache/mappings.py | 2 + pym/portage/dbapi/__init__.py | 16 ++++---- pym/portage/dbapi/porttree.py | 29 +++++++++----- pym/portage/dbapi/vartree.py | 55 +++++++++++++++++---------- pym/portage/getbinpkg.py | 6 +-- pym/portage/manifest.py | 4 +- pym/portage/tests/dep/test_src_uri.py | 2 + pym/portage/util.py | 6 +-- 12 files changed, 118 insertions(+), 82 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index b8f5a6837..5c6fcb366 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -468,9 +468,18 @@ hasgq() { hasg "$@" >/dev/null ; } econf() { local x - ! hasq "$EAPI" 0 1 && [[ $EBUILD_PHASE = compile && \ - $(type -t src_configure) = function ]] && \ - eqawarn "econf called in src_compile instead of src_configure" + local phase_func=$(_ebuild_arg_to_phase "$EAPI" "$EBUILD_PHASE") + if [[ -n $phase_func ]] ; then + if hasq "$EAPI" 0 1 ; then + [[ $phase_func != src_compile ]] && \ + eqawarn "QA Notice: econf called in" \ + "$phase_func instead of src_compile" + else + [[ $phase_func != src_configure ]] && \ + eqawarn "QA Notice: econf called in" \ + "$phase_func instead of src_configure" + fi + fi : ${ECONF_SOURCE:=.} if [ -x "${ECONF_SOURCE}/configure" ]; then @@ -933,7 +942,6 @@ dyn_test() { # like it's supposed to here. ! hasq test ${USE} && export USE="${USE} test" fi - ebuild_phase pre_src_test if [[ -e $PORTAGE_BUILDDIR/.tested ]] ; then vecho ">>> It appears that ${PN} has already been tested; skipping." return @@ -951,13 +959,14 @@ dyn_test() { vecho ">>> Test phase [explicitly disabled]: ${CATEGORY}/${PF}" else addpredict / + ebuild_phase pre_src_test ebuild_phase src_test + touch "$PORTAGE_BUILDDIR/.tested" || \ + die "Failed to 'touch .tested' in $PORTAGE_BUILDDIR" + ebuild_phase post_src_test SANDBOX_PREDICT="${SANDBOX_PREDICT%:/}" fi - touch "$PORTAGE_BUILDDIR/.tested" || \ - die "Failed to 'touch .tested' in $PORTAGE_BUILDDIR" - ebuild_phase post_src_test trap - SIGINT SIGQUIT } diff --git a/bin/emerge b/bin/emerge index 8fed84c58..a8bea23e3 100755 --- a/bin/emerge +++ b/bin/emerge @@ -46,4 +46,20 @@ if __name__ == "__main__": except ParseError, e: sys.stderr.write("%s\n" % str(e)) sys.exit(1) + except SystemExit: + raise + except Exception: + # If an unexpected exception occurs then we don't want the mod_echo + # output to obscure the traceback, so dump the mod_echo output before + # showing the traceback. + import traceback + tb_str = traceback.format_exc() + try: + from portage.elog import mod_echo + except ImportError: + pass + else: + mod_echo.finalize() + sys.stderr.write(tb_str) + sys.exit(1) sys.exit(retval) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 22f6bf4d1..93cee1ba8 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -4221,7 +4221,7 @@ class BlockerCache(portage.cache.mappings.MutableMapping): secpass >= 2: try: f = portage.util.atomic_ofstream(self._cache_filename, mode='wb') - pickle.dump(self._cache_data, f, -1) + pickle.dump(self._cache_data, f, protocol=2) f.close() portage.util.apply_secpass_permissions( self._cache_filename, gid=portage.portage_gid, mode=0644) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 28461d9c4..3527fa7de 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1124,7 +1124,7 @@ class config(object): def __init__(self, clone=None, mycpv=None, config_profile_path=None, config_incrementals=None, config_root=None, target_root=None, - local_config=True): + local_config=True, env=None): """ @param clone: If provided, init will use deepcopy to copy by value the instance. @type clone: Instance of config class. @@ -1133,7 +1133,8 @@ class config(object): @type mycpv: String @param config_profile_path: Configurable path to the profile (usually PROFILE_PATH from portage.const) @type config_profile_path: String - @param config_incrementals: List of incremental variables (usually portage.const.INCREMENTALS) + @param config_incrementals: List of incremental variables + (defaults to portage.const.INCREMENTALS) @type config_incrementals: List @param config_root: path to read local config from (defaults to "/", see PORTAGE_CONFIGROOT) @type config_root: String @@ -1142,6 +1143,9 @@ class config(object): @param local_config: Enables loading of local config (/etc/portage); used most by repoman to ignore local config (keywording and unmasking) @type local_config: Boolean + @param env: The calling environment which is used to override settings. + Defaults to os.environ if unspecified. + @type env: dict """ # When initializing the global portage.settings instance, avoid @@ -1265,8 +1269,7 @@ class config(object): else: self.profile_path = config_profile_path[:] - if not config_incrementals: - writemsg("incrementals not specified to class config\n") + if config_incrementals is None: self.incrementals = copy.deepcopy(portage.const.INCREMENTALS) else: self.incrementals = copy.deepcopy(config_incrementals) @@ -1459,7 +1462,9 @@ class config(object): expand_map.update(env_d) # backupenv is used for calculating incremental variables. - self.backupenv = os.environ.copy() + if env is None: + env = os.environ + self.backupenv = env.copy() if env_d: # Remove duplicate values so they don't override updated @@ -7619,7 +7624,7 @@ def commit_mtimedb(mydict=None, filename=None): d.update(mydict) try: f = atomic_ofstream(filename, mode='wb') - pickle.dump(d, f, -1) + pickle.dump(d, f, protocol=2) f.close() portage.util.apply_secpass_permissions(filename, uid=uid, gid=portage_gid, mode=0644) @@ -7978,27 +7983,11 @@ def create_trees(config_root=None, target_root=None, trees=None): myroots = [(settings["ROOT"], settings)] if settings["ROOT"] != "/": - settings = config(config_root=None, target_root="/", - config_incrementals=portage.const.INCREMENTALS) + # When ROOT != "/" we only want overrides from the calling # environment to apply to the config that's associated - # with ROOT != "/", so we wipe out the "backupenv" for the - # config that is associated with ROOT == "/" and regenerate - # it's incrementals. - # Preserve backupenv values that are initialized in the config - # constructor. Also, preserve XARGS since it is set by the - # portage.data module. - - backupenv_whitelist = settings._environ_whitelist - backupenv = settings.configdict["backupenv"] - env_d = settings.configdict["env.d"] - for k, v in os.environ.iteritems(): - if k in backupenv_whitelist: - continue - if k in env_d or \ - v == backupenv.get(k): - backupenv.pop(k, None) - settings.regenerate() + # with ROOT != "/", so pass an empty dict for the env parameter. + settings = config(config_root=None, target_root="/", env={}) settings.lock() myroots.append((settings["ROOT"], settings)) diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py index 77923b462..066927ac1 100644 --- a/pym/portage/cache/mappings.py +++ b/pym/portage/cache/mappings.py @@ -148,6 +148,8 @@ class UserDict(MutableMapping): def __init__(self, *args, **kwargs): + self.data = {} + if len(args) > 1: raise TypeError( "expected at most 1 positional argument, got " + \ diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index 6bea1e5d3..9956d0999 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -13,7 +13,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.locks:unlockfile', 'portage.output:colorize', 'portage.util:cmp_sort_key,writemsg', - 'portage.versions:catpkgsplit,pkgcmp', + 'portage.versions:catpkgsplit,vercmp', ) from portage import auxdbkeys, dep_expand @@ -60,14 +60,12 @@ class dbapi(object): # If the cpv includes explicit -r0, it has to be preserved # for consistency in findname and aux_get calls, so use a # dict to map strings back to their original values. - str_map = {} - for i, cpv in enumerate(cpv_list): - mysplit = tuple(catpkgsplit(cpv)[1:]) - str_map[mysplit] = cpv - cpv_list[i] = mysplit - cpv_list.sort(key=cmp_sort_key(pkgcmp)) - for i, mysplit in enumerate(cpv_list): - cpv_list[i] = str_map[mysplit] + ver_map = {} + for cpv in cpv_list: + ver_map[cpv] = '-'.join(catpkgsplit(cpv)[2:]) + def cmp_cpv(cpv1, cpv2): + return vercmp(ver_map[cpv1], ver_map[cpv2]) + cpv_list.sort(key=cmp_sort_key(cmp_cpv)) def cpv_all(self): """Return all CPVs in the db diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 57bdd9984..93730e9c3 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -45,7 +45,12 @@ def _src_uri_validate(cpv, eapi, src_uri): uri = None _src_uri_validate(cpv, eapi, x) continue - if x[:-1] == "?": + if x == '||': + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI contains invalid " + \ + "|| operator") % (cpv,)) + + if x[-1:] == "?": if operator is not None: raise portage.exception.InvalidDependString( ("getFetchMap(): '%s' SRC_URI arrow missing " + \ @@ -69,15 +74,19 @@ def _src_uri_validate(cpv, eapi, src_uri): if operator is None: uri = x continue - if operator is not None: - if "/" in x: - raise portage.exception.InvalidDependString( - ("getFetchMap(): '%s' SRC_URI '/' character in " + \ - "file name: '%s'") % (cpv, x)) - if x[-1] == "?": - raise portage.exception.InvalidDependString( - ("getFetchMap(): '%s' SRC_URI arrow missing " + \ - "right operand") % (cpv,)) + + # This should be the right operand of an arrow operator. + if "/" in x: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI '/' character in " + \ + "file name: '%s'") % (cpv, x)) + + if x[-1:] == "?": + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrow missing " + \ + "right operand") % (cpv,)) + + # Found the right operand, so reset state. uri = None operator = None diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 68cbd3572..256d15646 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -62,15 +62,21 @@ class PreservedLibsRegistry(object): def load(self): """ Reload the registry data from file """ + self._data = None try: self._data = pickle.load(open(self._filename, 'rb')) + except (ValueError, pickle.UnpicklingError), e: + writemsg_level("!!! Error loading '%s': %s\n" % \ + (self._filename, e), level=logging.ERROR, noiselevel=-1) except (EOFError, IOError), e: if isinstance(e, EOFError) or e.errno == errno.ENOENT: - self._data = {} + pass elif e.errno == PermissionDenied.errno: raise PermissionDenied(self._filename) else: - raise e + raise + if self._data is None: + self._data = {} self._data_orig = self._data.copy() def store(self): """ Store the registry data to file. No need to call this if autocommit @@ -81,7 +87,7 @@ class PreservedLibsRegistry(object): return try: f = atomic_ofstream(self._filename, 'wb') - pickle.dump(self._data, f) + pickle.dump(self._data, f, protocol=2) f.close() except EnvironmentError, e: if e.errno != PermissionDenied.errno: @@ -1559,7 +1565,7 @@ class vardbapi(dbapi): del self._aux_cache["modified"] try: f = atomic_ofstream(self._aux_cache_filename, 'wb') - pickle.dump(self._aux_cache, f, -1) + pickle.dump(self._aux_cache, f, protocol=2) f.close() apply_secpass_permissions( self._aux_cache_filename, gid=portage_gid, mode=0644) @@ -3535,6 +3541,15 @@ class dblink(object): return 1 + def _eqawarn(self, phase, lines): + from portage.elog.messages import eqawarn as _eqawarn + if self._scheduler is None: + for l in lines: + _eqawarn(l, phase=phase, key=self.settings.mycpv) + else: + self._scheduler.dblinkElog(self, + phase, _eqawarn, lines) + def _eerror(self, phase, lines): from portage.elog.messages import eerror as _eerror if self._scheduler is None: @@ -3591,29 +3606,27 @@ class dblink(object): level=logging.ERROR, noiselevel=-1) return 1 - inforoot_slot_file = os.path.join(inforoot, "SLOT") - slot = None - try: - f = open(inforoot_slot_file) + slot = '' + for var_name in ('CHOST', 'SLOT'): try: - slot = f.read().strip() - finally: - f.close() - except EnvironmentError, e: - if e.errno != errno.ENOENT: - raise - del e + val = open(os.path.join(inforoot, var_name)).readline().strip() + except EnvironmentError, e: + if e.errno != errno.ENOENT: + raise + del e + val = '' - if slot is None: - slot = "" + if var_name == 'SLOT': + slot = val + + if val != self.settings.get(var_name, ''): + self._eqawarn('preinst', + ["QA Notice: Expected %s='%s', got '%s'\n" % \ + (var_name, self.settings.get(var_name, ''), val)]) def eerror(lines): self._eerror("preinst", lines) - if slot != self.settings["SLOT"]: - showMessage("!!! WARNING: Expected SLOT='%s', got '%s'\n" % \ - (self.settings["SLOT"], slot), level=logging.WARN) - if not os.path.exists(self.dbcatdir): os.makedirs(self.dbcatdir) diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index d02207609..bf616637b 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -582,7 +582,7 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= sys.stderr.flush() try: metadatafile = open(metadatafilename, 'wb') - pickle.dump(metadata,metadatafile) + pickle.dump(metadata, metadatafile, protocol=2) metadatafile.close() except SystemExit, e: raise @@ -674,11 +674,11 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= if "modified" in metadata[baseurl] and metadata[baseurl]["modified"]: metadata[baseurl]["timestamp"] = int(time.time()) metadatafile = open(metadatafilename, 'wb') - pickle.dump(metadata,metadatafile) + pickle.dump(metadata, metadatafile, protocol=2) metadatafile.close() if makepickle: metadatafile = open(makepickle, 'wb') - pickle.dump(metadata[baseurl]["data"],metadatafile) + pickle.dump(metadata[baseurl]["data"], metadatafile, protocol=2) metadatafile.close() except SystemExit, e: raise diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 02836e7ec..9150674a9 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -21,9 +21,9 @@ def manifest2AuxfileFilter(filename): if "CVS" in mysplit: return False for x in mysplit: - if x.startswith("."): + if x[:1] == '.': return False - return not filename.startswith("digest-") + return not filename[:7] == 'digest-' def manifest2MiscfileFilter(filename): filename = filename.strip(os.sep) diff --git a/pym/portage/tests/dep/test_src_uri.py b/pym/portage/tests/dep/test_src_uri.py index e53297968..9955ca096 100644 --- a/pym/portage/tests/dep/test_src_uri.py +++ b/pym/portage/tests/dep/test_src_uri.py @@ -14,6 +14,7 @@ class SrcUri(TestCase): tests = [ ( "0", "http://foo/bar -> blah.tbz2" , False ), ( "1", "http://foo/bar -> blah.tbz2" , False ), + ( "2", "|| ( http://foo/bar -> blah.tbz2 )" , False ), ( "2", "http://foo/bar -> blah.tbz2" , True ), ( "2", "foo? ( http://foo/bar -> blah.tbz2 )" , True ), ( "2", "http://foo/bar -> foo? ( ftp://foo/a )" , False ), @@ -21,6 +22,7 @@ class SrcUri(TestCase): ( "2", "http://foo/bar blah.tbz2 ->" , False ), ( "2", "-> http://foo/bar blah.tbz2 )" , False ), ( "2", "http://foo/bar ->" , False ), + ( "2", "http://foo/bar -> foo? ( http://foo.com/foo )" , False ), ( "2", "foo? ( http://foo/bar -> ) blah.tbz2" , False ), ( "2", "http://foo/bar -> foo/blah.tbz2" , False ), ( "2", "http://foo.com/foo http://foo/bar -> blah.tbz2" , True ), diff --git a/pym/portage/util.py b/pym/portage/util.py index 938eb89a7..a1413c64d 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -669,12 +669,10 @@ class cmp_sort_key(object): self._obj = obj def __lt__(self, other): - if not isinstance(other, self.__class__): + if other.__class__ is not self.__class__: raise TypeError("Expected type %s, got %s" % \ (self.__class__, other.__class__)) - if self._cmp_func(self._obj, other._obj) < 0: - return True - return False + return self._cmp_func(self._obj, other._obj) < 0 def unique_array(s): """lifted from python cookbook, credit: Tim Peters -- 2.26.2