From: Fabian Groffen Date: Thu, 26 Mar 2009 20:42:02 +0000 (-0000) Subject: Merged from trunk -r13144:13174 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=dff9df687fc3c53ed3f850ab02978863153df979;p=portage.git Merged from trunk -r13144:13174 | 13145 | Replace xrange() usage with normal iteration and enumerate() | | zmedico | where appropriate. Thanks to Douglass Anderson | | | for the initial patch. | | 13146 | When triggering a fresh unpack inside dyn_unpack(), remove | | zmedico | the .prepared, .configured and .installed files along with | | | the others. | | 13147 | Add "prepare" to the list of phases that trigger a warning | | zmedico | about stale ${T}/environment. | | 13148 | Make ebuild(1) detect ebuild/eclass changes and | | zmedico | automatically source the ebuild atain in this case (even | | | though $T/environment may already exist). This should help | | | avoid confusion by ensuring that the latest changes to the | | | ebuild/eclasses are reflected in the environment. | | 13149 | Use a separate if/then to group boolean or'd expressions | | zmedico | together, rather than a subshell. | | 13150 | Only check for ebuild/eclass modification when a relevant | | zmedico | phase is called. | | 13168 | Fix misc typos in comments. | | zmedico | | | 13170 | When calling doebuild() to spawn the 'depend' phase, just | | zmedico | call config.setcpv() since that should be enough (no need | | | for separate reset or reload calls). | | 13173 | Add support for FEATURES=parse-eapi-ebuild-head, which is | | zmedico | similar to GLEP 55 except that the EAPI is parsed from the | | | head of the ebuild (first 30 lines). This feature is only | | | intended for experimental purposes and should not be enabled | | | under normal circumstances. | | 13174 | If a package is masked by EAPI then don't show any other | | zmedico | masking reasons. | svn path=/main/branches/prefix/; revision=13192 --- diff --git a/bin/ebuild b/bin/ebuild index 2b47de7ff..4d797177a 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -198,13 +198,21 @@ if opts.skip_manifest: tmpsettings.backup_changes("EBUILD_SKIP_MANIFEST") portage._doebuild_manifest_exempt_depend += 1 -build_dir_phases = set(["setup", "unpack", "configure", "compile", - "test", "install", "package", "rpm"]) +build_dir_phases = set(["setup", "unpack", "prepare", "configure", "compile", + "test", "install", "package", "rpm", "merge", "qmerge"]) + +# If the current metadata is invalid then force the ebuild to be +# sourced again even if $T/environment already exists. +ebuild_changed = False +if build_dir_phases.intersection(pargs): + metadata, st, emtime = \ + portage.portdb._pull_valid_cache(cpv, ebuild, ebuild_portdir) + if metadata is None: + ebuild_changed = True def stale_env_warning(): if "clean" not in pargs and \ "noauto" not in tmpsettings.features and \ - tmpsettings.get("PORTAGE_QUIET") != "1" and \ build_dir_phases.intersection(pargs): portage.doebuild_environment(ebuild, "setup", portage.root, tmpsettings, debug, 1, portage.portdb) @@ -218,6 +226,10 @@ def stale_env_warning(): for x in msg: portage.writemsg(">>> %s\n" % x) + if ebuild_changed: + open(os.path.join(tmpsettings['PORTAGE_BUILDDIR'], + '.ebuild_changed'), 'w') + from portage.exception import PermissionDenied, \ PortagePackageException, UnsupportedAPIException checked_for_stale_env = False diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 9429fb97d..e0baf1de9 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -676,7 +676,7 @@ dyn_unpack() { fi if [ "${newstuff}" == "yes" ]; then # We don't necessarily have privileges to do a full dyn_clean here. - rm -rf "${PORTAGE_BUILDDIR}"/{.unpacked,.compiled,.tested,.packaged,build-info} + rm -rf "${PORTAGE_BUILDDIR}"/{.unpacked,.prepared,.configured,.compiled,.tested,.installed,.packaged,build-info} rm -rf "${WORKDIR}" if [ -d "${T}" ] && \ ! hasq keeptemp $FEATURES && ! hasq keepwork $FEATURES ; then @@ -724,7 +724,7 @@ dyn_clean() { fi if [[ $EMERGE_FROM = binary ]] || ! hasq keepwork $FEATURES; then - rm -f "$PORTAGE_BUILDDIR"/.{exit_status,logid,unpacked,prepared} \ + rm -f "$PORTAGE_BUILDDIR"/.{ebuild_changed,exit_status,logid,unpacked,prepared} \ "$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged} rm -rf "${PORTAGE_BUILDDIR}/build-info" @@ -1876,12 +1876,11 @@ if ! hasq "$EBUILD_PHASE" clean cleanrm depend && \ unset PORTAGE_SANDBOX_ON fi -if ! hasq "$EBUILD_PHASE" clean cleanrm && \ - ( - hasq ${EBUILD_PHASE} depend || \ - [ ! -f "${T}"/environment ] || \ - hasq noauto ${FEATURES} - ) ; then +if ! hasq "$EBUILD_PHASE" clean cleanrm ; then +if [[ $EBUILD_PHASE = depend || ! -f $T/environment || \ + -f $PORTAGE_BUILDDIR/.ebuild_changed ]] || \ + hasq noauto $FEATURES ; then + # The bashrcs get an opportunity here to set aliases that will be expanded # during sourcing of ebuilds and eclasses. source_all_bashrcs @@ -1895,6 +1894,8 @@ if ! hasq "$EBUILD_PHASE" clean cleanrm && \ if [ "${EBUILD_PHASE}" != "depend" ] ; then RESTRICT=${PORTAGE_RESTRICT} + [[ -e $PORTAGE_BUILDDIR/.ebuild_changed ]] && \ + rm "$PORTAGE_BUILDDIR/.ebuild_changed" fi # This next line is not the same as export RDEPEND=${RDEPEND:-${DEPEND}} @@ -1919,6 +1920,7 @@ if ! hasq "$EBUILD_PHASE" clean cleanrm && \ # This needs to be exported since prepstrip is a separate shell script. [[ -n $QA_PRESTRIPPED ]] && export QA_PRESTRIPPED fi +fi # Set default EAPI if necessary, so that most # code can simply assume that it's defined. diff --git a/man/make.conf.5 b/man/make.conf.5 index 2f9c36509..268fd4c81 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -269,6 +269,11 @@ Fetch in the background while compiling. Run `tail \-f /var/log/emerge\-fetch.log` in a terminal to view parallel-fetch progress. .TP +.B parse\-eapi\-ebuild\-head +Parse \fBEAPI\fR from the head of the ebuild (first 30 lines). This feature +is only intended for experimental purposes and should not be enabled under +normal circumstances. +.TP .B preserve\-libs Preserve libraries when the sonames change during upgrade or downgrade. Libraries are preserved only if consumers of those libraries are detected. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index d78dea91a..12ed4a818 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -4,6 +4,7 @@ # $Id$ import array +import codecs from collections import deque import fcntl import formatter @@ -1467,9 +1468,15 @@ def get_mask_info(root_config, cpv, pkgsettings, if metadata is None: mreasons = ["corruption"] else: - pkg = Package(type_name=pkg_type, root_config=root_config, - cpv=cpv, built=built, installed=installed, metadata=metadata) - mreasons = get_masking_status(pkg, pkgsettings, root_config) + eapi = metadata['EAPI'] + if eapi[:1] == '-': + eapi = eapi[1:] + if not portage.eapi_is_supported(eapi): + mreasons = ['EAPI %s' % eapi] + else: + pkg = Package(type_name=pkg_type, root_config=root_config, + cpv=cpv, built=built, installed=installed, metadata=metadata) + mreasons = get_masking_status(pkg, pkgsettings, root_config) return metadata, mreasons def show_masked_packages(masked_packages): @@ -3054,8 +3061,21 @@ class EbuildMetadataPhase(SubProcess): def _start(self): settings = self.settings - settings.reset() + settings.setcpv(self.cpv) ebuild_path = self.ebuild_path + + if 'parse-eapi-ebuild-head' in settings.features: + eapi = portage._parse_eapi_ebuild_head(codecs.open(ebuild_path, + mode='r', encoding='utf_8', errors='replace')) + if not portage.eapi_is_supported(eapi): + self.metadata_callback(self.cpv, self.ebuild_path, + self.repo_path, {'EAPI' : eapi}, self.ebuild_mtime) + self.returncode = os.EX_OK + self.wait() + return + + settings.configdict['pkg']['EAPI'] = eapi + debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 1d5c0b482..96c887192 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1383,9 +1383,9 @@ class config(object): self.pusemask_list = [] rawpusemask = [grabdict_package(os.path.join(x, "package.use.mask"), recursive=1) for x in self.profiles] - for i in xrange(len(self.profiles)): + for pusemaskdict in rawpusemask: cpdict = {} - for k, v in rawpusemask[i].iteritems(): + for k, v in pusemaskdict.iteritems(): cpdict.setdefault(dep_getkey(k), {})[k] = v self.pusemask_list.append(cpdict) del rawpusemask @@ -1393,9 +1393,9 @@ class config(object): self.pkgprofileuse = [] rawprofileuse = [grabdict_package(os.path.join(x, "package.use"), juststrings=True, recursive=1) for x in self.profiles] - for i in xrange(len(self.profiles)): + for rawpusedict in rawprofileuse: cpdict = {} - for k, v in rawprofileuse[i].iteritems(): + for k, v in rawpusedict.iteritems(): cpdict.setdefault(dep_getkey(k), {})[k] = v self.pkgprofileuse.append(cpdict) del rawprofileuse @@ -1409,9 +1409,9 @@ class config(object): rawpuseforce = [grabdict_package( os.path.join(x, "package.use.force"), recursive=1) \ for x in self.profiles] - for i in xrange(len(self.profiles)): + for rawpusefdict in rawpuseforce: cpdict = {} - for k, v in rawpuseforce[i].iteritems(): + for k, v in rawpusefdict.iteritems(): cpdict.setdefault(dep_getkey(k), {})[k] = v self.puseforce_list.append(cpdict) del rawpuseforce @@ -1801,6 +1801,9 @@ class config(object): self["FEATURES"] = " ".join(sorted(self.features)) self.backup_changes("FEATURES") + global _validate_cache_for_unsupported_eapis + if 'parse-eapi-ebuild-head' in self.features: + _validate_cache_for_unsupported_eapis = False # inject EPREFIX as it's in no single config file (I hope), # but needs to be available using portageq @@ -2144,10 +2147,11 @@ class config(object): if pkginternaluse != self.configdict["pkginternal"].get("USE", ""): self.configdict["pkginternal"]["USE"] = pkginternaluse has_changed = True + defaults = [] pos = 0 - for i in xrange(len(self.profiles)): - cpdict = self.pkgprofileuse[i].get(cp, None) + for i, pkgprofileuse_dict in enumerate(self.pkgprofileuse): + cpdict = pkgprofileuse_dict.get(cp) if cpdict: keys = cpdict.keys() while keys: @@ -2315,8 +2319,8 @@ class config(object): cp = dep_getkey(pkg) usemask = [] pos = 0 - for i in xrange(len(self.profiles)): - cpdict = self.pusemask_list[i].get(cp, None) + for i, pusemask_dict in enumerate(self.pusemask_list): + cpdict = pusemask_dict.get(cp) if cpdict: keys = cpdict.keys() while keys: @@ -2338,8 +2342,8 @@ class config(object): cp = dep_getkey(pkg) useforce = [] pos = 0 - for i in xrange(len(self.profiles)): - cpdict = self.puseforce_list[i].get(cp, None) + for i, puseforce_dict in enumerate(self.puseforce_list): + cpdict = puseforce_dict.get(cp) if cpdict: keys = cpdict.keys() while keys: @@ -2414,8 +2418,8 @@ class config(object): pkg = "%s:%s" % (cpv, metadata["SLOT"]) keywords = [[x for x in metadata["KEYWORDS"].split() if x != "-*"]] pos = len(keywords) - for i in xrange(len(self.profiles)): - cpdict = self._pkeywords_list[i].get(cp, None) + for pkeywords_dict in self._pkeywords_list: + cpdict = pkeywords_dict.get(cp) if cpdict: keys = list(cpdict) while keys: @@ -5070,6 +5074,30 @@ def eapi_is_supported(eapi): return True +# Generally, it's best not to assume that cache entries for unsupported EAPIs +# can be validated. However, the current package manager specification does not +# guarantee that that the EAPI can be parsed without sourcing the ebuild, so +# it's too costly to discard existing cache entries for unsupported EAPIs. +# Therefore, by default, assume that cache entries for unsupported EAPIs can be +# validated. If FEATURES=parse-eapi-* is enabled, this assumption is discarded +# since the EAPI can be determined without the incurring the cost of sourcing +# the ebuild. +_validate_cache_for_unsupported_eapis = True + +_parse_eapi_ebuild_head_re = re.compile(r'^EAPI=[\'"]?([^\'"]*)') +_parse_eapi_ebuild_head_max_lines = 30 + +def _parse_eapi_ebuild_head(f): + count = 0 + for line in f: + m = _parse_eapi_ebuild_head_re.match(line) + if m is not None: + return m.group(1).strip() + count += 1 + if count >= _parse_eapi_ebuild_head_max_lines: + break + return '0' + def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, mydbapi): ebuild_path = os.path.abspath(myebuild) @@ -5091,14 +5119,14 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m tmpdir = mysettings["PORTAGE_TMPDIR"] if mycpv != mysettings.mycpv: - # Reload env.d variables and reset any previous settings. - mysettings.reload() - mysettings.reset() if mydo == 'depend': # Don't pass in mydbapi here since the resulting aux_get # call would lead to infinite 'depend' phase recursion. mysettings.setcpv(mycpv) else: + # Reload env.d variables and reset any previous settings. + mysettings.reload() + mysettings.reset() mysettings.setcpv(mycpv, mydb=mydbapi) # config.reset() might have reverted a change made by the caller, @@ -5144,6 +5172,15 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m if portage.util.noiselimit < 0: mysettings["PORTAGE_QUIET"] = "1" + if mydo == 'depend' and \ + 'EAPI' not in mysettings.configdict['pkg'] and \ + 'parse-eapi-ebuild-head' in mysettings.features: + eapi = _parse_eapi_ebuild_head(codecs.open(ebuild_path, + mode='r', encoding='utf_8', errors='replace')) + if not eapi_is_supported(eapi): + raise portage.exception.UnsupportedAPIException(mycpv, eapi) + mysettings.configdict['pkg']['EAPI'] = eapi + if mydo != "depend": # Metadata vars such as EAPI and RESTRICT are # set by the above config.setcpv() call. @@ -5576,7 +5613,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, for example. @type fd_pipes: Dictionary @param returnpid: Return a list of process IDs for a successful spawn, or - in integer value if spawn is unsuccessful. NOTE: This requires the + an integer value if spawn is unsuccessful. NOTE: This requires the caller clean up all returned PIDs. @type returnpid: Boolean @rtype: Boolean diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index 6588b3cc7..8d94d6a7c 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -165,7 +165,7 @@ class database(fs_template.FsBased): d[k] = result[0][column_index] # As a temporary workaround for bug #263081, convert unicode strings - # to raw byte strings. This helps to potential unicode errors that can + # to raw byte strings. This avoids potential unicode errors that can # be triggered elsewhere when attempting to join unicode strings with # raw byte strings. TODO: Convert all string handling code to use # unicode strings instead of raw byte strings (required for py3k diff --git a/pym/portage/cache/util.py b/pym/portage/cache/util.py index 9fcd4b49b..0f2685c0a 100644 --- a/pym/portage/cache/util.py +++ b/pym/portage/cache/util.py @@ -10,7 +10,8 @@ from portage.cache import cache_errors def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, verbose_instance=None): - from portage import eapi_is_supported + from portage import eapi_is_supported, \ + _validate_cache_for_unsupported_eapis if not src_cache.complete_eclass_entries and not eclass_cache: raise Exception("eclass_cache required for cache's of class %s!" % src_cache.__class__) @@ -39,6 +40,17 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, noise.exception(x, ce) del ce continue + + eapi = entry.get('EAPI') + if not eapi: + eapi = '0' + eapi = eapi.lstrip('-') + eapi_supported = eapi_is_supported(eapi) + if not eapi_supported: + if not _validate_cache_for_unsupported_eapis: + noise.misc(x, "unable to validate cache for EAPI='%s'" % eapi) + continue + write_it = True trg = None try: @@ -102,13 +114,10 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, continue entry["_eclasses_"] = eclasses - eapi = entry.get("EAPI") - if not eapi: - eapi = "0" - if not eapi_is_supported(eapi): + if not eapi_supported: for k in set(entry).difference(("_mtime_", "_eclasses_")): entry[k] = "" - entry["EAPI"] = "-" + eapi.lstrip("-") + entry["EAPI"] = "-" + eapi # by this time, if it reaches here, the eclass has been validated, and the entry has # been updated/translated (if needs be, for metadata/cache mainly) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 93730e9c3..4cbf7af30 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -25,7 +25,7 @@ from portage import eclass_cache, auxdbkeys, doebuild, flatten, \ listdir, dep_expand, eapi_is_supported, key_expand, dep_check, \ _eapi_is_deprecated -import os, stat +import codecs, os, stat from itertools import izip def _src_uri_validate(cpv, eapi, src_uri): @@ -417,14 +417,24 @@ class portdbapi(dbapi): writemsg("doregen: %s %s\n" % (doregen, mycpv), 2) writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n", 1) - self.doebuild_settings.reset() + self.doebuild_settings.setcpv(mycpv) mydata = {} - myret = doebuild(myebuild, "depend", - self.doebuild_settings["ROOT"], self.doebuild_settings, - dbkey=mydata, tree="porttree", mydbapi=self) - if myret != os.EX_OK: - self._broken_ebuilds.add(myebuild) - raise KeyError(mycpv) + eapi = None + + if 'parse-eapi-ebuild-head' in self.doebuild_settings.features: + eapi = portage._parse_eapi_ebuild_head(codecs.open(myebuild, + mode='r', encoding='utf_8', errors='replace')) + self.doebuild_settings.configdict['pkg']['EAPI'] = eapi + + if eapi is not None and not portage.eapi_is_supported(eapi): + mydata['EAPI'] = eapi + else: + myret = doebuild(myebuild, "depend", + self.doebuild_settings["ROOT"], self.doebuild_settings, + dbkey=mydata, tree="porttree", mydbapi=self) + if myret != os.EX_OK: + self._broken_ebuilds.add(myebuild) + raise KeyError(mycpv) self._metadata_callback( mycpv, myebuild, mylocation, mydata, emtime)