From 20817801dd4ef0117bcc7b33c90650da1e920385 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 8 May 2012 23:38:47 -0700 Subject: [PATCH] Parse EAPI with pattern from PMS section 7.3.1. This implements the specification that was approved in Gentoo's council meeting on May 8, 2012 (see bug #402167). The parse-eapi-ebuild-head FEATURES setting is now enabled by default, and causes non-conformant ebuilds to be treated as invalid. This behavior will soon become enabled unconditionally. --- bin/ebuild.sh | 6 +- bin/repoman | 2 +- cnf/make.globals | 2 +- man/make.conf.5 | 6 +- man/repoman.1 | 2 +- pym/_emerge/EbuildMetadataPhase.py | 112 +++++++++++++++++++------ pym/_emerge/actions.py | 8 +- pym/portage/__init__.py | 34 ++++---- pym/portage/dbapi/porttree.py | 42 +++------- pym/portage/package/ebuild/config.py | 3 - pym/portage/package/ebuild/doebuild.py | 6 +- pym/repoman/checks.py | 7 +- 12 files changed, 132 insertions(+), 98 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 32dc64f0c..d161bc65d 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -534,7 +534,7 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then # In order to ensure correct interaction between ebuilds and # eclasses, they need to be unset before this process of # interaction begins. - unset DEPEND RDEPEND PDEPEND INHERITED IUSE REQUIRED_USE \ + unset EAPI DEPEND RDEPEND PDEPEND INHERITED IUSE REQUIRED_USE \ ECLASS E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND if [[ $PORTAGE_DEBUG != 1 || ${-/x/} != $- ]] ; then @@ -551,7 +551,7 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then rm "$PORTAGE_BUILDDIR/.ebuild_changed" fi - [[ -n $EAPI ]] || EAPI=0 + [ "${EAPI+set}" = set ] || EAPI=0 if has "$EAPI" 0 1 2 3 3_pre2 ; then export RDEPEND=${RDEPEND-${DEPEND}} @@ -670,7 +670,7 @@ if [[ $EBUILD_PHASE = depend ]] ; then PROPERTIES DEFINED_PHASES UNUSED_05 UNUSED_04 UNUSED_03 UNUSED_02 UNUSED_01" - [ -n "${EAPI}" ] || EAPI=0 + [ "${EAPI+set}" = set ] || EAPI=0 # The extra $(echo) commands remove newlines. if [ -n "${dbkey}" ] ; then diff --git a/bin/repoman b/bin/repoman index cfe4b8e01..3d1a4b0c1 100755 --- a/bin/repoman +++ b/bin/repoman @@ -322,7 +322,7 @@ qahelp={ "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable", "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable", "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len, - "EAPI.definition":"EAPI definition does not conform to PMS section 8.3.1 (first non-comment, non-blank line)", + "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)", "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI", "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI", "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)", diff --git a/cnf/make.globals b/cnf/make.globals index 975726ab6..1717baf68 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -52,7 +52,7 @@ FETCHCOMMAND_SFTP="bash -c \"x=\\\${2#sftp://} ; host=\\\${x%%/*} ; port=\\\${ho # Default user options FEATURES="assume-digests binpkg-logs distlocks ebuild-locks - fixlafiles news parallel-fetch protect-owned + fixlafiles news parallel-fetch parse-eapi-ebuild-head protect-owned sandbox sfperms strict unknown-features-warn unmerge-logs unmerge-orphans userfetch" diff --git a/man/make.conf.5 b/man/make.conf.5 index b7108945c..9897073c6 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -407,9 +407,9 @@ parallelization. For additional parallelization, disable \fIebuild\-locks\fR. .TP .B parse\-eapi\-ebuild\-head -Parse \fBEAPI\fR from the head of the ebuild (first 30 lines). This feature -is only intended for experimental purposes and should not be enabled under -normal circumstances. +Parse \fBEAPI\fR from the head of the ebuild as specified in PMS section +7.3.1, and treat non\-conformant ebuilds as invalid. This feature is +enabled by default, and will soon become enabled unconditionally. .TP .B prelink\-checksums If \fBprelink\fR(8) is installed then use it to undo any prelinks on files diff --git a/man/repoman.1 b/man/repoman.1 index 0e0715c9d..4305ce7d7 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -146,7 +146,7 @@ Syntax error in DEPEND (usually an extra/missing space/parenthesis) Ebuilds that have a missing or empty DESCRIPTION variable .TP .B EAPI.definition -EAPI definition does not conform to PMS section 8.3.1 (first +EAPI definition does not conform to PMS section 7.3.1 (first non\-comment, non\-blank line) .TP .B EAPI.deprecated diff --git a/pym/_emerge/EbuildMetadataPhase.py b/pym/_emerge/EbuildMetadataPhase.py index a34542d43..f89077ed1 100644 --- a/pym/_emerge/EbuildMetadataPhase.py +++ b/pym/_emerge/EbuildMetadataPhase.py @@ -9,9 +9,13 @@ from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode +from portage.dep import _repo_separator +from portage.elog import elog_process +from portage.elog.messages import eerror import errno import fcntl import io +import textwrap class EbuildMetadataPhase(SubProcess): @@ -20,9 +24,9 @@ class EbuildMetadataPhase(SubProcess): used to extract metadata from the ebuild. """ - __slots__ = ("cpv", "eapi", "ebuild_hash", "fd_pipes", "metadata_callback", - "metadata", "portdb", "repo_path", "settings") + \ - ("_raw_metadata",) + __slots__ = ("cpv", "ebuild_hash", "fd_pipes", + "metadata_callback", "metadata", "portdb", "repo_path", "settings") + \ + ("_eapi", "_eapi_lineno", "_raw_metadata",) _file_names = ("ebuild",) _files_dict = slot_dict_class(_file_names, prefix="") @@ -33,26 +37,31 @@ class EbuildMetadataPhase(SubProcess): settings.setcpv(self.cpv) ebuild_path = self.ebuild_hash.location - # the caller can pass in eapi in order to avoid - # redundant _parse_eapi_ebuild_head calls - eapi = self.eapi - if eapi is None and \ - 'parse-eapi-ebuild-head' in settings.features: - with io.open(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - eapi = portage._parse_eapi_ebuild_head(f) - - if eapi is not None: - if not portage.eapi_is_supported(eapi): - self.metadata = self.metadata_callback(self.cpv, - self.repo_path, {'EAPI' : eapi}, self.ebuild_hash) - self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() - return - - settings.configdict['pkg']['EAPI'] = eapi + with io.open(_unicode_encode(ebuild_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + self._eapi, self._eapi_lineno = portage._parse_eapi_ebuild_head(f) + + parsed_eapi = self._eapi + if parsed_eapi is None: + parsed_eapi = "0" + + if not parsed_eapi: + # An empty EAPI setting is invalid. + self._eapi_invalid(None) + self._set_returncode((self.pid, 1 << 8)) + self.wait() + return + + if not portage.eapi_is_supported(parsed_eapi): + self.metadata = self.metadata_callback(self.cpv, + self.repo_path, {'EAPI' : parsed_eapi}, self.ebuild_hash) + self._set_returncode((self.pid, os.EX_OK << 8)) + self.wait() + return + + settings.configdict['pkg']['EAPI'] = parsed_eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None @@ -144,7 +153,58 @@ class EbuildMetadataPhase(SubProcess): # number of lines is incorrect. self.returncode = 1 else: - metadata = zip(portage.auxdbkeys, metadata_lines) - self.metadata = self.metadata_callback(self.cpv, - self.repo_path, metadata, self.ebuild_hash) + metadata_valid = True + metadata = dict(zip(portage.auxdbkeys, metadata_lines)) + parsed_eapi = self._eapi + if parsed_eapi is None: + parsed_eapi = "0" + if portage.eapi_is_supported(metadata["EAPI"]) and \ + metadata["EAPI"] != parsed_eapi: + self._eapi_invalid(metadata) + if 'parse-eapi-ebuild-head' in self.settings.features: + metadata_valid = False + + if metadata_valid: + self.metadata = self.metadata_callback(self.cpv, + self.repo_path, metadata, self.ebuild_hash) + else: + self.returncode = 1 + + def _eapi_invalid(self, metadata): + + repo_name = self.portdb.getRepositoryName(self.repo_path) + msg = [] + msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" + " conform with PMS section 7.3.1:") % + (self.cpv, _repo_separator, repo_name), 70)) + + if not self._eapi: + # None means the assignment was not found, while an + # empty string indicates an (invalid) empty assingment. + msg.append( + "\tvalid EAPI assignment must" + " occur on or before line: %s" % + self._eapi_lineno) + else: + msg.append(("\tbash returned EAPI '%s' which does not match " + "assignment on line: %s") % + (metadata["EAPI"], self._eapi_lineno)) + + if 'parse-eapi-ebuild-head' in self.settings.features: + msg.extend(textwrap.wrap(("NOTE: This error will soon" + " become unconditionally fatal in a future version of Portage," + " but at this time, it can by made non-fatal by setting" + " FEATURES=-parse-eapi-ebuild-head in" + " make.conf."), 70)) + else: + msg.extend(textwrap.wrap(("NOTE: This error will soon" + " become unconditionally fatal in a future version of Portage." + " At the earliest opportunity, please enable" + " FEATURES=parse-eapi-ebuild-head in make.conf in order to" + " make this error fatal."), 70)) + + for line in msg: + eerror(line, phase="other", key=self.cpv) + elog_process(self.cpv, self.settings, + phasefilter=("other",)) diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 9c8712050..62f3ff79d 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -28,7 +28,7 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage.localization import _ from portage import os from portage import shutil -from portage import _unicode_decode +from portage import eapi_is_supported, _unicode_decode from portage.cache.cache_errors import CacheError from portage.const import GLOBAL_CONFIG_PATH from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_SET_CONFIG @@ -1716,9 +1716,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None): if onProgress is not None: onProgress(maxval, curval) - from portage import eapi_is_supported, \ - _validate_cache_for_unsupported_eapis - # TODO: Display error messages, but do not interfere with the progress bar. # Here's how: # 1) erase the progress bar @@ -1758,8 +1755,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None): eapi = eapi.lstrip('-') eapi_supported = eapi_is_supported(eapi) if not eapi_supported: - if not _validate_cache_for_unsupported_eapis: - continue + continue dest = None try: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 3495b96ec..31d580742 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -435,29 +435,25 @@ def eapi_is_supported(eapi): return False return eapi <= portage.const.EAPI -# Generally, it's best not to assume that cache entries for unsupported EAPIs -# can be validated. However, the current package manager specification does not -# guarantee that the EAPI can be parsed without sourcing the ebuild, so -# it's too costly to discard existing cache entries for unsupported EAPIs. -# Therefore, by default, assume that cache entries for unsupported EAPIs can be -# validated. If FEATURES=parse-eapi-* is enabled, this assumption is discarded -# since the EAPI can be determined without the incurring the cost of sourcing -# the ebuild. -_validate_cache_for_unsupported_eapis = True - -_parse_eapi_ebuild_head_re = re.compile(r'^EAPI=[\'"]?([^\'"#]*)') -_parse_eapi_ebuild_head_max_lines = 30 +# This pattern is specified by PMS section 7.3.1. +_pms_eapi_re = re.compile(r"^[ \t]*EAPI=(['\"]?)([A-Za-z0-9+_.-]*)\1[ \t]*(#.*)?$") +_comment_or_blank_line = re.compile(r"^\s*(#.*)?$") def _parse_eapi_ebuild_head(f): - count = 0 + eapi = None + eapi_lineno = None + lineno = 0 for line in f: - m = _parse_eapi_ebuild_head_re.match(line) - if m is not None: - return m.group(1).strip() - count += 1 - if count >= _parse_eapi_ebuild_head_max_lines: + lineno += 1 + m = _comment_or_blank_line.match(line) + if m is None: + eapi_lineno = lineno + m = _pms_eapi_re.match(line) + if m is not None: + eapi = m.group(2) break - return '0' + + return (eapi, eapi_lineno) def _movefile(src, dest, **kwargs): """Calls movefile and raises a PortageException if an error occurs.""" diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index bdb87f13e..16962b566 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -485,38 +485,22 @@ class portdbapi(dbapi): raise KeyError(mycpv) self.doebuild_settings.setcpv(mycpv) - eapi = None - - if eapi is None and \ - 'parse-eapi-ebuild-head' in self.doebuild_settings.features: - with io.open(_unicode_encode(myebuild, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - eapi = portage._parse_eapi_ebuild_head(f) - - if eapi is not None: - self.doebuild_settings.configdict['pkg']['EAPI'] = eapi - - if eapi is not None and not portage.eapi_is_supported(eapi): - mydata = self._metadata_callback( - mycpv, mylocation, {'EAPI':eapi}, ebuild_hash) - else: - proc = EbuildMetadataPhase(cpv=mycpv, eapi=eapi, - ebuild_hash=ebuild_hash, - metadata_callback=self._metadata_callback, portdb=self, - repo_path=mylocation, - scheduler=PollScheduler().sched_iface, - settings=self.doebuild_settings) - proc.start() - proc.wait() + proc = EbuildMetadataPhase(cpv=mycpv, + ebuild_hash=ebuild_hash, + metadata_callback=self._metadata_callback, portdb=self, + repo_path=mylocation, + scheduler=PollScheduler().sched_iface, + settings=self.doebuild_settings) + + proc.start() + proc.wait() - if proc.returncode != os.EX_OK: - self._broken_ebuilds.add(myebuild) - raise KeyError(mycpv) + if proc.returncode != os.EX_OK: + self._broken_ebuilds.add(myebuild) + raise KeyError(mycpv) - mydata = proc.metadata + mydata = proc.metadata mydata["repository"] = self.repositories.get_name_for_location(mylocation) mydata["_mtime_"] = ebuild_hash.mtime diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index 38f15c081..32d65367e 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -776,9 +776,6 @@ class config(object): if bsd_chflags: self.features.add('chflags') - if 'parse-eapi-ebuild-head' in self.features: - portage._validate_cache_for_unsupported_eapis = False - self._iuse_implicit_match = _iuse_implicit_match_cache(self) self._validate_commands() diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index da30bda38..4f7d4a8a1 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -337,12 +337,14 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, # when uninstalling a package that has corrupt EAPI metadata. eapi = None if mydo == 'depend' and 'EAPI' not in mysettings.configdict['pkg']: - if eapi is None and 'parse-eapi-ebuild-head' in mysettings.features: + if eapi is None: with io.open(_unicode_encode(ebuild_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace') as f: - eapi = _parse_eapi_ebuild_head(f) + eapi, eapi_lineno = _parse_eapi_ebuild_head(f) + if eapi is None: + eapi = "0" if eapi is not None: if not eapi_is_supported(eapi): diff --git a/pym/repoman/checks.py b/pym/repoman/checks.py index 733bbc3c1..77df603a2 100644 --- a/pym/repoman/checks.py +++ b/pym/repoman/checks.py @@ -8,6 +8,7 @@ and correctness of an ebuild.""" import re import time import repoman.errors as errors +import portage from portage.eapi import eapi_supports_prefix, eapi_has_implicit_rdepend, \ eapi_has_src_prepare_and_src_configure, eapi_has_dosed_dohard, \ eapi_exports_AA, eapi_exports_KV @@ -284,14 +285,12 @@ class EbuildUselessCdS(LineCheck): class EapiDefinition(LineCheck): """ - Check that EAPI assignment conforms to PMS section 8.3.1 + Check that EAPI assignment conforms to PMS section 7.3.1 (first non-comment, non-blank line). """ repoman_check_name = 'EAPI.definition' ignore_comment = True - - # This pattern is specified by PMS section 8.3.1. - _eapi_re = re.compile(r"^[ \t]*EAPI=(['\"]?)([A-Za-z0-9+_.-]*)\1[ \t]*(#.*)?$") + _eapi_re = portage._pms_eapi_re def new(self, pkg): self._cached_eapi = pkg.metadata['EAPI'] -- 2.26.2