From a7f28692acdd06bcf7e926b8f7056baff06bc427 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sun, 8 Jun 2008 11:27:54 +0000 Subject: [PATCH] Merged from trunk 10585:10604 | 10586 | Add vardbapi.aux_get() support for an "_mtime_" value which | | zmedico | corresponds to the installed package directory's mtime | | | (numeric value from stat result). This value can be used to | | | validate indexes or caches used in the optimization of vdb | | | query operations for various types of package metadata such | | | as dependencies and file contents. | | 10587 | Add Package.counter and mtime attributes. | | zmedico | | | 10588 | Add a PackageVirtualDbapi.__getitem__() method. | | zmedico | | | 10589 | Bug #224271 - Add a 'IUSE is not defined' ebuild.minorsyn | | zmedico | warning. | | 10591 | remove old/unused gen_wrapper function | | SpankMan | | | 10592 | As suggested by Flameeyes, add a new 'inherit.autotools' | | zmedico | check which warns when autotools has been inherited but | | | none of the eautomake, eautoconf or eautoreconf functions | | | are called like they are supposed to be. | | 10593 | Bug #224271 - New check 'IUSE.undefined', renamed from | | zmedico | generic 'ebuild.minorsyn'. | | 10594 | Make load_emerge_config() filter out empty | | zmedico | PORTAGE_CONFIGROOT and ROOT values, and those containing | | | only whitespace. | | 10596 | Validate ROOT from make.conf. | | zmedico | | | 10598 | Inside, create_trees(), pass target_root="/" into the | | zmedico | config constructor when necessary to override the setting | | | in make.conf. | | 10600 | Make repoman pass Package instances into run_checks(), so | | zmedico | that the checks can use the Package.mtime and inherited | | | attributes. | | 10601 | Fix typo. | | zmedico | | | 10602 | Add additional functions such as eaclocal and eautoheader | | zmedico | to the inherit.autotools regex. Thanks to Arfrever. | | 10603 | Make portdbapi.aux_get() return float type for _mtime_. | | zmedico | | | 10604 | Bug #225285 - Allow digestcheck() to succeed when the | | zmedico | Manifest is missing and strict mode is disabled. | svn path=/main/branches/prefix/; revision=10605 --- bin/ebuild.sh | 9 ------- bin/isolated-functions.sh | 2 +- bin/repoman | 27 +++++++++++++++----- man/repoman.1 | 6 +++++ pym/_emerge/__init__.py | 48 ++++++++++++++++++++++++++++++++--- pym/portage/__init__.py | 6 ++++- pym/portage/dbapi/porttree.py | 5 +++- pym/portage/dbapi/vartree.py | 30 ++++++++++++++-------- pym/repoman/checks.py | 25 ++++++++++++++++-- 9 files changed, 122 insertions(+), 36 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index eb7bc58fb..1e9ff29df 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -621,15 +621,6 @@ src_install() { return } -# Used to generate the /lib/cpp and /usr/bin/cc wrappers -gen_wrapper() { - cat > "$1" <<-EOF - #!${EPREFIX}/bin/sh - exec $2 "\$@" - EOF - chmod 0755 "$1" -} - ebuild_phase() { [ "$(type -t ${1})" == "function" ] && qa_call ${1} } diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index ddb8e040e..e23af3cb6 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -467,7 +467,7 @@ save_ebuild_env() { hasv hasq qa_source qa_call addread addwrite adddeny addpredict \ lchown lchgrp esyslog use usev useq has_version portageq \ best_version use_with use_enable register_die_hook check_KV \ - keepdir unpack strip_duplicate_slashes econf einstall gen_wrapper \ + keepdir unpack strip_duplicate_slashes econf einstall \ dyn_setup dyn_unpack dyn_clean into insinto exeinto docinto \ insopts diropts exeopts libopts abort_handler abort_compile \ abort_test abort_install dyn_compile dyn_test dyn_install \ diff --git a/bin/repoman b/bin/repoman index 27efd4ae7..f015ab548 100755 --- a/bin/repoman +++ b/bin/repoman @@ -54,6 +54,9 @@ del os.environ["PORTAGE_LEGACY_GLOBALS"] from repoman.checks import run_checks from repoman import utilities +from _emerge import Package, RootConfig +from portage.sets import load_default_config + import portage.checksum import portage.const import portage.dep @@ -253,6 +256,7 @@ qahelp={ "file.size":"Files in the files directory must be under 20k", "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars, "file.UTF8":"File is not UTF8 compliant", + "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf", "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass", "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch", "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable", @@ -290,6 +294,7 @@ qahelp={ "variable.readonly":"Assigning a readonly variable", "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.", "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file", + "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)", "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.", "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found", "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.", @@ -327,12 +332,14 @@ qawarnings=[ "KEYWORDS.dropped", "KEYWORDS.stupid", "KEYWORDS.missing", +"IUSE.undefined", "RDEPEND.suspect", "RESTRICT.invalid", "ebuild.minorsyn", "ebuild.badheader", "ebuild.patches", "file.size", +"inherit.autotools", "java.eclassesnotused", "metadata.missing", "metadata.bad", @@ -344,7 +351,9 @@ qawarnings=[ ] missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"] -allvars=portage.auxdbkeys +allvars = set(portage.auxdbkeys) +allvars.update(Package.metadata_keys) +allvars = sorted(allvars) commitmessage=None for x in missingvars: x += ".missing" @@ -506,6 +515,8 @@ trees = portage.create_trees() trees["/"]["porttree"].settings = repoman_settings portdb = trees["/"]["porttree"].dbapi portdb.mysettings = repoman_settings +setconfig = load_default_config(repoman_settings, trees["/"]) +root_config = RootConfig(repoman_settings, trees["/"], setconfig) # We really only need to cache the metadata that's necessary for visibility # filtering. Anything else can be discarded to reduce memory consumption. for k in ("DEPEND", "LICENCE", "PDEPEND", @@ -774,7 +785,7 @@ for x in scanlist: checkdirlist=os.listdir(checkdir) ebuildlist=[] - ebuild_metadata = {} + pkgs = {} for y in checkdirlist: if y in no_exec and \ stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0111: @@ -798,7 +809,8 @@ for x in scanlist: stats["EAPI.unsupported"] += 1 fails["EAPI.unsupported"].append(os.path.join(x, y)) continue - ebuild_metadata[pf] = myaux + pkgs[pf] = Package(cpv=cpv, metadata=myaux, + root_config=root_config) # Sort ebuilds in ascending order for the KEYWORDS.dropped check. pkgsplits = {} @@ -813,7 +825,7 @@ for x in scanlist: slot_keywords = {} - if len(ebuild_metadata) != len(ebuildlist): + if len(pkgs) != len(ebuildlist): # If we can't access all the metadata then it's totally unsafe to # commit since there's no way to generate a correct Manifest. # Do not try to do any more QA checks on this package since missing @@ -1069,9 +1081,10 @@ for x in scanlist: fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild") continue - myaux = ebuild_metadata[y] + pkg = pkgs[y] + myaux = pkg.metadata eapi = myaux["EAPI"] - inherited = myaux["INHERITED"].split() + inherited = pkg.inherited # Test for negative logic and bad words in the RESTRICT var. #for x in myaux[allvars.index("RESTRICT")].split(): @@ -1379,7 +1392,7 @@ for x in scanlist: full_path = os.path.join(repodir, relative_path) f = open(full_path, 'rb') try: - for check_name, e in run_checks(f, os.stat(full_path).st_mtime): + for check_name, e in run_checks(f, pkg): stats[check_name] += 1 fails[check_name].append(relative_path + ': %s' % e) finally: diff --git a/man/repoman.1 b/man/repoman.1 index a9e240140..7f03365c2 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -114,6 +114,9 @@ Ebuilds that have a missing or empty HOMEPAGE variable .B IUSE.invalid This build has a variable in IUSE that is not in the use.desc or use.local.desc file .TP +.B IUSE.undefined +This ebuild does not define IUSE (style guideline says to define IUSE even when empty) +.TP .B KEYWORDS.dropped Ebuilds that appear to have dropped KEYWORDS for some arch .TP @@ -244,6 +247,9 @@ Files in the files directory must be under 20k .B filedir.missing Package lacks a files directory .TP +.B inherit.autotools +Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf +.TP .B java.eclassesnotused With virtual/jdk in DEPEND you must inherit a java eclass. Refer to \fIhttp://www.gentoo.org/proj/en/java/java\-devel.xml\fR for more information. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 773de639f..13eb25267 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1301,13 +1301,14 @@ class Package(Task): __slots__ = ("built", "cpv", "depth", "installed", "metadata", "onlydeps", "operation", "root_config", "type_name", - "category", "cp", "cpv_split", "iuse", + "category", "counter", "cp", "cpv_split", + "inherited", "iuse", "mtime", "pf", "pv_split", "root", "slot", "slot_atom", "use") metadata_keys = [ "CHOST", "COUNTER", "DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", - "repository", "RESTRICT", "SLOT", "USE"] + "repository", "RESTRICT", "SLOT", "USE", "_mtime_"] def __init__(self, **kwargs): Task.__init__(self, **kwargs) @@ -1363,7 +1364,8 @@ class Package(Task): """ Detect metadata updates and synchronize Package attributes. """ - _wrapped_keys = frozenset(["IUSE", "SLOT", "USE"]) + _wrapped_keys = frozenset( + ["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"]) def __init__(self, pkg, metadata): dict.__init__(self) @@ -1381,6 +1383,11 @@ class Package(Task): if k in self._wrapped_keys: getattr(self, "_set_" + k.lower())(k, v) + def _set_inherited(self, k, v): + if isinstance(v, basestring): + v = frozenset(v.split()) + self._pkg.inherited = v + def _set_iuse(self, k, v): self._pkg.iuse = self._pkg._iuse( v.split(), self._pkg.root_config.iuse_implicit) @@ -1391,6 +1398,22 @@ class Package(Task): def _set_use(self, k, v): self._pkg.use = self._pkg._use(v.split()) + def _set_counter(self, k, v): + if isinstance(v, basestring): + try: + v = int(v.strip()) + except ValueError: + v = 0 + self._pkg.counter = v + + def _set__mtime_(self, k, v): + if isinstance(v, basestring): + try: + v = float(v.strip()) + except ValueError: + v = 0 + self._pkg.mtime = v + def _get_hash_key(self): hash_key = getattr(self, "_hash_key", None) if hash_key is None: @@ -1798,6 +1821,21 @@ class PackageVirtualDbapi(portage.dbapi): return True return False + def __getitem__(self, k): + cpv = getattr(k, "cpv", None) + if cpv is None: + try: + cpv = k[2] + except (TypeError, IndexError): + raise KeyError(k) + if cpv is None: + raise KeyError(k) + existing = self._cpv_map.get(cpv) + if existing is not None and \ + existing == k: + return existing + raise KeyError(k) + def match_pkgs(self, atom): return [self._cpv_map[cpv] for cpv in self.match(atom)] @@ -8900,7 +8938,9 @@ def validate_ebuild_environment(trees): def load_emerge_config(trees=None): kwargs = {} for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")): - kwargs[k] = os.environ.get(envvar, None) + v = os.environ.get(envvar, None) + if v and v.strip(): + kwargs[k] = v trees = portage.create_trees(trees=trees, **kwargs) for root, root_trees in trees.iteritems(): diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index a498c748c..7616be6d8 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1286,6 +1286,8 @@ class config(object): # by the constructor argument (from the calling environment). if target_root is None and "ROOT" in make_conf: target_root = make_conf["ROOT"] + if not target_root.strip(): + target_root = None if target_root is None: target_root = "/" @@ -4060,6 +4062,8 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0): noiselevel=-1) if strict: return 0 + else: + return 1 mf = Manifest(pkgdir, mysettings["DISTDIR"]) eout = portage.output.EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" @@ -6841,7 +6845,7 @@ def create_trees(config_root=None, target_root=None, trees=None): myroots = [(settings["ROOT"], settings)] if settings["ROOT"] != "/": - settings = config(config_root=None, target_root=None, + settings = config(config_root=None, target_root="/", config_incrementals=portage.const.INCREMENTALS) # When ROOT != "/" we only want overrides from the calling # environment to apply to the config that's associated diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 83069bd2c..d6d5657c3 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -283,7 +283,8 @@ class portdbapi(dbapi): try: - emtime = os.stat(myebuild)[stat.ST_MTIME] + st = os.stat(myebuild) + emtime = st[stat.ST_MTIME] except OSError: writemsg("!!! aux_get(): ebuild for '%(cpv)s' does not exist at:\n" % {"cpv":mycpv}, noiselevel=-1) @@ -379,6 +380,8 @@ class portdbapi(dbapi): for x in mylist: if x == "INHERITED": returnme.append(' '.join(mydata.get("_eclasses_", []))) + elif x == "_mtime_": + returnme.append(st.st_mtime) else: returnme.append(mydata.get(x,"")) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index dd6cba471..b461a3f11 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -735,7 +735,7 @@ class vardbapi(dbapi): mydir_mtime = long(mydir_stat.st_mtime) pkg_data = self._aux_cache["packages"].get(mycpv) pull_me = cache_these.union(wants) - mydata = {} + mydata = {"_mtime_" : mydir_mtime} cache_valid = False cache_incomplete = False cache_mtime = None @@ -759,7 +759,8 @@ class vardbapi(dbapi): if pull_me: # pull any needed data and cache it aux_keys = list(pull_me) - for k, v in izip(aux_keys, self._aux_get(mycpv, aux_keys)): + for k, v in izip(aux_keys, + self._aux_get(mycpv, aux_keys, st=mydir_stat)): mydata[k] = v if not cache_valid or cache_these.difference(metadata): cache_data = {} @@ -771,18 +772,25 @@ class vardbapi(dbapi): self._aux_cache["modified"].add(mycpv) return [mydata[x] for x in wants] - def _aux_get(self, mycpv, wants): + def _aux_get(self, mycpv, wants, st=None): mydir = self.getpath(mycpv) - try: - if not stat.S_ISDIR(os.stat(mydir).st_mode): - raise KeyError(mycpv) - except OSError, e: - if e.errno == errno.ENOENT: - raise KeyError(mycpv) - del e - raise + if st is None: + try: + st = os.stat(mydir) + except OSError, e: + if e.errno == errno.ENOENT: + raise KeyError(mycpv) + elif e.errno == PermissionDenied.errno: + raise PermissionDenied(mydir) + else: + raise + if not stat.S_ISDIR(st.st_mode): + raise KeyError(mycpv) results = [] for x in wants: + if x == "_mtime_": + results.append(st.st_mtime) + continue try: myf = open(os.path.join(mydir, x), "r") try: diff --git a/pym/repoman/checks.py b/pym/repoman/checks.py index 8f9fd1abc..e340a57f3 100644 --- a/pym/repoman/checks.py +++ b/pym/repoman/checks.py @@ -217,13 +217,34 @@ _constant_checks = tuple((c() for c in ( EbuildUselessCdS, EbuildNestedDie, EbuildPatches, EbuildQuotedA))) -def run_checks(contents, st_mtime): +_iuse_def_re = re.compile(r'^IUSE=.*') +_comment_re = re.compile(r'(^|\s*)#') +_autotools_funcs = ( + "eaclocal", "eautoconf", "eautoheader", + "eautomake", "eautoreconf", "_elibtoolize") +_autotools_func_re = re.compile(r'(^|\s)(' + \ + "|".join(_autotools_funcs) + ')(\s|$)') + +def run_checks(contents, pkg): checks = list(_constant_checks) - checks.append(EbuildHeader(st_mtime)) + checks.append(EbuildHeader(pkg.mtime)) + iuse_def = None + inherit_autotools = "autotools" in pkg.inherited + autotools_func_call = None for num, line in enumerate(contents): + comment = _comment_re.match(line) + if comment is None: + if inherit_autotools and autotools_func_call is None: + autotools_func_call = _autotools_func_re.search(line) + if iuse_def is None: + iuse_def = _iuse_def_re.match(line) for lc in checks: ignore = lc.ignore_line if not ignore or not ignore.match(line): e = lc.check(num, line) if e: yield lc.repoman_check_name, e % (num + 1) + if iuse_def is None: + yield 'IUSE.undefined', 'IUSE is not defined' + if inherit_autotools and autotools_func_call is None: + yield 'inherit.autotools', 'no eauto* function called' -- 2.26.2