-#!/usr/bin/python -O
-# Copyright 1999-2013 Gentoo Foundation
+#!/usr/bin/python -bbO
+# Copyright 1999-2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# Next to do: dep syntax checking in mask files
import formatter
import io
import logging
-import optparse
import re
import signal
import stat
from portage import os
from portage import _encodings
from portage import _unicode_encode
+import repoman.checks
from repoman.checks import run_checks
from repoman import utilities
from repoman.herdbase import make_herd_base
green, nocolor, red
from portage.output import ConsoleStyleFile, StyleWriter
from portage.util import writemsg_level
+from portage.util._argparse import ArgumentParser
from portage.package.ebuild.digestgen import digestgen
from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
warn(txt)
sys.exit(1)
-def exithandler(signum=None, frame=None):
+def exithandler(signum=None, _frame=None):
logging.fatal("Interrupted; exiting...")
if signum is None:
sys.exit(1)
signal.signal(signal.SIGINT, exithandler)
-class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
- """Repoman needs its own HelpFormatter for now, because the default ones
- murder the help text."""
-
- def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
- optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
-
- def format_description(self, description):
- return description
-
-class RepomanOptionParser(optparse.OptionParser):
- """Add the on_tail function, ruby has it, optionParser should too
- """
-
- def __init__(self, *args, **kwargs):
- optparse.OptionParser.__init__(self, *args, **kwargs)
- self.tail = ""
-
- def on_tail(self, description):
- self.tail += description
-
- def format_help(self, formatter=None):
- result = optparse.OptionParser.format_help(self, formatter)
- result += self.tail
- return result
-
-
def ParseArgs(argv, qahelp):
"""This function uses a customized optionParser to parse command line arguments for repoman
Args:
mode_keys = list(modes)
mode_keys.sort()
- parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
- parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
- parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
- parser.description += "\nDistributed under the terms of the GNU General Public License v2"
- parser.description += "\nmodes: " + " | ".join(map(green, mode_keys))
+ parser = ArgumentParser(usage="repoman [options] [mode]",
+ description="Modes: %s" % " | ".join(mode_keys),
+ epilog="For more help consult the man page.")
- parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
+ parser.add_argument('-a', '--ask', dest='ask', action='store_true', default=False,
help='Request a confirmation before commiting')
- parser.add_option('-m', '--commitmsg', dest='commitmsg',
+ parser.add_argument('-m', '--commitmsg', dest='commitmsg',
help='specify a commit message on the command line')
- parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
+ parser.add_argument('-M', '--commitmsgfile', dest='commitmsgfile',
help='specify a path to a file that contains a commit message')
- parser.add_option('--digest',
- type='choice', choices=('y', 'n'), metavar='<y|n>',
+ parser.add_argument('--digest',
+ choices=('y', 'n'), metavar='<y|n>',
help='Automatically update Manifest digests for modified files')
- parser.add_option('-p', '--pretend', dest='pretend', default=False,
+ parser.add_argument('-p', '--pretend', dest='pretend', default=False,
action='store_true', help='don\'t commit or fix anything; just show what would be done')
- parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
+ parser.add_argument('-q', '--quiet', dest="quiet", action="count", default=0,
help='do not print unnecessary messages')
- parser.add_option(
- '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
+ parser.add_argument(
+ '--echangelog', choices=('y', 'n', 'force'), metavar="<y|n|force>",
help='for commit mode, call echangelog if ChangeLog is unmodified (or '
'regardless of modification if \'force\' is specified)')
- parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
+ parser.add_argument('--experimental-inherit', choices=('y', 'n'),
+ metavar="<y|n>", default='n',
+ help='Enable experimental inherit.missing checks which may misbehave'
+ ' when the internal eclass database becomes outdated')
+
+ parser.add_argument('-f', '--force', dest='force', default=False, action='store_true',
help='Commit with QA violations')
- parser.add_option('--vcs', dest='vcs',
+ parser.add_argument('--vcs', dest='vcs',
help='Force using specific VCS instead of autodetection')
- parser.add_option('-v', '--verbose', dest="verbosity", action='count',
+ parser.add_argument('-v', '--verbose', dest="verbosity", action='count',
help='be very verbose in output', default=0)
- parser.add_option('-V', '--version', dest='version', action='store_true',
+ parser.add_argument('-V', '--version', dest='version', action='store_true',
help='show version info')
- parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
+ parser.add_argument('-x', '--xmlparse', dest='xml_parse', action='store_true',
default=False, help='forces the metadata.xml parse check to be carried out')
- parser.add_option(
- '--if-modified', type='choice', choices=('y', 'n'), default='n',
+ parser.add_argument(
+ '--if-modified', choices=('y', 'n'), default='n',
metavar="<y|n>",
help='only check packages that have uncommitted modifications')
- parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
+ parser.add_argument('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
default=False, help='ignore arch-specific failures (where arch != host)')
- parser.add_option("--ignore-default-opts",
+ parser.add_argument("--ignore-default-opts",
action="store_true",
help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
- parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
+ parser.add_argument('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
default=False, help='ignore masked packages (not allowed with commit mode)')
- parser.add_option('--include-arches', dest='include_arches',
+ parser.add_argument('--include-arches', dest='include_arches',
metavar='ARCHES', action='append',
help='A space separated list of arches used to '
'filter the selection of profiles for dependency checks')
- parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
+ parser.add_argument('-d', '--include-dev', dest='include_dev', action='store_true',
default=False, help='include dev profiles in dependency checks')
- parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
+ parser.add_argument('-e', '--include-exp-profiles', choices=('y', 'n'),
+ default=False, help='include exp profiles in dependency checks',
+ metavar='<y|n>')
+
+ parser.add_argument('--unmatched-removal', dest='unmatched_removal', action='store_true',
default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
- parser.add_option('--without-mask', dest='without_mask', action='store_true',
+ parser.add_argument('--without-mask', dest='without_mask', action='store_true',
default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
- parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
+ parser.add_argument('--mode', dest='mode', choices=mode_keys,
help='specify which mode repoman will run in (default=full)')
- parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
-
- for k in mode_keys:
- parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
-
- parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
-
- sorted_qa = list(qahelp)
- sorted_qa.sort()
- for k in sorted_qa:
- parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
-
- opts, args = parser.parse_args(argv[1:])
+ opts, args = parser.parse_known_args(argv[1:])
if not opts.ignore_default_opts:
default_opts = portage.util.shlex_split(
repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
if default_opts:
- opts, args = parser.parse_args(default_opts + sys.argv[1:])
+ opts, args = parser.parse_known_args(default_opts + sys.argv[1:])
if opts.mode == 'help':
parser.print_help(short=False)
"LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
"IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
"IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
+ "IUSE.rubydeprecated": "The ebuild has set a ruby interpreter in USE_RUBY, that is not available as a ruby target anymore",
"LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
"LICENSE.deprecated": "This ebuild is listing a deprecated license.",
"KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
"upstream.workaround",
"LIVEVCS.stable",
"LIVEVCS.unmasked",
+"IUSE.rubydeprecated",
))
-if portage.const._ENABLE_INHERIT_CHECK:
- # This is experimental, so it's non-fatal.
- qawarnings.add("inherit.missing")
-
non_ascii_re = re.compile(r'[^\x00-\x7f]')
missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
"fetch", "installsources", "mirror", "preserve-libs",
"primaryuri", "splitdebug", "strip", "test", "userpriv"])
-live_eclasses = frozenset([
- "bzr",
- "cvs",
- "darcs",
- "git",
- "git-2",
- "mercurial",
- "subversion",
- "tla",
-])
+live_eclasses = portage.const.LIVE_ECLASSES
suspect_rdepend = frozenset([
"app-arch/cabextract",
"dev-libs/libusb-compat":"virtual/libusb",
}
+ruby_deprecated = frozenset([
+ "ruby_targets_ree18",
+])
+
metadata_xml_encoding = 'UTF-8'
metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
(metadata_xml_encoding,)
print("Portage", portage.VERSION)
sys.exit(0)
+if options.experimental_inherit == 'y':
+ # This is experimental, so it's non-fatal.
+ qawarnings.add("inherit.missing")
+ repoman.checks._init(experimental_inherit=True)
+
# Set this to False when an extraordinary issue (generally
# something other than a QA issue) makes it impossible to
# commit (like if Manifest generation fails).
# Pass GNUPGHOME to git for bug #462362.
commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
+ # Pass GPG_TTY to git for bug #477728.
+ try:
+ commit_env["GPG_TTY"] = os.ttyname(sys.stdin.fileno())
+ except OSError:
+ pass
+
# In order to disable manifest signatures, repos may set
# "sign-manifests = false" in metadata/layout.conf. This
# can be used to prevent merge conflicts like those that
chain(mychanged, mynew, myremoved)))
for x in effective_scanlist:
- #ebuilds and digests added to cvs respectively.
+ # ebuilds and digests added to cvs respectively.
logging.info("checking package %s" % x)
# save memory by discarding xmatch caches from previous package(s)
arch_xmatch_caches.clear()
allvalid = True
for y in checkdirlist:
if (y in no_exec or y.endswith(".ebuild")) and \
- stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
- stats["file.executable"] += 1
- fails["file.executable"].append(os.path.join(checkdir, y))
+ stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
+ stats["file.executable"] += 1
+ fails["file.executable"].append(os.path.join(checkdir, y))
if y.endswith(".ebuild"):
pf = y[:-7]
ebuildlist.append(pf)
fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
del e
- #Only carry out if in package directory or check forced
+ # Only carry out if in package directory or check forced
if xmllint_capable and not metadata_bad:
# xmlint can produce garbage output even on success, so only dump
# the ouput when it fails.
fails['changelog.ebuildadded'].append(relative_path)
if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
- #ebuild not added to vcs
+ # ebuild not added to vcs
stats["ebuild.notadded"] += 1
fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
myesplit = portage.pkgsplit(y)
stats["IUSE.invalid"] += 1
fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
+ # Check for outdated RUBY targets
+ if "ruby-ng" in inherited or "ruby-fakegem" in inherited or "ruby" in inherited:
+ ruby_intersection = pkg.iuse.all.intersection(ruby_deprecated)
+ if ruby_intersection:
+ for myruby in ruby_intersection:
+ stats["IUSE.rubydeprecated"] += 1
+ fails["IUSE.rubydeprecated"].append(
+ (relative_path + ": Deprecated ruby target: %s") % myruby)
+
# license checks
if not badlicsyntax:
# Parse the LICENSE variable, remove USE conditions and
stats["LICENSE.deprecated"] += 1
fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
- #keyword checks
+ # keyword checks
myuse = myaux["KEYWORDS"].split()
for mykey in myuse:
if mykey not in ("-*", "*", "~*"):
stats["KEYWORDS.invalid"] += 1
fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
- #restrict checks
+ # restrict checks
myrestrict = None
try:
myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
stats["RESTRICT.invalid"] += len(mybadrestrict)
for mybad in mybadrestrict:
fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
- #REQUIRED_USE check
+ # REQUIRED_USE check
required_use = myaux["REQUIRED_USE"]
if required_use:
if not eapi_has_required_use(eapi):
for keyword, groups, prof in relevant_profiles:
- if prof.status not in ("stable", "dev") or \
- prof.status == "dev" and not options.include_dev:
- continue
+ if not (prof.status == "stable" or \
+ (prof.status == "dev" and options.include_dev) or \
+ (prof.status == "exp" and options.include_exp_profiles == 'y')):
+ continue
- dep_settings = arch_caches.get(prof.sub_path)
- if dep_settings is None:
- dep_settings = portage.config(
- config_profile_path=prof.abs_path,
- config_incrementals=repoman_incrementals,
- config_root=config_root,
- local_config=False,
- _unmatched_removal=options.unmatched_removal,
- env=env, repositories=repoman_settings.repositories)
- dep_settings.categories = repoman_settings.categories
- if options.without_mask:
- dep_settings._mask_manager_obj = \
- copy.deepcopy(dep_settings._mask_manager)
- dep_settings._mask_manager._pmaskdict.clear()
- arch_caches[prof.sub_path] = dep_settings
-
- xmatch_cache_key = (prof.sub_path, tuple(groups))
- xcache = arch_xmatch_caches.get(xmatch_cache_key)
- if xcache is None:
- portdb.melt()
- portdb.freeze()
- xcache = portdb.xcache
- xcache.update(shared_xmatch_caches)
- arch_xmatch_caches[xmatch_cache_key] = xcache
-
- trees[root]["porttree"].settings = dep_settings
- portdb.settings = dep_settings
- portdb.xcache = xcache
-
- dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
- # just in case, prevent config.reset() from nuking these.
- dep_settings.backup_changes("ACCEPT_KEYWORDS")
-
- # This attribute is used in dbapi._match_use() to apply
- # use.stable.{mask,force} settings based on the stable
- # status of the parent package. This is required in order
- # for USE deps of unstable packages to be resolved correctly,
- # since otherwise use.stable.{mask,force} settings of
- # dependencies may conflict (see bug #456342).
- dep_settings._parent_stable = dep_settings._isStable(pkg)
-
- # Handle package.use*.{force,mask) calculation, for use
- # in dep_check.
- dep_settings.useforce = dep_settings._use_manager.getUseForce(
- pkg, stable=dep_settings._parent_stable)
- dep_settings.usemask = dep_settings._use_manager.getUseMask(
- pkg, stable=dep_settings._parent_stable)
-
- if not baddepsyntax:
- ismasked = not ebuild_archs or \
- pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
- if ismasked:
- if not have_pmasked:
- have_pmasked = bool(dep_settings._getMaskAtom(
- pkg.cpv, pkg._metadata))
- if options.ignore_masked:
- continue
- #we are testing deps for a masked package; give it some lee-way
- suffix = "masked"
- matchmode = "minimum-all"
- else:
- suffix = ""
- matchmode = "minimum-visible"
-
- if not have_dev_keywords:
- have_dev_keywords = \
- bool(dev_keywords.intersection(keywords))
-
- if prof.status == "dev":
- suffix = suffix + "indev"
-
- for mytype in Package._dep_keys:
-
- mykey = "dependency.bad" + suffix
- myvalue = myaux[mytype]
- if not myvalue:
- continue
-
- success, atoms = portage.dep_check(myvalue, portdb,
- dep_settings, use="all", mode=matchmode,
- trees=trees)
-
- if success:
- if atoms:
-
- # Don't bother with dependency.unknown for
- # cases in which *DEPEND.bad is triggered.
- for atom in atoms:
- # dep_check returns all blockers and they
- # aren't counted for *DEPEND.bad, so we
- # ignore them here.
- if not atom.blocker:
- unknown_pkgs.discard(
- (mytype, atom.unevaluated_atom))
-
- if not prof.sub_path:
- # old-style virtuals currently aren't
- # resolvable with empty profile, since
- # 'virtuals' mappings are unavailable
- # (it would be expensive to search
- # for PROVIDE in all ebuilds)
- atoms = [atom for atom in atoms if not \
- (atom.cp.startswith('virtual/') and \
- not portdb.cp_list(atom.cp))]
-
- #we have some unsolvable deps
- #remove ! deps, which always show up as unsatisfiable
- atoms = [str(atom.unevaluated_atom) \
- for atom in atoms if not atom.blocker]
-
- #if we emptied out our list, continue:
- if not atoms:
- continue
- stats[mykey] += 1
- fails[mykey].append("%s: %s: %s(%s) %s" % \
- (relative_path, mytype, keyword,
- prof, repr(atoms)))
- else:
+ dep_settings = arch_caches.get(prof.sub_path)
+ if dep_settings is None:
+ dep_settings = portage.config(
+ config_profile_path=prof.abs_path,
+ config_incrementals=repoman_incrementals,
+ config_root=config_root,
+ local_config=False,
+ _unmatched_removal=options.unmatched_removal,
+ env=env, repositories=repoman_settings.repositories)
+ dep_settings.categories = repoman_settings.categories
+ if options.without_mask:
+ dep_settings._mask_manager_obj = \
+ copy.deepcopy(dep_settings._mask_manager)
+ dep_settings._mask_manager._pmaskdict.clear()
+ arch_caches[prof.sub_path] = dep_settings
+
+ xmatch_cache_key = (prof.sub_path, tuple(groups))
+ xcache = arch_xmatch_caches.get(xmatch_cache_key)
+ if xcache is None:
+ portdb.melt()
+ portdb.freeze()
+ xcache = portdb.xcache
+ xcache.update(shared_xmatch_caches)
+ arch_xmatch_caches[xmatch_cache_key] = xcache
+
+ trees[root]["porttree"].settings = dep_settings
+ portdb.settings = dep_settings
+ portdb.xcache = xcache
+
+ dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
+ # just in case, prevent config.reset() from nuking these.
+ dep_settings.backup_changes("ACCEPT_KEYWORDS")
+
+ # This attribute is used in dbapi._match_use() to apply
+ # use.stable.{mask,force} settings based on the stable
+ # status of the parent package. This is required in order
+ # for USE deps of unstable packages to be resolved correctly,
+ # since otherwise use.stable.{mask,force} settings of
+ # dependencies may conflict (see bug #456342).
+ dep_settings._parent_stable = dep_settings._isStable(pkg)
+
+ # Handle package.use*.{force,mask) calculation, for use
+ # in dep_check.
+ dep_settings.useforce = dep_settings._use_manager.getUseForce(
+ pkg, stable=dep_settings._parent_stable)
+ dep_settings.usemask = dep_settings._use_manager.getUseMask(
+ pkg, stable=dep_settings._parent_stable)
+
+ if not baddepsyntax:
+ ismasked = not ebuild_archs or \
+ pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
+ if ismasked:
+ if not have_pmasked:
+ have_pmasked = bool(dep_settings._getMaskAtom(
+ pkg.cpv, pkg._metadata))
+ if options.ignore_masked:
+ continue
+ # we are testing deps for a masked package; give it some lee-way
+ suffix = "masked"
+ matchmode = "minimum-all"
+ else:
+ suffix = ""
+ matchmode = "minimum-visible"
+
+ if not have_dev_keywords:
+ have_dev_keywords = \
+ bool(dev_keywords.intersection(keywords))
+
+ if prof.status == "dev":
+ suffix = suffix + "indev"
+
+ for mytype in Package._dep_keys:
+
+ mykey = "dependency.bad" + suffix
+ myvalue = myaux[mytype]
+ if not myvalue:
+ continue
+
+ success, atoms = portage.dep_check(myvalue, portdb,
+ dep_settings, use="all", mode=matchmode,
+ trees=trees)
+
+ if success:
+ if atoms:
+
+ # Don't bother with dependency.unknown for
+ # cases in which *DEPEND.bad is triggered.
+ for atom in atoms:
+ # dep_check returns all blockers and they
+ # aren't counted for *DEPEND.bad, so we
+ # ignore them here.
+ if not atom.blocker:
+ unknown_pkgs.discard(
+ (mytype, atom.unevaluated_atom))
+
+ if not prof.sub_path:
+ # old-style virtuals currently aren't
+ # resolvable with empty profile, since
+ # 'virtuals' mappings are unavailable
+ # (it would be expensive to search
+ # for PROVIDE in all ebuilds)
+ atoms = [atom for atom in atoms if not \
+ (atom.cp.startswith('virtual/') and \
+ not portdb.cp_list(atom.cp))]
+
+ # we have some unsolvable deps
+ # remove ! deps, which always show up as unsatisfiable
+ atoms = [str(atom.unevaluated_atom) \
+ for atom in atoms if not atom.blocker]
+
+ # if we emptied out our list, continue:
+ if not atoms:
+ continue
stats[mykey] += 1
fails[mykey].append("%s: %s: %s(%s) %s" % \
(relative_path, mytype, keyword,
prof, repr(atoms)))
+ else:
+ stats[mykey] += 1
+ fails[mykey].append("%s: %s: %s(%s) %s" % \
+ (relative_path, mytype, keyword,
+ prof, repr(atoms)))
if not baddepsyntax and unknown_pkgs:
type_map = {}
commitmessage += commit_footer
+ broken_changelog_manifests = []
if options.echangelog in ('y', 'force'):
logging.info("checking for unmodified ChangeLog files")
committer_name = utilities.get_committer_name(env=repoman_settings)
# regenerate Manifest for modified ChangeLog (bug #420735)
repoman_settings["O"] = checkdir
digestgen(mysettings=repoman_settings, myportdb=portdb)
+ else:
+ broken_changelog_manifests.append(x)
if myautoadd:
print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
repoman_settings["O"] = os.path.join(repodir, x)
digestgen(mysettings=repoman_settings, myportdb=portdb)
+ elif broken_changelog_manifests:
+ for x in broken_changelog_manifests:
+ repoman_settings["O"] = os.path.join(repodir, x)
+ digestgen(mysettings=repoman_settings, myportdb=portdb)
+
signed = False
if sign_manifests:
signed = True