bin/repoman

   1 #!/usr/bin/python -bbO
   2 # Copyright 1999-2014 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import codecs
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import re
  18 import signal
  19 import stat
  20 import subprocess
  21 import sys
  22 import tempfile
  23 import textwrap
  24 import time
  25 import platform
  26 from itertools import chain
  27 from stat import S_ISDIR
  28
  29 try:
  30         from urllib.parse import urlparse
  31 except ImportError:
  32         from urlparse import urlparse
  33
  34 from os import path as osp
  35 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  36 sys.path.insert(0, pym_path)
  37 import portage
  38 portage._internal_caller = True
  39 portage._disable_legacy_globals()
  40
  41 try:
  42         import xml.etree.ElementTree
  43         from xml.parsers.expat import ExpatError
  44 except (SystemExit, KeyboardInterrupt):
  45         raise
  46 except (ImportError, SystemError, RuntimeError, Exception):
  47         # broken or missing xml support
  48         # http://bugs.python.org/issue14988
  49         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  50         from portage.output import EOutput
  51         out = EOutput()
  52         for line in msg:
  53                 out.eerror(line)
  54         sys.exit(1)
  55
  56 from portage import os
  57 from portage import _encodings
  58 from portage import _unicode_encode
  59 import repoman.checks
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 import portage.repository.config
  69 from portage import cvstree, normalize_path
  70 from portage import util
  71 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  72         ParseError, PermissionDenied)
  73 from portage.dep import Atom
  74 from portage.process import find_binary, spawn
  75 from portage.output import bold, create_color_func, \
  76         green, nocolor, red
  77 from portage.output import ConsoleStyleFile, StyleWriter
  78 from portage.util import writemsg_level
  79 from portage.util._argparse import ArgumentParser
  80 from portage.package.ebuild.digestgen import digestgen
  81 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  82
  83 if sys.hexversion >= 0x3000000:
  84         basestring = str
  85
  86 util.initialize_logger()
  87
  88 # 14 is the length of DESCRIPTION=""
  89 max_desc_len = 100
  90 allowed_filename_chars="a-zA-Z0-9._-+:"
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  93 bad = create_color_func("BAD")
  94
  95 # A sane umask is needed for files that portage creates.
  96 os.umask(0o22)
  97 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  98 # behave incrementally.
  99 repoman_incrementals = tuple(x for x in \
 100         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 101 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 102 repoman_settings = portage.config(config_root=config_root, local_config=False)
 103
 104 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 105         repoman_settings.get('TERM') == 'dumb' or \
 106         not sys.stdout.isatty():
 107         nocolor()
 108
 109 def warn(txt):
 110         print("repoman: " + txt)
 111
 112 def err(txt):
 113         warn(txt)
 114         sys.exit(1)
 115
 116 def exithandler(signum=None, _frame=None):
 117         logging.fatal("Interrupted; exiting...")
 118         if signum is None:
 119                 sys.exit(1)
 120         else:
 121                 sys.exit(128 + signum)
 122
 123 signal.signal(signal.SIGINT, exithandler)
 124
 125 def ParseArgs(argv, qahelp):
 126         """This function uses a customized optionParser to parse command line arguments for repoman
 127         Args:
 128           argv - a sequence of command line arguments
 129                 qahelp - a dict of qa warning to help message
 130         Returns:
 131           (opts, args), just like a call to parser.parse_args()
 132         """
 133
 134         argv = portage._decode_argv(argv)
 135
 136         modes = {
 137                 'commit' : 'Run a scan then commit changes',
 138                 'ci' : 'Run a scan then commit changes',
 139                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 140                 'full' : 'Scan directory tree and print all issues (not a summary)',
 141                 'help' : 'Show this screen',
 142                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 143                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 144                 'scan' : 'Scan directory tree for QA issues'
 145         }
 146
 147         mode_keys = list(modes)
 148         mode_keys.sort()
 149
 150         parser = ArgumentParser(usage="repoman [options] [mode]",
 151                 description="Modes: %s" % " | ".join(mode_keys),
 152                 epilog="For more help consult the man page.")
 153
 154         parser.add_argument('-a', '--ask', dest='ask', action='store_true', default=False,
 155                 help='Request a confirmation before commiting')
 156
 157         parser.add_argument('-m', '--commitmsg', dest='commitmsg',
 158                 help='specify a commit message on the command line')
 159
 160         parser.add_argument('-M', '--commitmsgfile', dest='commitmsgfile',
 161                 help='specify a path to a file that contains a commit message')
 162
 163         parser.add_argument('--digest',
 164                 choices=('y', 'n'), metavar='<y|n>',
 165                 help='Automatically update Manifest digests for modified files')
 166
 167         parser.add_argument('-p', '--pretend', dest='pretend', default=False,
 168                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 169
 170         parser.add_argument('-q', '--quiet', dest="quiet", action="count", default=0,
 171                 help='do not print unnecessary messages')
 172
 173         parser.add_argument(
 174                 '--echangelog', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 175                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 176                 'regardless of modification if \'force\' is specified)')
 177
 178         parser.add_argument('--experimental-inherit', choices=('y', 'n'),
 179                 metavar="<y|n>", default='n',
 180                 help='Enable experimental inherit.missing checks which may misbehave'
 181                         ' when the internal eclass database becomes outdated')
 182
 183         parser.add_argument('-f', '--force', dest='force', default=False, action='store_true',
 184                 help='Commit with QA violations')
 185
 186         parser.add_argument('--vcs', dest='vcs',
 187                 help='Force using specific VCS instead of autodetection')
 188
 189         parser.add_argument('-v', '--verbose', dest="verbosity", action='count',
 190                 help='be very verbose in output', default=0)
 191
 192         parser.add_argument('-V', '--version', dest='version', action='store_true',
 193                 help='show version info')
 194
 195         parser.add_argument('-x', '--xmlparse', dest='xml_parse', action='store_true',
 196                 default=False, help='forces the metadata.xml parse check to be carried out')
 197
 198         parser.add_argument(
 199                 '--if-modified', choices=('y', 'n'), default='n',
 200                 metavar="<y|n>",
 201                 help='only check packages that have uncommitted modifications')
 202
 203         parser.add_argument('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 204                 default=False, help='ignore arch-specific failures (where arch != host)')
 205
 206         parser.add_argument("--ignore-default-opts",
 207                 action="store_true",
 208                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 209
 210         parser.add_argument('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 211                 default=False, help='ignore masked packages (not allowed with commit mode)')
 212
 213         parser.add_argument('--include-arches', dest='include_arches',
 214                 metavar='ARCHES', action='append',
 215                 help='A space separated list of arches used to '
 216                 'filter the selection of profiles for dependency checks')
 217
 218         parser.add_argument('-d', '--include-dev', dest='include_dev', action='store_true',
 219                 default=False, help='include dev profiles in dependency checks')
 220
 221         parser.add_argument('-e', '--include-exp-profiles', choices=('y', 'n'),
 222                 default=False, help='include exp profiles in dependency checks',
 223                 metavar='<y|n>')
 224
 225         parser.add_argument('--unmatched-removal', dest='unmatched_removal', action='store_true',
 226                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 227
 228         parser.add_argument('--without-mask', dest='without_mask', action='store_true',
 229                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 230
 231         parser.add_argument('--mode', dest='mode', choices=mode_keys,
 232                 help='specify which mode repoman will run in (default=full)')
 233
 234         opts, args = parser.parse_known_args(argv[1:])
 235
 236         if not opts.ignore_default_opts:
 237                 default_opts = portage.util.shlex_split(
 238                         repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
 239                 if default_opts:
 240                         opts, args = parser.parse_known_args(default_opts + sys.argv[1:])
 241
 242         if opts.mode == 'help':
 243                 parser.print_help(short=False)
 244
 245         for arg in args:
 246                 if arg in modes:
 247                         if not opts.mode:
 248                                 opts.mode = arg
 249                                 break
 250                 else:
 251                         parser.error("invalid mode: %s" % arg)
 252
 253         if not opts.mode:
 254                 opts.mode = 'full'
 255
 256         if opts.mode == 'ci':
 257                 opts.mode = 'commit'  # backwards compat shortcut
 258
 259         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 260         for val in range(opts.verbosity):
 261                 logger = logging.getLogger()
 262                 logger.setLevel(logger.getEffectiveLevel() - 10)
 263
 264         for val in range(opts.quiet):
 265                 logger = logging.getLogger()
 266                 logger.setLevel(logger.getEffectiveLevel() + 10)
 267
 268         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 269                 if opts.ignore_masked:
 270                         opts.ignore_masked = False
 271                         logging.warn('Commit mode automatically disables --ignore-masked')
 272                 if opts.without_mask:
 273                         opts.without_mask = False
 274                         logging.warn('Commit mode automatically disables --without-mask')
 275
 276         return (opts, args)
 277
 278 qahelp = {
 279         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 280         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 281         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 282         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 283         "changelog.missing": "Missing ChangeLog files",
 284         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 285         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 286         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 287         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 288         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 289         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 290         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 291         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 292         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 293         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 294         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 295         "file.size": "Files in the files directory must be under 20 KiB",
 296         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 297         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 298         "file.UTF8": "File is not UTF8 compliant",
 299         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 300         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 301         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 302         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 303         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 304         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 305         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 306         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 307         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 308         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 309         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 310         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 311         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 312         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 313         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 314         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 315         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 316         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 317         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 318         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 319         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 320         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 321         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 322         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 323         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 324         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 325         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 326         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 327         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 328         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 329         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 330         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 331         "variable.readonly": "Assigning a readonly variable",
 332         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 333         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 334         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 335         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 336         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 337         "IUSE.rubydeprecated": "The ebuild has set a ruby interpreter in USE_RUBY, that is not available as a ruby target anymore",
 338         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 339         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 340         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 341         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 342         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 343         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 344         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 345         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 346         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 347         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 348         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 349         "ebuild.badheader": "This ebuild has a malformed header",
 350         "manifest.bad": "Manifest has missing or incorrect digests",
 351         "metadata.missing": "Missing metadata.xml files",
 352         "metadata.bad": "Bad metadata.xml files",
 353         "metadata.warning": "Warnings in metadata.xml files",
 354         "portage.internal": "The ebuild uses an internal Portage function or variable",
 355         "repo.eapi.banned": "The ebuild uses an EAPI which is banned by the repository's metadata/layout.conf settings",
 356         "repo.eapi.deprecated": "The ebuild uses an EAPI which is deprecated by the repository's metadata/layout.conf settings",
 357         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 358         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 359         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 360         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 361 }
 362
 363 qacats = list(qahelp)
 364 qacats.sort()
 365
 366 qawarnings = set((
 367 "changelog.missing",
 368 "changelog.notadded",
 369 "dependency.unknown",
 370 "digest.assumed",
 371 "digest.unused",
 372 "ebuild.notadded",
 373 "ebuild.nesteddie",
 374 "dependency.badmasked",
 375 "dependency.badindev",
 376 "dependency.badmaskedindev",
 377 "dependency.badtilde",
 378 "DESCRIPTION.toolong",
 379 "EAPI.deprecated",
 380 "HOMEPAGE.virtual",
 381 "LICENSE.deprecated",
 382 "LICENSE.virtual",
 383 "KEYWORDS.dropped",
 384 "KEYWORDS.stupid",
 385 "KEYWORDS.missing",
 386 "PDEPEND.suspect",
 387 "RDEPEND.implicit",
 388 "RDEPEND.suspect",
 389 "virtual.suspect",
 390 "RESTRICT.invalid",
 391 "ebuild.minorsyn",
 392 "ebuild.badheader",
 393 "ebuild.patches",
 394 "file.size",
 395 "inherit.unused",
 396 "inherit.deprecated",
 397 "java.eclassesnotused",
 398 "wxwidgets.eclassnotused",
 399 "metadata.warning",
 400 "portage.internal",
 401 "repo.eapi.deprecated",
 402 "usage.obsolete",
 403 "upstream.workaround",
 404 "LIVEVCS.stable",
 405 "LIVEVCS.unmasked",
 406 "IUSE.rubydeprecated",
 407 ))
 408
 409 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 410
 411 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 412 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 413 allvars.update(Package.metadata_keys)
 414 allvars = sorted(allvars)
 415 commitmessage = None
 416 for x in missingvars:
 417         x += ".missing"
 418         if x not in qacats:
 419                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 420                 qacats.append(x)
 421                 qawarnings.add(x)
 422
 423 valid_restrict = frozenset(["binchecks", "bindist",
 424         "fetch", "installsources", "mirror", "preserve-libs",
 425         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 426
 427 live_eclasses = portage.const.LIVE_ECLASSES
 428
 429 suspect_rdepend = frozenset([
 430         "app-arch/cabextract",
 431         "app-arch/rpm2targz",
 432         "app-doc/doxygen",
 433         "dev-lang/nasm",
 434         "dev-lang/swig",
 435         "dev-lang/yasm",
 436         "dev-perl/extutils-pkgconfig",
 437         "dev-util/byacc",
 438         "dev-util/cmake",
 439         "dev-util/ftjam",
 440         "dev-util/gperf",
 441         "dev-util/gtk-doc",
 442         "dev-util/gtk-doc-am",
 443         "dev-util/intltool",
 444         "dev-util/jam",
 445         "dev-util/pkg-config-lite",
 446         "dev-util/pkgconf",
 447         "dev-util/pkgconfig",
 448         "dev-util/pkgconfig-openbsd",
 449         "dev-util/scons",
 450         "dev-util/unifdef",
 451         "dev-util/yacc",
 452         "media-gfx/ebdftopcf",
 453         "sys-apps/help2man",
 454         "sys-devel/autoconf",
 455         "sys-devel/automake",
 456         "sys-devel/bin86",
 457         "sys-devel/bison",
 458         "sys-devel/dev86",
 459         "sys-devel/flex",
 460         "sys-devel/m4",
 461         "sys-devel/pmake",
 462         "virtual/linux-sources",
 463         "virtual/pkgconfig",
 464         "x11-misc/bdftopcf",
 465         "x11-misc/imake",
 466 ])
 467
 468 suspect_virtual = {
 469         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 470         "dev-util/pkgconf":"virtual/pkgconfig",
 471         "dev-util/pkgconfig":"virtual/pkgconfig",
 472         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 473         "dev-libs/libusb":"virtual/libusb",
 474         "dev-libs/libusbx":"virtual/libusb",
 475         "dev-libs/libusb-compat":"virtual/libusb",
 476 }
 477
 478 ruby_deprecated = frozenset([
 479         "ruby_targets_ree18",
 480 ])
 481
 482 metadata_xml_encoding = 'UTF-8'
 483 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 484         (metadata_xml_encoding,)
 485 metadata_doctype_name = 'pkgmetadata'
 486 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 487 # force refetch if the local copy creation time is older than this
 488 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 489
 490 # file.executable
 491 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 492
 493 options, arguments = ParseArgs(sys.argv, qahelp)
 494
 495 if options.version:
 496         print("Portage", portage.VERSION)
 497         sys.exit(0)
 498
 499 if options.experimental_inherit == 'y':
 500         # This is experimental, so it's non-fatal.
 501         qawarnings.add("inherit.missing")
 502         repoman.checks._init(experimental_inherit=True)
 503
 504 # Set this to False when an extraordinary issue (generally
 505 # something other than a QA issue) makes it impossible to
 506 # commit (like if Manifest generation fails).
 507 can_force = True
 508
 509 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 510 if portdir is None:
 511         sys.exit(1)
 512
 513 myreporoot = os.path.basename(portdir_overlay)
 514 myreporoot += mydir[len(portdir_overlay):]
 515
 516 if options.vcs:
 517         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 518                 vcs = options.vcs
 519         else:
 520                 vcs = None
 521 else:
 522         vcses = utilities.FindVCS()
 523         if len(vcses) > 1:
 524                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 525                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 526                 sys.exit(1)
 527         elif vcses:
 528                 vcs = vcses[0]
 529         else:
 530                 vcs = None
 531
 532 if options.if_modified == "y" and vcs is None:
 533         logging.info("Not in a version controlled repository; "
 534                 "disabling --if-modified.")
 535         options.if_modified = "n"
 536
 537 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 538 vcs_preserves_mtime = vcs in ('cvs',)
 539
 540 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 541 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 542 if vcs_global_opts is None:
 543         if vcs in ('cvs', 'svn'):
 544                 vcs_global_opts = "-q"
 545         else:
 546                 vcs_global_opts = ""
 547 vcs_global_opts = vcs_global_opts.split()
 548
 549 if options.mode == 'commit' and not options.pretend and not vcs:
 550         logging.info("Not in a version controlled repository; enabling pretend mode.")
 551         options.pretend = True
 552
 553 # Ensure that current repository is in the list of enabled repositories.
 554 repodir = os.path.realpath(portdir_overlay)
 555 try:
 556         repoman_settings.repositories.get_repo_for_location(repodir)
 557 except KeyError:
 558         repo_name = portage.repository.config.RepoConfig._read_valid_repo_name(portdir_overlay)[0]
 559         layout_conf_data = portage.repository.config.parse_layout_conf(portdir_overlay)[0]
 560         if layout_conf_data['repo-name']:
 561                 repo_name = layout_conf_data['repo-name']
 562         tmp_conf_file = io.StringIO(textwrap.dedent("""
 563                 [%s]
 564                 location = %s
 565                 """) % (repo_name, portdir_overlay))
 566         # Ensure that the repository corresponding to $PWD overrides a
 567         # repository of the same name referenced by the existing PORTDIR
 568         # or PORTDIR_OVERLAY settings.
 569         repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 570                 (repoman_settings.get('PORTDIR_OVERLAY', ''),
 571                 portage._shell_quote(portdir_overlay))
 572         repositories = portage.repository.config.load_repository_config(repoman_settings, extra_files=[tmp_conf_file])
 573         # We have to call the config constructor again so that attributes
 574         # dependent on config.repositories are initialized correctly.
 575         repoman_settings = portage.config(config_root=config_root, local_config=False, repositories=repositories)
 576
 577 root = repoman_settings['EROOT']
 578 trees = {
 579         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 580 }
 581 portdb = trees[root]['porttree'].dbapi
 582
 583 # Constrain dependency resolution to the master(s)
 584 # that are specified in layout.conf.
 585 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 586 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 587 portdir = portdb.porttrees[0]
 588 commit_env = os.environ.copy()
 589 # list() is for iteration on a copy.
 590 for repo in list(repoman_settings.repositories):
 591         # all paths are canonical
 592         if repo.location not in repo_config.eclass_db.porttrees:
 593                 del repoman_settings.repositories[repo.name]
 594
 595 if repo_config.allow_provide_virtual:
 596         qawarnings.add("virtual.oldstyle")
 597
 598 if repo_config.sign_commit:
 599         if vcs == 'git':
 600                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 601                 # the commit arguments. If key_id is unspecified, then it must be
 602                 # configured by `git config user.signingkey key_id`.
 603                 vcs_local_opts.append("--gpg-sign")
 604                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 605                         # Pass GNUPGHOME to git for bug #462362.
 606                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 607
 608                 # Pass GPG_TTY to git for bug #477728.
 609                 try:
 610                         commit_env["GPG_TTY"] = os.ttyname(sys.stdin.fileno())
 611                 except OSError:
 612                         pass
 613
 614 # In order to disable manifest signatures, repos may set
 615 # "sign-manifests = false" in metadata/layout.conf. This
 616 # can be used to prevent merge conflicts like those that
 617 # thin-manifests is designed to prevent.
 618 sign_manifests = "sign" in repoman_settings.features and \
 619         repo_config.sign_manifest
 620
 621 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 622         options.mode in ("commit",) and not sign_manifests:
 623         msg = ("The '%s' repository has manifest signatures enabled, "
 624         "but FEATURES=sign is currently disabled. In order to avoid this "
 625         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 626         "repositories can disable manifest signatures by setting "
 627         "'sign-manifests = false' in metadata/layout.conf.") % \
 628         (repo_config.name,)
 629         for line in textwrap.wrap(msg, 60):
 630                 logging.warn(line)
 631
 632 if sign_manifests and options.mode in ("commit",) and \
 633         repoman_settings.get("PORTAGE_GPG_KEY") and \
 634         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 635         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 636         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 637                 repoman_settings["PORTAGE_GPG_KEY"])
 638         sys.exit(1)
 639
 640 manifest_hashes = repo_config.manifest_hashes
 641 if manifest_hashes is None:
 642         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 643
 644 if options.mode in ("commit", "fix", "manifest"):
 645         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 646                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 647                 "metadata/layout.conf does not contain the '%s' hash which "
 648                 "is required by this portage version. You will have to "
 649                 "upgrade portage if you want to generate valid manifests for "
 650                 "this repository.") % \
 651                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 652                 for line in textwrap.wrap(msg, 70):
 653                         logging.error(line)
 654                 sys.exit(1)
 655
 656         unsupported_hashes = manifest_hashes.difference(
 657                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 658         if unsupported_hashes:
 659                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 660                 "metadata/layout.conf contains one or more hash types '%s' "
 661                 "which are not supported by this portage version. You will "
 662                 "have to upgrade portage if you want to generate valid "
 663                 "manifests for this repository.") % \
 664                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 665                 for line in textwrap.wrap(msg, 70):
 666                         logging.error(line)
 667                 sys.exit(1)
 668
 669 if options.echangelog is None and repo_config.update_changelog:
 670         options.echangelog = 'y'
 671
 672 if vcs is None:
 673         options.echangelog = 'n'
 674
 675 # The --echangelog option causes automatic ChangeLog generation,
 676 # which invalidates changelog.ebuildadded and changelog.missing
 677 # checks.
 678 # Note: Some don't use ChangeLogs in distributed SCMs.
 679 # It will be generated on server side from scm log,
 680 # before package moves to the rsync server.
 681 # This is needed because they try to avoid merge collisions.
 682 # Gentoo's Council decided to always use the ChangeLog file.
 683 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 684 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 685
 686 if 'digest' in repoman_settings.features and options.digest != 'n':
 687         options.digest = 'y'
 688
 689 logging.debug("vcs: %s" % (vcs,))
 690 logging.debug("repo config: %s" % (repo_config,))
 691 logging.debug("options: %s" % (options,))
 692
 693 # It's confusing if these warnings are displayed without the user
 694 # being told which profile they come from, so disable them.
 695 env = os.environ.copy()
 696 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 697
 698 categories = []
 699 for path in repo_config.eclass_db.porttrees:
 700         categories.extend(portage.util.grabfile(
 701                 os.path.join(path, 'profiles', 'categories')))
 702 repoman_settings.categories = frozenset(
 703         portage.util.stack_lists([categories], incremental=1))
 704 categories = repoman_settings.categories
 705
 706 portdb.settings = repoman_settings
 707 root_config = RootConfig(repoman_settings, trees[root], None)
 708 # We really only need to cache the metadata that's necessary for visibility
 709 # filtering. Anything else can be discarded to reduce memory consumption.
 710 portdb._aux_cache_keys.clear()
 711 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 712
 713 reposplit = myreporoot.split(os.path.sep)
 714 repolevel = len(reposplit)
 715
 716 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 717 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 718 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 719 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 720         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 721         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 722         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 723         print(red("***"))
 724         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 725
 726 # Make startdir relative to the canonical repodir, so that we can pass
 727 # it to digestgen and it won't have to be canonicalized again.
 728 if repolevel == 1:
 729         startdir = repodir
 730 else:
 731         startdir = normalize_path(mydir)
 732         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 733
 734 def caterror(mycat):
 735         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 736
 737 def repoman_getstatusoutput(cmd):
 738         """
 739         Implements an interface similar to getstatusoutput(), but with
 740         customized unicode handling (see bug #310789) and without the shell.
 741         """
 742         args = portage.util.shlex_split(cmd)
 743
 744         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 745                 not os.path.isabs(args[0]):
 746                 # Python 3.1 _execvp throws TypeError for non-absolute executable
 747                 # path passed as bytes (see http://bugs.python.org/issue8513).
 748                 fullname = find_binary(args[0])
 749                 if fullname is None:
 750                         raise portage.exception.CommandNotFound(args[0])
 751                 args[0] = fullname
 752
 753         encoding = _encodings['fs']
 754         args = [_unicode_encode(x,
 755                 encoding=encoding, errors='strict') for x in args]
 756         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 757                 stderr=subprocess.STDOUT)
 758         output = portage._unicode_decode(proc.communicate()[0],
 759                 encoding=encoding, errors='strict')
 760         if output and output[-1] == "\n":
 761                 # getstatusoutput strips one newline
 762                 output = output[:-1]
 763         return (proc.wait(), output)
 764
 765 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 766         """
 767         Implements an interface similar to os.popen(), but with customized
 768         unicode handling (see bug #310789) and without the shell.
 769         """
 770
 771         __slots__ = ('_proc', '_stdout')
 772
 773         def __init__(self, cmd):
 774                 args = portage.util.shlex_split(cmd)
 775
 776                 if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 777                         not os.path.isabs(args[0]):
 778                         # Python 3.1 _execvp throws TypeError for non-absolute executable
 779                         # path passed as bytes (see http://bugs.python.org/issue8513).
 780                         fullname = find_binary(args[0])
 781                         if fullname is None:
 782                                 raise portage.exception.CommandNotFound(args[0])
 783                         args[0] = fullname
 784
 785                 encoding = _encodings['fs']
 786                 args = [_unicode_encode(x,
 787                         encoding=encoding, errors='strict') for x in args]
 788                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 789                 object.__setattr__(self, '_proc', proc)
 790                 object.__setattr__(self, '_stdout',
 791                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 792
 793         def _get_target(self):
 794                 return object.__getattribute__(self, '_stdout')
 795
 796         __enter__ = _get_target
 797
 798         def __exit__(self, exc_type, exc_value, traceback):
 799                 proc = object.__getattribute__(self, '_proc')
 800                 proc.wait()
 801                 proc.stdout.close()
 802
 803 class ProfileDesc(object):
 804         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 805         def __init__(self, arch, status, sub_path, tree_path):
 806                 self.arch = arch
 807                 self.status = status
 808                 if sub_path:
 809                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 810                 self.sub_path = sub_path
 811                 self.tree_path = tree_path
 812                 if tree_path:
 813                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 814                 else:
 815                         self.abs_path = tree_path
 816
 817         def __str__(self):
 818                 if self.sub_path:
 819                         return self.sub_path
 820                 return 'empty profile'
 821
 822 profile_list = []
 823 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 824
 825 # get lists of valid keywords, licenses, and use
 826 kwlist = set()
 827 liclist = set()
 828 uselist = set()
 829 global_pmasklines = []
 830
 831 for path in portdb.porttrees:
 832         try:
 833                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 834         except OSError:
 835                 pass
 836         kwlist.update(portage.grabfile(os.path.join(path,
 837                 "profiles", "arch.list")))
 838
 839         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 840         for x in use_desc:
 841                 x = x.split()
 842                 if x:
 843                         uselist.add(x[0])
 844
 845         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 846         try:
 847                 expand_list = os.listdir(expand_desc_dir)
 848         except OSError:
 849                 pass
 850         else:
 851                 for fn in expand_list:
 852                         if not fn[-5:] == '.desc':
 853                                 continue
 854                         use_prefix = fn[:-5].lower() + '_'
 855                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 856                                 x = x.split()
 857                                 if x:
 858                                         uselist.add(use_prefix + x[0])
 859
 860         global_pmasklines.append(portage.util.grabfile_package(
 861                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 862
 863         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 864         try:
 865                 desc_file = io.open(_unicode_encode(desc_path,
 866                         encoding=_encodings['fs'], errors='strict'),
 867                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 868         except EnvironmentError:
 869                 pass
 870         else:
 871                 for i, x in enumerate(desc_file):
 872                         if x[0] == "#":
 873                                 continue
 874                         arch = x.split()
 875                         if len(arch) == 0:
 876                                 continue
 877                         if len(arch) != 3:
 878                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 879                                         desc_path + " line %d" % (i + 1, ))
 880                         elif arch[0] not in kwlist:
 881                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 882                                         desc_path + " line %d" % (i + 1, ))
 883                         elif arch[2] not in valid_profile_types:
 884                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 885                                         desc_path + " line %d" % (i + 1, ))
 886                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 887                         if not os.path.isdir(profile_desc.abs_path):
 888                                 logging.error(
 889                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 890                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 891                                 continue
 892                         if os.path.exists(
 893                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 894                                 continue
 895                         profile_list.append(profile_desc)
 896                 desc_file.close()
 897
 898 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 899 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 900
 901 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 902 global_pmaskdict = {}
 903 for x in global_pmasklines:
 904         global_pmaskdict.setdefault(x.cp, []).append(x)
 905 del global_pmasklines
 906
 907 def has_global_mask(pkg):
 908         mask_atoms = global_pmaskdict.get(pkg.cp)
 909         if mask_atoms:
 910                 pkg_list = [pkg]
 911                 for x in mask_atoms:
 912                         if portage.dep.match_from_list(x, pkg_list):
 913                                 return x
 914         return None
 915
 916 # Ensure that profile sub_path attributes are unique. Process in reverse order
 917 # so that profiles with duplicate sub_path from overlays will override
 918 # profiles with the same sub_path from parent repos.
 919 profiles = {}
 920 profile_list.reverse()
 921 profile_sub_paths = set()
 922 for prof in profile_list:
 923         if prof.sub_path in profile_sub_paths:
 924                 continue
 925         profile_sub_paths.add(prof.sub_path)
 926         profiles.setdefault(prof.arch, []).append(prof)
 927
 928 # Use an empty profile for checking dependencies of
 929 # packages that have empty KEYWORDS.
 930 prof = ProfileDesc('**', 'stable', '', '')
 931 profiles.setdefault(prof.arch, []).append(prof)
 932
 933 for x in repoman_settings.archlist():
 934         if x[0] == "~":
 935                 continue
 936         if x not in profiles:
 937                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 938                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 939                 print(red("up with the " + x + " team."))
 940                 print()
 941
 942 liclist_deprecated = set()
 943 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 944         liclist_deprecated.update(
 945                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 946
 947 if not liclist:
 948         logging.fatal("Couldn't find licenses?")
 949         sys.exit(1)
 950
 951 if not kwlist:
 952         logging.fatal("Couldn't read KEYWORDS from arch.list")
 953         sys.exit(1)
 954
 955 if not uselist:
 956         logging.fatal("Couldn't find use.desc?")
 957         sys.exit(1)
 958
 959 scanlist = []
 960 if repolevel == 2:
 961         # we are inside a category directory
 962         catdir = reposplit[-1]
 963         if catdir not in categories:
 964                 caterror(catdir)
 965         mydirlist = os.listdir(startdir)
 966         for x in mydirlist:
 967                 if x == "CVS" or x.startswith("."):
 968                         continue
 969                 if os.path.isdir(startdir + "/" + x):
 970                         scanlist.append(catdir + "/" + x)
 971         repo_subdir = catdir + os.sep
 972 elif repolevel == 1:
 973         for x in categories:
 974                 if not os.path.isdir(startdir + "/" + x):
 975                         continue
 976                 for y in os.listdir(startdir + "/" + x):
 977                         if y == "CVS" or y.startswith("."):
 978                                 continue
 979                         if os.path.isdir(startdir + "/" + x + "/" + y):
 980                                 scanlist.append(x + "/" + y)
 981         repo_subdir = ""
 982 elif repolevel == 3:
 983         catdir = reposplit[-2]
 984         if catdir not in categories:
 985                 caterror(catdir)
 986         scanlist.append(catdir + "/" + reposplit[-1])
 987         repo_subdir = scanlist[-1] + os.sep
 988 else:
 989         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 990                 ' from the current working directory'
 991         logging.critical(msg)
 992         sys.exit(1)
 993
 994 repo_subdir_len = len(repo_subdir)
 995 scanlist.sort()
 996
 997 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 998
 999 def vcs_files_to_cps(vcs_file_iter):
1000         """
1001         Iterate over the given modified file paths returned from the vcs,
1002         and return a frozenset containing category/pn strings for each
1003         modified package.
1004         """
1005
1006         modified_cps = []
1007
1008         if repolevel == 3:
1009                 if reposplit[-2] in categories and \
1010                         next(vcs_file_iter, None) is not None:
1011                         modified_cps.append("/".join(reposplit[-2:]))
1012
1013         elif repolevel == 2:
1014                 category = reposplit[-1]
1015                 if category in categories:
1016                         for filename in vcs_file_iter:
1017                                 f_split = filename.split(os.sep)
1018                                 # ['.', pn, ...]
1019                                 if len(f_split) > 2:
1020                                         modified_cps.append(category + "/" + f_split[1])
1021
1022         else:
1023                 # repolevel == 1
1024                 for filename in vcs_file_iter:
1025                         f_split = filename.split(os.sep)
1026                         # ['.', category, pn, ...]
1027                         if len(f_split) > 3 and f_split[1] in categories:
1028                                 modified_cps.append("/".join(f_split[1:3]))
1029
1030         return frozenset(modified_cps)
1031
1032 def git_supports_gpg_sign():
1033         status, cmd_output = \
1034                 repoman_getstatusoutput("git --version")
1035         cmd_output = cmd_output.split()
1036         if cmd_output:
1037                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1038                 if version is not None:
1039                         version = [int(x) for x in version.groups()]
1040                         if version[0] > 1 or \
1041                                 (version[0] == 1 and version[1] > 7) or \
1042                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1043                                 return True
1044         return False
1045
1046 def dev_keywords(profiles):
1047         """
1048         Create a set of KEYWORDS values that exist in 'dev'
1049         profiles. These are used
1050         to trigger a message notifying the user when they might
1051         want to add the --include-dev option.
1052         """
1053         type_arch_map = {}
1054         for arch, arch_profiles in profiles.items():
1055                 for prof in arch_profiles:
1056                         arch_set = type_arch_map.get(prof.status)
1057                         if arch_set is None:
1058                                 arch_set = set()
1059                                 type_arch_map[prof.status] = arch_set
1060                         arch_set.add(arch)
1061
1062         dev_keywords = type_arch_map.get('dev', set())
1063         dev_keywords.update(['~' + arch for arch in dev_keywords])
1064         return frozenset(dev_keywords)
1065
1066 dev_keywords = dev_keywords(profiles)
1067
1068 stats = {}
1069 fails = {}
1070
1071 for x in qacats:
1072         stats[x] = 0
1073         fails[x] = []
1074
1075 xmllint_capable = False
1076 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1077
1078 def fetch_metadata_dtd():
1079         """
1080         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1081         metadata_dtd_ctime_interval.
1082         @rtype: bool
1083         @return: True if successful, otherwise False
1084         """
1085
1086         must_fetch = True
1087         metadata_dtd_st = None
1088         current_time = int(time.time())
1089         try:
1090                 metadata_dtd_st = os.stat(metadata_dtd)
1091         except EnvironmentError as e:
1092                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1093                         raise
1094                 del e
1095         else:
1096                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1097                 if abs(current_time - metadata_dtd_st.st_ctime) \
1098                         < metadata_dtd_ctime_interval:
1099                         must_fetch = False
1100
1101         if must_fetch:
1102                 print()
1103                 print(green("***") + " the local copy of metadata.dtd " + \
1104                         "needs to be refetched, doing that now")
1105                 print()
1106                 parsed_url = urlparse(metadata_dtd_uri)
1107                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
1108                 fcmd = repoman_settings.get(setting)
1109                 if not fcmd:
1110                         fcmd = repoman_settings.get('FETCHCOMMAND')
1111                         if not fcmd:
1112                                 logging.error("FETCHCOMMAND is unset")
1113                                 return False
1114
1115                 destdir = repoman_settings["DISTDIR"]
1116                 fd, metadata_dtd_tmp = tempfile.mkstemp(
1117                         prefix='metadata.dtd.', dir=destdir)
1118                 os.close(fd)
1119
1120                 try:
1121                         if not portage.getbinpkg.file_get(metadata_dtd_uri,
1122                                 destdir, fcmd=fcmd,
1123                                 filename=os.path.basename(metadata_dtd_tmp)):
1124                                 logging.error("failed to fetch metadata.dtd from '%s'" %
1125                                         metadata_dtd_uri)
1126                                 return False
1127
1128                         try:
1129                                 portage.util.apply_secpass_permissions(metadata_dtd_tmp,
1130                                         gid=portage.data.portage_gid, mode=0o664, mask=0o2)
1131                         except portage.exception.PortageException:
1132                                 pass
1133
1134                         os.rename(metadata_dtd_tmp, metadata_dtd)
1135                 finally:
1136                         try:
1137                                 os.unlink(metadata_dtd_tmp)
1138                         except OSError:
1139                                 pass
1140
1141         return True
1142
1143 if options.mode == "manifest":
1144         pass
1145 elif not find_binary('xmllint'):
1146         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1147         if options.xml_parse or repolevel == 3:
1148                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1149                 sys.exit(1)
1150 else:
1151         if not fetch_metadata_dtd():
1152                 sys.exit(1)
1153         # this can be problematic if xmllint changes their output
1154         xmllint_capable = True
1155
1156 if options.mode == 'commit' and vcs:
1157         utilities.detect_vcs_conflicts(options, vcs)
1158
1159 if options.mode == "manifest":
1160         pass
1161 elif options.pretend:
1162         print(green("\nRepoMan does a once-over of the neighborhood..."))
1163 else:
1164         print(green("\nRepoMan scours the neighborhood..."))
1165
1166 new_ebuilds = set()
1167 modified_ebuilds = set()
1168 modified_changelogs = set()
1169 mychanged = []
1170 mynew = []
1171 myremoved = []
1172
1173 if vcs == "cvs":
1174         mycvstree = cvstree.getentries("./", recursive=1)
1175         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1176         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1177         if options.if_modified == "y":
1178                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1179
1180 elif vcs == "svn":
1181         with repoman_popen("svn status") as f:
1182                 svnstatus = f.readlines()
1183         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1184         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1185         if options.if_modified == "y":
1186                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1187
1188 elif vcs == "git":
1189         with repoman_popen("git diff-index --name-only "
1190                 "--relative --diff-filter=M HEAD") as f:
1191                 mychanged = f.readlines()
1192         mychanged = ["./" + elem[:-1] for elem in mychanged]
1193
1194         with repoman_popen("git diff-index --name-only "
1195                 "--relative --diff-filter=A HEAD") as f:
1196                 mynew = f.readlines()
1197         mynew = ["./" + elem[:-1] for elem in mynew]
1198         if options.if_modified == "y":
1199                 with repoman_popen("git diff-index --name-only "
1200                         "--relative --diff-filter=D HEAD") as f:
1201                         myremoved = f.readlines()
1202                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1203
1204 elif vcs == "bzr":
1205         with repoman_popen("bzr status -S .") as f:
1206                 bzrstatus = f.readlines()
1207         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1208         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1209         if options.if_modified == "y":
1210                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1211
1212 elif vcs == "hg":
1213         with repoman_popen("hg status --no-status --modified .") as f:
1214                 mychanged = f.readlines()
1215         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1216         with repoman_popen("hg status --no-status --added .") as f:
1217                 mynew = f.readlines()
1218         mynew = ["./" + elem.rstrip() for elem in mynew]
1219         if options.if_modified == "y":
1220                 with repoman_popen("hg status --no-status --removed .") as f:
1221                         myremoved = f.readlines()
1222                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1223
1224 if vcs:
1225         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1226         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1227         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1228                 if os.path.basename(x) == "ChangeLog")
1229
1230 def vcs_new_changed(relative_path):
1231         for x in chain(mychanged, mynew):
1232                 if x == relative_path:
1233                         return True
1234         return False
1235
1236 have_pmasked = False
1237 have_dev_keywords = False
1238 dofail = 0
1239
1240 # NOTE: match-all caches are not shared due to potential
1241 # differences between profiles in _get_implicit_iuse.
1242 arch_caches = {}
1243 arch_xmatch_caches = {}
1244 shared_xmatch_caches = {"cp-list":{}}
1245
1246 include_arches = None
1247 if options.include_arches:
1248         include_arches = set()
1249         include_arches.update(*[x.split() for x in options.include_arches])
1250
1251 # Disable the "ebuild.notadded" check when not in commit mode and
1252 # running `svn status` in every package dir will be too expensive.
1253
1254 check_ebuild_notadded = not \
1255         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1256
1257 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1258 thirdpartymirrors = {}
1259 for k, v in repoman_settings.thirdpartymirrors().items():
1260         for v in v:
1261                 if not v.endswith("/"):
1262                         v += "/"
1263                 thirdpartymirrors[v] = k
1264
1265 class _XMLParser(xml.etree.ElementTree.XMLParser):
1266
1267         def __init__(self, data, **kwargs):
1268                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1269                 self._portage_data = data
1270                 if hasattr(self, 'parser'):
1271                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1272                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1273                         self._base_StartDoctypeDeclHandler = \
1274                                 self.parser.StartDoctypeDeclHandler
1275                         self.parser.StartDoctypeDeclHandler = \
1276                                 self._portage_StartDoctypeDeclHandler
1277
1278         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1279                 if self._base_XmlDeclHandler is not None:
1280                         self._base_XmlDeclHandler(version, encoding, standalone)
1281                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1282
1283         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1284                 has_internal_subset):
1285                 if self._base_StartDoctypeDeclHandler is not None:
1286                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1287                                 has_internal_subset)
1288                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1289
1290 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1291         """
1292         Implements doctype() as required to avoid deprecation warnings with
1293         >=python-2.7.
1294         """
1295         def doctype(self, name, pubid, system):
1296                 pass
1297
1298 try:
1299         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1300 except (EnvironmentError, ParseError, PermissionDenied) as e:
1301         err(str(e))
1302 except FileNotFound:
1303         # TODO: Download as we do for metadata.dtd, but add a way to
1304         # disable for non-gentoo repoman users who may not have herds.
1305         herd_base = None
1306
1307 effective_scanlist = scanlist
1308 if options.if_modified == "y":
1309         effective_scanlist = sorted(vcs_files_to_cps(
1310                 chain(mychanged, mynew, myremoved)))
1311
1312 for x in effective_scanlist:
1313         # ebuilds and digests added to cvs respectively.
1314         logging.info("checking package %s" % x)
1315         # save memory by discarding xmatch caches from previous package(s)
1316         arch_xmatch_caches.clear()
1317         eadded = []
1318         catdir, pkgdir = x.split("/")
1319         checkdir = repodir + "/" + x
1320         checkdir_relative = ""
1321         if repolevel < 3:
1322                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1323         if repolevel < 2:
1324                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1325         checkdir_relative = os.path.join(".", checkdir_relative)
1326         generated_manifest = False
1327
1328         if options.mode == "manifest" or \
1329           (options.mode != 'manifest-check' and options.digest == 'y') or \
1330           options.mode in ('commit', 'fix') and not options.pretend:
1331                 auto_assumed = set()
1332                 fetchlist_dict = portage.FetchlistDict(checkdir,
1333                         repoman_settings, portdb)
1334                 if options.mode == 'manifest' and options.force:
1335                         portage._doebuild_manifest_exempt_depend += 1
1336                         try:
1337                                 distdir = repoman_settings['DISTDIR']
1338                                 mf = repoman_settings.repositories.get_repo_for_location(
1339                                         os.path.dirname(os.path.dirname(checkdir)))
1340                                 mf = mf.load_manifest(checkdir, distdir,
1341                                         fetchlist_dict=fetchlist_dict)
1342                                 mf.create(requiredDistfiles=None,
1343                                         assumeDistHashesAlways=True)
1344                                 for distfiles in fetchlist_dict.values():
1345                                         for distfile in distfiles:
1346                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1347                                                         mf.fhashdict['DIST'].pop(distfile, None)
1348                                                 else:
1349                                                         auto_assumed.add(distfile)
1350                                 mf.write()
1351                         finally:
1352                                 portage._doebuild_manifest_exempt_depend -= 1
1353
1354                 repoman_settings["O"] = checkdir
1355                 try:
1356                         generated_manifest = digestgen(
1357                                 mysettings=repoman_settings, myportdb=portdb)
1358                 except portage.exception.PermissionDenied as e:
1359                         generated_manifest = False
1360                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1361                                 level=logging.ERROR, noiselevel=-1)
1362
1363                 if not generated_manifest:
1364                         print("Unable to generate manifest.")
1365                         dofail = 1
1366
1367                 if options.mode == "manifest":
1368                         if not dofail and options.force and auto_assumed and \
1369                                 'assume-digests' in repoman_settings.features:
1370                                 # Show which digests were assumed despite the --force option
1371                                 # being given. This output will already have been shown by
1372                                 # digestgen() if assume-digests is not enabled, so only show
1373                                 # it here if assume-digests is enabled.
1374                                 pkgs = list(fetchlist_dict)
1375                                 pkgs.sort()
1376                                 portage.writemsg_stdout("  digest.assumed" + \
1377                                         portage.output.colorize("WARN",
1378                                         str(len(auto_assumed)).rjust(18)) + "\n")
1379                                 for cpv in pkgs:
1380                                         fetchmap = fetchlist_dict[cpv]
1381                                         pf = portage.catsplit(cpv)[1]
1382                                         for distfile in sorted(fetchmap):
1383                                                 if distfile in auto_assumed:
1384                                                         portage.writemsg_stdout(
1385                                                                 "   %s::%s\n" % (pf, distfile))
1386                         continue
1387                 elif dofail:
1388                         sys.exit(1)
1389
1390         if not generated_manifest:
1391                 repoman_settings['O'] = checkdir
1392                 repoman_settings['PORTAGE_QUIET'] = '1'
1393                 if not portage.digestcheck([], repoman_settings, strict=1):
1394                         stats["manifest.bad"] += 1
1395                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1396                 repoman_settings.pop('PORTAGE_QUIET', None)
1397
1398         if options.mode == 'manifest-check':
1399                 continue
1400
1401         checkdirlist = os.listdir(checkdir)
1402         ebuildlist = []
1403         pkgs = {}
1404         allvalid = True
1405         for y in checkdirlist:
1406                 if (y in no_exec or y.endswith(".ebuild")) and \
1407                                 stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1408                         stats["file.executable"] += 1
1409                         fails["file.executable"].append(os.path.join(checkdir, y))
1410                 if y.endswith(".ebuild"):
1411                         pf = y[:-7]
1412                         ebuildlist.append(pf)
1413                         cpv = "%s/%s" % (catdir, pf)
1414                         try:
1415                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1416                         except KeyError:
1417                                 allvalid = False
1418                                 stats["ebuild.syntax"] += 1
1419                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1420                                 continue
1421                         except IOError:
1422                                 allvalid = False
1423                                 stats["ebuild.output"] += 1
1424                                 fails["ebuild.output"].append(os.path.join(x, y))
1425                                 continue
1426                         if not portage.eapi_is_supported(myaux["EAPI"]):
1427                                 allvalid = False
1428                                 stats["EAPI.unsupported"] += 1
1429                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1430                                 continue
1431                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1432                                 root_config=root_config, type_name="ebuild")
1433
1434         slot_keywords = {}
1435
1436         if len(pkgs) != len(ebuildlist):
1437                 # If we can't access all the metadata then it's totally unsafe to
1438                 # commit since there's no way to generate a correct Manifest.
1439                 # Do not try to do any more QA checks on this package since missing
1440                 # metadata leads to false positives for several checks, and false
1441                 # positives confuse users.
1442                 can_force = False
1443                 continue
1444
1445         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1446         ebuildlist = sorted(pkgs.values())
1447         ebuildlist = [pkg.pf for pkg in ebuildlist]
1448
1449         for y in checkdirlist:
1450                 index = repo_config.find_invalid_path_char(y)
1451                 if index != -1:
1452                         y_relative = os.path.join(checkdir_relative, y)
1453                         if vcs is not None and not vcs_new_changed(y_relative):
1454                                 # If the file isn't in the VCS new or changed set, then
1455                                 # assume that it's an irrelevant temporary file (Manifest
1456                                 # entries are not generated for file names containing
1457                                 # prohibited characters). See bug #406877.
1458                                 index = -1
1459                 if index != -1:
1460                         stats["file.name"] += 1
1461                         fails["file.name"].append("%s/%s: char '%s'" % \
1462                                 (checkdir, y, y[index]))
1463
1464                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1465                         continue
1466                 f = None
1467                 try:
1468                         line = 1
1469                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1470                                 encoding=_encodings['fs'], errors='strict'),
1471                                 mode='r', encoding=_encodings['repo.content'])
1472                         for l in f:
1473                                 line += 1
1474                 except UnicodeDecodeError as ue:
1475                         stats["file.UTF8"] += 1
1476                         s = ue.object[:ue.start]
1477                         l2 = s.count("\n")
1478                         line += l2
1479                         if l2 != 0:
1480                                 s = s[s.rfind("\n") + 1:]
1481                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1482                 finally:
1483                         if f is not None:
1484                                 f.close()
1485
1486         if vcs in ("git", "hg") and check_ebuild_notadded:
1487                 if vcs == "git":
1488                         myf = repoman_popen("git ls-files --others %s" % \
1489                                 (portage._shell_quote(checkdir_relative),))
1490                 if vcs == "hg":
1491                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1492                                 (portage._shell_quote(checkdir_relative),))
1493                 for l in myf:
1494                         if l[:-1][-7:] == ".ebuild":
1495                                 stats["ebuild.notadded"] += 1
1496                                 fails["ebuild.notadded"].append(
1497                                         os.path.join(x, os.path.basename(l[:-1])))
1498                 myf.close()
1499
1500         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1501                 try:
1502                         if vcs == "cvs":
1503                                 myf = open(checkdir + "/CVS/Entries", "r")
1504                         if vcs == "svn":
1505                                 myf = repoman_popen("svn status --depth=files --verbose " +
1506                                         portage._shell_quote(checkdir))
1507                         if vcs == "bzr":
1508                                 myf = repoman_popen("bzr ls -v --kind=file " +
1509                                         portage._shell_quote(checkdir))
1510                         myl = myf.readlines()
1511                         myf.close()
1512                         for l in myl:
1513                                 if vcs == "cvs":
1514                                         if l[0] != "/":
1515                                                 continue
1516                                         splitl = l[1:].split("/")
1517                                         if not len(splitl):
1518                                                 continue
1519                                         if splitl[0][-7:] == ".ebuild":
1520                                                 eadded.append(splitl[0][:-7])
1521                                 if vcs == "svn":
1522                                         if l[:1] == "?":
1523                                                 continue
1524                                         if l[:7] == '      >':
1525                                                 # tree conflict, new in subversion 1.6
1526                                                 continue
1527                                         l = l.split()[-1]
1528                                         if l[-7:] == ".ebuild":
1529                                                 eadded.append(os.path.basename(l[:-7]))
1530                                 if vcs == "bzr":
1531                                         if l[1:2] == "?":
1532                                                 continue
1533                                         l = l.split()[-1]
1534                                         if l[-7:] == ".ebuild":
1535                                                 eadded.append(os.path.basename(l[:-7]))
1536                         if vcs == "svn":
1537                                 myf = repoman_popen("svn status " +
1538                                         portage._shell_quote(checkdir))
1539                                 myl = myf.readlines()
1540                                 myf.close()
1541                                 for l in myl:
1542                                         if l[0] == "A":
1543                                                 l = l.rstrip().split(' ')[-1]
1544                                                 if l[-7:] == ".ebuild":
1545                                                         eadded.append(os.path.basename(l[:-7]))
1546                 except IOError:
1547                         if vcs == "cvs":
1548                                 stats["CVS/Entries.IO_error"] += 1
1549                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1550                         else:
1551                                 raise
1552                         continue
1553
1554         mf = repoman_settings.repositories.get_repo_for_location(
1555                 os.path.dirname(os.path.dirname(checkdir)))
1556         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1557         mydigests = mf.getTypeDigests("DIST")
1558
1559         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1560         myfiles_all = []
1561         src_uri_error = False
1562         for mykey in fetchlist_dict:
1563                 try:
1564                         myfiles_all.extend(fetchlist_dict[mykey])
1565                 except portage.exception.InvalidDependString as e:
1566                         src_uri_error = True
1567                         try:
1568                                 portdb.aux_get(mykey, ["SRC_URI"])
1569                         except KeyError:
1570                                 # This will be reported as an "ebuild.syntax" error.
1571                                 pass
1572                         else:
1573                                 stats["SRC_URI.syntax"] += 1
1574                                 fails["SRC_URI.syntax"].append(
1575                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1576         del fetchlist_dict
1577         if not src_uri_error:
1578                 # This test can produce false positives if SRC_URI could not
1579                 # be parsed for one or more ebuilds. There's no point in
1580                 # producing a false error here since the root cause will
1581                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1582                 # or "ebuild.sytax".
1583                 myfiles_all = set(myfiles_all)
1584                 for entry in mydigests:
1585                         if entry not in myfiles_all:
1586                                 stats["digest.unused"] += 1
1587                                 fails["digest.unused"].append(checkdir + "::" + entry)
1588                 for entry in myfiles_all:
1589                         if entry not in mydigests:
1590                                 stats["digest.missing"] += 1
1591                                 fails["digest.missing"].append(checkdir + "::" + entry)
1592         del myfiles_all
1593
1594         if os.path.exists(checkdir + "/files"):
1595                 filesdirlist = os.listdir(checkdir + "/files")
1596
1597                 # recurse through files directory
1598                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1599                 while filesdirlist:
1600                         y = filesdirlist.pop(0)
1601                         relative_path = os.path.join(x, "files", y)
1602                         full_path = os.path.join(repodir, relative_path)
1603                         try:
1604                                 mystat = os.stat(full_path)
1605                         except OSError as oe:
1606                                 if oe.errno == 2:
1607                                         # don't worry about it.  it likely was removed via fix above.
1608                                         continue
1609                                 else:
1610                                         raise oe
1611                         if S_ISDIR(mystat.st_mode):
1612                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1613                                 if y == "CVS" or y == ".svn":
1614                                         continue
1615                                 for z in os.listdir(checkdir + "/files/" + y):
1616                                         if z == "CVS" or z == ".svn":
1617                                                 continue
1618                                         filesdirlist.append(y + "/" + z)
1619                         # Current policy is no files over 20 KiB, these are the checks. File size between
1620                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1621                         elif mystat.st_size > 61440:
1622                                 stats["file.size.fatal"] += 1
1623                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1624                         elif mystat.st_size > 20480:
1625                                 stats["file.size"] += 1
1626                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1627
1628                         index = repo_config.find_invalid_path_char(y)
1629                         if index != -1:
1630                                 y_relative = os.path.join(checkdir_relative, "files", y)
1631                                 if vcs is not None and not vcs_new_changed(y_relative):
1632                                         # If the file isn't in the VCS new or changed set, then
1633                                         # assume that it's an irrelevant temporary file (Manifest
1634                                         # entries are not generated for file names containing
1635                                         # prohibited characters). See bug #406877.
1636                                         index = -1
1637                         if index != -1:
1638                                 stats["file.name"] += 1
1639                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1640                                         (checkdir, y, y[index]))
1641         del mydigests
1642
1643         if check_changelog and "ChangeLog" not in checkdirlist:
1644                 stats["changelog.missing"] += 1
1645                 fails["changelog.missing"].append(x + "/ChangeLog")
1646
1647         musedict = {}
1648         # metadata.xml file check
1649         if "metadata.xml" not in checkdirlist:
1650                 stats["metadata.missing"] += 1
1651                 fails["metadata.missing"].append(x + "/metadata.xml")
1652         # metadata.xml parse check
1653         else:
1654                 metadata_bad = False
1655                 xml_info = {}
1656                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1657
1658                 # read metadata.xml into memory
1659                 try:
1660                         _metadata_xml = xml.etree.ElementTree.parse(
1661                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1662                                 encoding=_encodings['fs'], errors='strict'),
1663                                 parser=xml_parser)
1664                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1665                         metadata_bad = True
1666                         stats["metadata.bad"] += 1
1667                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1668                         del e
1669                 else:
1670                         if not hasattr(xml_parser, 'parser') or \
1671                                 sys.hexversion < 0x2070000 or \
1672                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1673                                 # doctype is not parsed with python 2.6 or 3.1
1674                                 pass
1675                         else:
1676                                 if "XML_DECLARATION" not in xml_info:
1677                                         stats["metadata.bad"] += 1
1678                                         fails["metadata.bad"].append("%s/metadata.xml: "
1679                                                 "xml declaration is missing on first line, "
1680                                                 "should be '%s'" % (x, metadata_xml_declaration))
1681                                 else:
1682                                         xml_version, xml_encoding, xml_standalone = \
1683                                                 xml_info["XML_DECLARATION"]
1684                                         if xml_encoding is None or \
1685                                                 xml_encoding.upper() != metadata_xml_encoding:
1686                                                 stats["metadata.bad"] += 1
1687                                                 if xml_encoding is None:
1688                                                         encoding_problem = "but it is undefined"
1689                                                 else:
1690                                                         encoding_problem = "not '%s'" % xml_encoding
1691                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1692                                                         "xml declaration encoding should be '%s', %s" %
1693                                                         (x, metadata_xml_encoding, encoding_problem))
1694
1695                                 if "DOCTYPE" not in xml_info:
1696                                         metadata_bad = True
1697                                         stats["metadata.bad"] += 1
1698                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1699                                                 "DOCTYPE is missing"))
1700                                 else:
1701                                         doctype_name, doctype_system, doctype_pubid = \
1702                                                 xml_info["DOCTYPE"]
1703                                         if doctype_system != metadata_dtd_uri:
1704                                                 stats["metadata.bad"] += 1
1705                                                 if doctype_system is None:
1706                                                         system_problem = "but it is undefined"
1707                                                 else:
1708                                                         system_problem = "not '%s'" % doctype_system
1709                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1710                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1711                                                         (x, metadata_dtd_uri, system_problem))
1712
1713                                         if doctype_name != metadata_doctype_name:
1714                                                 stats["metadata.bad"] += 1
1715                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1716                                                         "DOCTYPE: name should be '%s', not '%s'" %
1717                                                         (x, metadata_doctype_name, doctype_name))
1718
1719                         # load USE flags from metadata.xml
1720                         try:
1721                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1722                         except portage.exception.ParseError as e:
1723                                 metadata_bad = True
1724                                 stats["metadata.bad"] += 1
1725                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1726                         else:
1727                                 for atom in chain(*musedict.values()):
1728                                         if atom is None:
1729                                                 continue
1730                                         try:
1731                                                 atom = Atom(atom)
1732                                         except InvalidAtom as e:
1733                                                 stats["metadata.bad"] += 1
1734                                                 fails["metadata.bad"].append(
1735                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1736                                         else:
1737                                                 if atom.cp != x:
1738                                                         stats["metadata.bad"] += 1
1739                                                         fails["metadata.bad"].append(
1740                                                                 ("%s/metadata.xml: Atom contains "
1741                                                                 "unexpected cat/pn: %s") % (x, atom))
1742
1743                         # Run other metadata.xml checkers
1744                         try:
1745                                 utilities.check_metadata(_metadata_xml, herd_base)
1746                         except (utilities.UnknownHerdsError, ) as e:
1747                                 metadata_bad = True
1748                                 stats["metadata.bad"] += 1
1749                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1750                                 del e
1751
1752                 # Only carry out if in package directory or check forced
1753                 if xmllint_capable and not metadata_bad:
1754                         # xmlint can produce garbage output even on success, so only dump
1755                         # the ouput when it fails.
1756                         st, out = repoman_getstatusoutput(
1757                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1758                                  (portage._shell_quote(metadata_dtd),
1759                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1760                         if st != os.EX_OK:
1761                                 print(red("!!!") + " metadata.xml is invalid:")
1762                                 for z in out.splitlines():
1763                                         print(red("!!! ") + z)
1764                                 stats["metadata.bad"] += 1
1765                                 fails["metadata.bad"].append(x + "/metadata.xml")
1766
1767                 del metadata_bad
1768         muselist = frozenset(musedict)
1769
1770         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1771         changelog_modified = changelog_path in modified_changelogs
1772
1773         # detect unused local USE-descriptions
1774         used_useflags = set()
1775
1776         for y in ebuildlist:
1777                 relative_path = os.path.join(x, y + ".ebuild")
1778                 full_path = os.path.join(repodir, relative_path)
1779                 ebuild_path = y + ".ebuild"
1780                 if repolevel < 3:
1781                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1782                 if repolevel < 2:
1783                         ebuild_path = os.path.join(catdir, ebuild_path)
1784                 ebuild_path = os.path.join(".", ebuild_path)
1785                 if check_changelog and not changelog_modified \
1786                         and ebuild_path in new_ebuilds:
1787                         stats['changelog.ebuildadded'] += 1
1788                         fails['changelog.ebuildadded'].append(relative_path)
1789
1790                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1791                         # ebuild not added to vcs
1792                         stats["ebuild.notadded"] += 1
1793                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1794                 myesplit = portage.pkgsplit(y)
1795                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1796                             or pv_toolong_re.search(myesplit[1]) \
1797                             or pv_toolong_re.search(myesplit[2]):
1798                         stats["ebuild.invalidname"] += 1
1799                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1800                         continue
1801                 elif myesplit[0] != pkgdir:
1802                         print(pkgdir, myesplit[0])
1803                         stats["ebuild.namenomatch"] += 1
1804                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1805                         continue
1806
1807                 pkg = pkgs[y]
1808
1809                 if pkg.invalid:
1810                         allvalid = False
1811                         for k, msgs in pkg.invalid.items():
1812                                 for msg in msgs:
1813                                         stats[k] += 1
1814                                         fails[k].append("%s: %s" % (relative_path, msg))
1815                         continue
1816
1817                 myaux = pkg._metadata
1818                 eapi = myaux["EAPI"]
1819                 inherited = pkg.inherited
1820                 live_ebuild = live_eclasses.intersection(inherited)
1821
1822                 if repo_config.eapi_is_banned(eapi):
1823                         stats["repo.eapi.banned"] += 1
1824                         fails["repo.eapi.banned"].append(
1825                                 "%s: %s" % (relative_path, eapi))
1826
1827                 elif repo_config.eapi_is_deprecated(eapi):
1828                         stats["repo.eapi.deprecated"] += 1
1829                         fails["repo.eapi.deprecated"].append(
1830                                 "%s: %s" % (relative_path, eapi))
1831
1832                 for k, v in myaux.items():
1833                         if not isinstance(v, basestring):
1834                                 continue
1835                         m = non_ascii_re.search(v)
1836                         if m is not None:
1837                                 stats["variable.invalidchar"] += 1
1838                                 fails["variable.invalidchar"].append(
1839                                         ("%s: %s variable contains non-ASCII " + \
1840                                         "character at position %s") % \
1841                                         (relative_path, k, m.start() + 1))
1842
1843                 if not src_uri_error:
1844                         # Check that URIs don't reference a server from thirdpartymirrors.
1845                         for uri in portage.dep.use_reduce( \
1846                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1847                                 contains_mirror = False
1848                                 for mirror, mirror_alias in thirdpartymirrors.items():
1849                                         if uri.startswith(mirror):
1850                                                 contains_mirror = True
1851                                                 break
1852                                 if not contains_mirror:
1853                                         continue
1854
1855                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1856                                 stats["SRC_URI.mirror"] += 1
1857                                 fails["SRC_URI.mirror"].append(
1858                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1859                                         (relative_path, mirror, new_uri))
1860
1861                 if myaux.get("PROVIDE"):
1862                         stats["virtual.oldstyle"] += 1
1863                         fails["virtual.oldstyle"].append(relative_path)
1864
1865                 for pos, missing_var in enumerate(missingvars):
1866                         if not myaux.get(missing_var):
1867                                 if catdir == "virtual" and \
1868                                         missing_var in ("HOMEPAGE", "LICENSE"):
1869                                         continue
1870                                 if live_ebuild and missing_var == "KEYWORDS":
1871                                         continue
1872                                 myqakey = missingvars[pos] + ".missing"
1873                                 stats[myqakey] += 1
1874                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1875
1876                 if catdir == "virtual":
1877                         for var in ("HOMEPAGE", "LICENSE"):
1878                                 if myaux.get(var):
1879                                         myqakey = var + ".virtual"
1880                                         stats[myqakey] += 1
1881                                         fails[myqakey].append(relative_path)
1882
1883                 # 14 is the length of DESCRIPTION=""
1884                 if len(myaux['DESCRIPTION']) > max_desc_len:
1885                         stats['DESCRIPTION.toolong'] += 1
1886                         fails['DESCRIPTION.toolong'].append(
1887                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1888                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1889
1890                 keywords = myaux["KEYWORDS"].split()
1891                 stable_keywords = []
1892                 for keyword in keywords:
1893                         if not keyword.startswith("~") and \
1894                                 not keyword.startswith("-"):
1895                                 stable_keywords.append(keyword)
1896                 if stable_keywords:
1897                         if ebuild_path in new_ebuilds and catdir != "virtual":
1898                                 stable_keywords.sort()
1899                                 stats["KEYWORDS.stable"] += 1
1900                                 fails["KEYWORDS.stable"].append(
1901                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1902                                                 " ".join(stable_keywords))
1903
1904                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1905                         if not kw.startswith("-"))
1906
1907                 previous_keywords = slot_keywords.get(pkg.slot)
1908                 if previous_keywords is None:
1909                         slot_keywords[pkg.slot] = set()
1910                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1911                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1912                         if dropped_keywords:
1913                                 stats["KEYWORDS.dropped"] += 1
1914                                 fails["KEYWORDS.dropped"].append(
1915                                         relative_path + ": %s" % \
1916                                         " ".join(sorted(dropped_keywords)))
1917
1918                 slot_keywords[pkg.slot].update(ebuild_archs)
1919
1920                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1921                 if "-*" in keywords:
1922                         haskeyword = False
1923                         for kw in keywords:
1924                                 if kw[0] == "~":
1925                                         kw = kw[1:]
1926                                 if kw in kwlist:
1927                                         haskeyword = True
1928                         if not haskeyword:
1929                                 stats["KEYWORDS.stupid"] += 1
1930                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1931
1932                 """
1933                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1934                 not be allowed to be marked stable
1935                 """
1936                 if live_ebuild and repo_config.name == "gentoo":
1937                         bad_stable_keywords = []
1938                         for keyword in keywords:
1939                                 if not keyword.startswith("~") and \
1940                                         not keyword.startswith("-"):
1941                                         bad_stable_keywords.append(keyword)
1942                                 del keyword
1943                         if bad_stable_keywords:
1944                                 stats["LIVEVCS.stable"] += 1
1945                                 fails["LIVEVCS.stable"].append(
1946                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1947                                                 bad_stable_keywords)
1948                         del bad_stable_keywords
1949
1950                         if keywords and not has_global_mask(pkg):
1951                                 stats["LIVEVCS.unmasked"] += 1
1952                                 fails["LIVEVCS.unmasked"].append(relative_path)
1953
1954                 if options.ignore_arches:
1955                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1956                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1957                 else:
1958                         arches = set()
1959                         for keyword in keywords:
1960                                 if keyword[0] == "-":
1961                                         continue
1962                                 elif keyword[0] == "~":
1963                                         arch = keyword[1:]
1964                                         if arch == "*":
1965                                                 for expanded_arch in profiles:
1966                                                         if expanded_arch == "**":
1967                                                                 continue
1968                                                         arches.add((keyword, expanded_arch,
1969                                                                 (expanded_arch, "~" + expanded_arch)))
1970                                         else:
1971                                                 arches.add((keyword, arch, (arch, keyword)))
1972                                 else:
1973                                         if keyword == "*":
1974                                                 for expanded_arch in profiles:
1975                                                         if expanded_arch == "**":
1976                                                                 continue
1977                                                         arches.add((keyword, expanded_arch,
1978                                                                 (expanded_arch,)))
1979                                         else:
1980                                                 arches.add((keyword, keyword, (keyword,)))
1981                         if not arches:
1982                                 # Use an empty profile for checking dependencies of
1983                                 # packages that have empty KEYWORDS.
1984                                 arches.add(('**', '**', ('**',)))
1985
1986                 unknown_pkgs = set()
1987                 baddepsyntax = False
1988                 badlicsyntax = False
1989                 badprovsyntax = False
1990                 catpkg = catdir + "/" + y
1991
1992                 inherited_java_eclass = "java-pkg-2" in inherited or \
1993                         "java-pkg-opt-2" in inherited
1994                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1995                 operator_tokens = set(["||", "(", ")"])
1996                 type_list, badsyntax = [], []
1997                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
1998                         mydepstr = myaux[mytype]
1999
2000                         buildtime = mytype in Package._buildtime_keys
2001                         runtime = mytype in Package._runtime_keys
2002                         token_class = None
2003                         if mytype.endswith("DEPEND"):
2004                                 token_class = portage.dep.Atom
2005
2006                         try:
2007                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
2008                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
2009                         except portage.exception.InvalidDependString as e:
2010                                 atoms = None
2011                                 badsyntax.append(str(e))
2012
2013                         if atoms and mytype.endswith("DEPEND"):
2014                                 if runtime and \
2015                                         "test?" in mydepstr.split():
2016                                         stats[mytype + '.suspect'] += 1
2017                                         fails[mytype + '.suspect'].append(relative_path + \
2018                                                 ": 'test?' USE conditional in %s" % mytype)
2019
2020                                 for atom in atoms:
2021                                         if atom == "||":
2022                                                 continue
2023
2024                                         # Skip dependency.unknown for blockers, so that we
2025                                         # don't encourage people to remove necessary blockers,
2026                                         # as discussed in bug #382407.
2027                                         if atom.blocker is None and \
2028                                                 not portdb.xmatch("match-all", atom) and \
2029                                                 not atom.cp.startswith("virtual/"):
2030                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2031
2032                                         is_blocker = atom.blocker
2033
2034                                         if catdir != "virtual":
2035                                                 if not is_blocker and \
2036                                                         atom.cp in suspect_virtual:
2037                                                         stats['virtual.suspect'] += 1
2038                                                         fails['virtual.suspect'].append(
2039                                                                 relative_path +
2040                                                                 ": %s: consider using '%s' instead of '%s'" %
2041                                                                 (mytype, suspect_virtual[atom.cp], atom))
2042
2043                                         if buildtime and \
2044                                                 not is_blocker and \
2045                                                 not inherited_java_eclass and \
2046                                                 atom.cp == "virtual/jdk":
2047                                                 stats['java.eclassesnotused'] += 1
2048                                                 fails['java.eclassesnotused'].append(relative_path)
2049                                         elif buildtime and \
2050                                                 not is_blocker and \
2051                                                 not inherited_wxwidgets_eclass and \
2052                                                 atom.cp == "x11-libs/wxGTK":
2053                                                 stats['wxwidgets.eclassnotused'] += 1
2054                                                 fails['wxwidgets.eclassnotused'].append(
2055                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2056                                                         " without inheriting wxwidgets.eclass") % mytype)
2057                                         elif runtime:
2058                                                 if not is_blocker and \
2059                                                         atom.cp in suspect_rdepend:
2060                                                         stats[mytype + '.suspect'] += 1
2061                                                         fails[mytype + '.suspect'].append(
2062                                                                 relative_path + ": '%s'" % atom)
2063
2064                                         if atom.operator == "~" and \
2065                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2066                                                 qacat = 'dependency.badtilde'
2067                                                 stats[qacat] += 1
2068                                                 fails[qacat].append(
2069                                                         (relative_path + ": %s uses the ~ operator"
2070                                                          " with a non-zero revision:" + \
2071                                                          " '%s'") % (mytype, atom))
2072
2073                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2074
2075                 for m, b in zip(type_list, badsyntax):
2076                         if m.endswith("DEPEND"):
2077                                 qacat = "dependency.syntax"
2078                         else:
2079                                 qacat = m + ".syntax"
2080                         stats[qacat] += 1
2081                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2082
2083                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2084                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2085                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2086                 badlicsyntax = badlicsyntax > 0
2087                 badprovsyntax = badprovsyntax > 0
2088
2089                 # uselist checks - global
2090                 myuse = []
2091                 default_use = []
2092                 for myflag in myaux["IUSE"].split():
2093                         flag_name = myflag.lstrip("+-")
2094                         used_useflags.add(flag_name)
2095                         if myflag != flag_name:
2096                                 default_use.append(myflag)
2097                         if flag_name not in uselist:
2098                                 myuse.append(flag_name)
2099
2100                 # uselist checks - metadata
2101                 for mypos in range(len(myuse)-1, -1, -1):
2102                         if myuse[mypos] and (myuse[mypos] in muselist):
2103                                 del myuse[mypos]
2104
2105                 if default_use and not eapi_has_iuse_defaults(eapi):
2106                         for myflag in default_use:
2107                                 stats['EAPI.incompatible'] += 1
2108                                 fails['EAPI.incompatible'].append(
2109                                         (relative_path + ": IUSE defaults" + \
2110                                         " not supported with EAPI='%s':" + \
2111                                         " '%s'") % (eapi, myflag))
2112
2113                 for mypos in range(len(myuse)):
2114                         stats["IUSE.invalid"] += 1
2115                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2116
2117                 # Check for outdated RUBY targets
2118                 if "ruby-ng" in inherited or "ruby-fakegem" in inherited or "ruby" in inherited:
2119                         ruby_intersection = pkg.iuse.all.intersection(ruby_deprecated)
2120                         if ruby_intersection:
2121                                 for myruby in ruby_intersection:
2122                                         stats["IUSE.rubydeprecated"] += 1
2123                                         fails["IUSE.rubydeprecated"].append(
2124                                                 (relative_path + ": Deprecated ruby target: %s") % myruby)
2125
2126                 # license checks
2127                 if not badlicsyntax:
2128                         # Parse the LICENSE variable, remove USE conditions and
2129                         # flatten it.
2130                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2131                         # Check each entry to ensure that it exists in PORTDIR's
2132                         # license directory.
2133                         for lic in licenses:
2134                                 # Need to check for "||" manually as no portage
2135                                 # function will remove it without removing values.
2136                                 if lic not in liclist and lic != "||":
2137                                         stats["LICENSE.invalid"] += 1
2138                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2139                                 elif lic in liclist_deprecated:
2140                                         stats["LICENSE.deprecated"] += 1
2141                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2142
2143                 # keyword checks
2144                 myuse = myaux["KEYWORDS"].split()
2145                 for mykey in myuse:
2146                         if mykey not in ("-*", "*", "~*"):
2147                                 myskey = mykey
2148                                 if myskey[:1] == "-":
2149                                         myskey = myskey[1:]
2150                                 if myskey[:1] == "~":
2151                                         myskey = myskey[1:]
2152                                 if myskey not in kwlist:
2153                                         stats["KEYWORDS.invalid"] += 1
2154                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2155                                 elif myskey not in profiles:
2156                                         stats["KEYWORDS.invalid"] += 1
2157                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2158
2159                 # restrict checks
2160                 myrestrict = None
2161                 try:
2162                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2163                 except portage.exception.InvalidDependString as e:
2164                         stats["RESTRICT.syntax"] += 1
2165                         fails["RESTRICT.syntax"].append(
2166                                 "%s: RESTRICT: %s" % (relative_path, e))
2167                         del e
2168                 if myrestrict:
2169                         myrestrict = set(myrestrict)
2170                         mybadrestrict = myrestrict.difference(valid_restrict)
2171                         if mybadrestrict:
2172                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2173                                 for mybad in mybadrestrict:
2174                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2175                 # REQUIRED_USE check
2176                 required_use = myaux["REQUIRED_USE"]
2177                 if required_use:
2178                         if not eapi_has_required_use(eapi):
2179                                 stats['EAPI.incompatible'] += 1
2180                                 fails['EAPI.incompatible'].append(
2181                                         relative_path + ": REQUIRED_USE" + \
2182                                         " not supported with EAPI='%s'" % (eapi,))
2183                         try:
2184                                 portage.dep.check_required_use(required_use, (),
2185                                         pkg.iuse.is_valid_flag, eapi=eapi)
2186                         except portage.exception.InvalidDependString as e:
2187                                 stats["REQUIRED_USE.syntax"] += 1
2188                                 fails["REQUIRED_USE.syntax"].append(
2189                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2190                                 del e
2191
2192                 # Syntax Checks
2193                 relative_path = os.path.join(x, y + ".ebuild")
2194                 full_path = os.path.join(repodir, relative_path)
2195                 if not vcs_preserves_mtime:
2196                         if ebuild_path not in new_ebuilds and \
2197                                 ebuild_path not in modified_ebuilds:
2198                                 pkg.mtime = None
2199                 try:
2200                         # All ebuilds should have utf_8 encoding.
2201                         f = io.open(_unicode_encode(full_path,
2202                                 encoding=_encodings['fs'], errors='strict'),
2203                                 mode='r', encoding=_encodings['repo.content'])
2204                         try:
2205                                 for check_name, e in run_checks(f, pkg):
2206                                         stats[check_name] += 1
2207                                         fails[check_name].append(relative_path + ': %s' % e)
2208                         finally:
2209                                 f.close()
2210                 except UnicodeDecodeError:
2211                         # A file.UTF8 failure will have already been recorded above.
2212                         pass
2213
2214                 if options.force:
2215                         # The dep_check() calls are the most expensive QA test. If --force
2216                         # is enabled, there's no point in wasting time on these since the
2217                         # user is intent on forcing the commit anyway.
2218                         continue
2219
2220                 relevant_profiles = []
2221                 for keyword, arch, groups in arches:
2222                         if arch not in profiles:
2223                                 # A missing profile will create an error further down
2224                                 # during the KEYWORDS verification.
2225                                 continue
2226
2227                         if include_arches is not None:
2228                                 if arch not in include_arches:
2229                                         continue
2230
2231                         relevant_profiles.extend((keyword, groups, prof)
2232                                 for prof in profiles[arch])
2233
2234                 def sort_key(item):
2235                         return item[2].sub_path
2236
2237                 relevant_profiles.sort(key=sort_key)
2238
2239                 for keyword, groups, prof in relevant_profiles:
2240
2241                         if not (prof.status == "stable" or \
2242                                 (prof.status == "dev" and options.include_dev) or \
2243                                 (prof.status == "exp" and options.include_exp_profiles == 'y')):
2244                                 continue
2245
2246                         dep_settings = arch_caches.get(prof.sub_path)
2247                         if dep_settings is None:
2248                                 dep_settings = portage.config(
2249                                         config_profile_path=prof.abs_path,
2250                                         config_incrementals=repoman_incrementals,
2251                                         config_root=config_root,
2252                                         local_config=False,
2253                                         _unmatched_removal=options.unmatched_removal,
2254                                         env=env, repositories=repoman_settings.repositories)
2255                                 dep_settings.categories = repoman_settings.categories
2256                                 if options.without_mask:
2257                                         dep_settings._mask_manager_obj = \
2258                                                 copy.deepcopy(dep_settings._mask_manager)
2259                                         dep_settings._mask_manager._pmaskdict.clear()
2260                                 arch_caches[prof.sub_path] = dep_settings
2261
2262                         xmatch_cache_key = (prof.sub_path, tuple(groups))
2263                         xcache = arch_xmatch_caches.get(xmatch_cache_key)
2264                         if xcache is None:
2265                                 portdb.melt()
2266                                 portdb.freeze()
2267                                 xcache = portdb.xcache
2268                                 xcache.update(shared_xmatch_caches)
2269                                 arch_xmatch_caches[xmatch_cache_key] = xcache
2270
2271                         trees[root]["porttree"].settings = dep_settings
2272                         portdb.settings = dep_settings
2273                         portdb.xcache = xcache
2274
2275                         dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2276                         # just in case, prevent config.reset() from nuking these.
2277                         dep_settings.backup_changes("ACCEPT_KEYWORDS")
2278
2279                         # This attribute is used in dbapi._match_use() to apply
2280                         # use.stable.{mask,force} settings based on the stable
2281                         # status of the parent package. This is required in order
2282                         # for USE deps of unstable packages to be resolved correctly,
2283                         # since otherwise use.stable.{mask,force} settings of
2284                         # dependencies may conflict (see bug #456342).
2285                         dep_settings._parent_stable = dep_settings._isStable(pkg)
2286
2287                         # Handle package.use*.{force,mask) calculation, for use
2288                         # in dep_check.
2289                         dep_settings.useforce = dep_settings._use_manager.getUseForce(
2290                                 pkg, stable=dep_settings._parent_stable)
2291                         dep_settings.usemask = dep_settings._use_manager.getUseMask(
2292                                 pkg, stable=dep_settings._parent_stable)
2293
2294                         if not baddepsyntax:
2295                                 ismasked = not ebuild_archs or \
2296                                         pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2297                                 if ismasked:
2298                                         if not have_pmasked:
2299                                                 have_pmasked = bool(dep_settings._getMaskAtom(
2300                                                         pkg.cpv, pkg._metadata))
2301                                         if options.ignore_masked:
2302                                                 continue
2303                                         # we are testing deps for a masked package; give it some lee-way
2304                                         suffix = "masked"
2305                                         matchmode = "minimum-all"
2306                                 else:
2307                                         suffix = ""
2308                                         matchmode = "minimum-visible"
2309
2310                                 if not have_dev_keywords:
2311                                         have_dev_keywords = \
2312                                                 bool(dev_keywords.intersection(keywords))
2313
2314                                 if prof.status == "dev":
2315                                         suffix = suffix + "indev"
2316
2317                                 for mytype in Package._dep_keys:
2318
2319                                         mykey = "dependency.bad" + suffix
2320                                         myvalue = myaux[mytype]
2321                                         if not myvalue:
2322                                                 continue
2323
2324                                         success, atoms = portage.dep_check(myvalue, portdb,
2325                                                 dep_settings, use="all", mode=matchmode,
2326                                                 trees=trees)
2327
2328                                         if success:
2329                                                 if atoms:
2330
2331                                                         # Don't bother with dependency.unknown for
2332                                                         # cases in which *DEPEND.bad is triggered.
2333                                                         for atom in atoms:
2334                                                                 # dep_check returns all blockers and they
2335                                                                 # aren't counted for *DEPEND.bad, so we
2336                                                                 # ignore them here.
2337                                                                 if not atom.blocker:
2338                                                                         unknown_pkgs.discard(
2339                                                                                 (mytype, atom.unevaluated_atom))
2340
2341                                                         if not prof.sub_path:
2342                                                                 # old-style virtuals currently aren't
2343                                                                 # resolvable with empty profile, since
2344                                                                 # 'virtuals' mappings are unavailable
2345                                                                 # (it would be expensive to search
2346                                                                 # for PROVIDE in all ebuilds)
2347                                                                 atoms = [atom for atom in atoms if not \
2348                                                                         (atom.cp.startswith('virtual/') and \
2349                                                                         not portdb.cp_list(atom.cp))]
2350
2351                                                         # we have some unsolvable deps
2352                                                         # remove ! deps, which always show up as unsatisfiable
2353                                                         atoms = [str(atom.unevaluated_atom) \
2354                                                                 for atom in atoms if not atom.blocker]
2355
2356                                                         # if we emptied out our list, continue:
2357                                                         if not atoms:
2358                                                                 continue
2359                                                         stats[mykey] += 1
2360                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2361                                                                 (relative_path, mytype, keyword,
2362                                                                 prof, repr(atoms)))
2363                                         else:
2364                                                 stats[mykey] += 1
2365                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2366                                                         (relative_path, mytype, keyword,
2367                                                         prof, repr(atoms)))
2368
2369                 if not baddepsyntax and unknown_pkgs:
2370                         type_map = {}
2371                         for mytype, atom in unknown_pkgs:
2372                                 type_map.setdefault(mytype, set()).add(atom)
2373                         for mytype, atoms in type_map.items():
2374                                 stats["dependency.unknown"] += 1
2375                                 fails["dependency.unknown"].append("%s: %s: %s" %
2376                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2377
2378         # check if there are unused local USE-descriptions in metadata.xml
2379         # (unless there are any invalids, to avoid noise)
2380         if allvalid:
2381                 for myflag in muselist.difference(used_useflags):
2382                         stats["metadata.warning"] += 1
2383                         fails["metadata.warning"].append(
2384                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2385                                 (x, myflag))
2386
2387 if options.if_modified == "y" and len(effective_scanlist) < 1:
2388         logging.warn("--if-modified is enabled, but no modified packages were found!")
2389
2390 if options.mode == "manifest":
2391         sys.exit(dofail)
2392
2393 # dofail will be set to 1 if we have failed in at least one non-warning category
2394 dofail = 0
2395 # dowarn will be set to 1 if we tripped any warnings
2396 dowarn = 0
2397 # dofull will be set if we should print a "repoman full" informational message
2398 dofull = options.mode != 'full'
2399
2400 for x in qacats:
2401         if not stats[x]:
2402                 continue
2403         dowarn = 1
2404         if x not in qawarnings:
2405                 dofail = 1
2406
2407 if dofail or \
2408         (dowarn and not (options.quiet or options.mode == "scan")):
2409         dofull = 0
2410
2411 # Save QA output so that it can be conveniently displayed
2412 # in $EDITOR while the user creates a commit message.
2413 # Otherwise, the user would not be able to see this output
2414 # once the editor has taken over the screen.
2415 qa_output = io.StringIO()
2416 style_file = ConsoleStyleFile(sys.stdout)
2417 if options.mode == 'commit' and \
2418         (not commitmessage or not commitmessage.strip()):
2419         style_file.write_listener = qa_output
2420 console_writer = StyleWriter(file=style_file, maxcol=9999)
2421 console_writer.style_listener = style_file.new_styles
2422
2423 f = formatter.AbstractFormatter(console_writer)
2424
2425 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2426
2427 style_file.flush()
2428 del console_writer, f, style_file
2429 qa_output = qa_output.getvalue()
2430 qa_output = qa_output.splitlines(True)
2431
2432 suggest_ignore_masked = False
2433 suggest_include_dev = False
2434
2435 if have_pmasked and not (options.without_mask or options.ignore_masked):
2436         suggest_ignore_masked = True
2437 if have_dev_keywords and not options.include_dev:
2438         suggest_include_dev = True
2439
2440 if suggest_ignore_masked or suggest_include_dev:
2441         print()
2442         if suggest_ignore_masked:
2443                 print(bold("Note: use --without-mask to check " + \
2444                         "KEYWORDS on dependencies of masked packages"))
2445
2446         if suggest_include_dev:
2447                 print(bold("Note: use --include-dev (-d) to check " + \
2448                         "dependencies for 'dev' profiles"))
2449         print()
2450
2451 if options.mode != 'commit':
2452         if dofull:
2453                 print(bold("Note: type \"repoman full\" for a complete listing."))
2454         if dowarn and not dofail:
2455                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2456         elif not dofail:
2457                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2458         elif dofail:
2459                 print(bad("Please fix these important QA issues first."))
2460                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2461                 sys.exit(1)
2462 else:
2463         if dofail and can_force and options.force and not options.pretend:
2464                 print(green("RepoMan sez:") + \
2465                         " \"You want to commit even with these QA issues?\n" + \
2466                         "              I'll take it this time, but I'm not happy.\"\n")
2467         elif dofail:
2468                 if options.force and not can_force:
2469                         print(bad("The --force option has been disabled due to extraordinary issues."))
2470                 print(bad("Please fix these important QA issues first."))
2471                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2472                 sys.exit(1)
2473
2474         if options.pretend:
2475                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2476
2477         myunadded = []
2478         if vcs == "cvs":
2479                 try:
2480                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2481                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2482                 except SystemExit as e:
2483                         raise  # TODO propagate this
2484                 except:
2485                         err("Error retrieving CVS tree; exiting.")
2486         if vcs == "svn":
2487                 try:
2488                         with repoman_popen("svn status --no-ignore") as f:
2489                                 svnstatus = f.readlines()
2490                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2491                 except SystemExit as e:
2492                         raise  # TODO propagate this
2493                 except:
2494                         err("Error retrieving SVN info; exiting.")
2495         if vcs == "git":
2496                 # get list of files not under version control or missing
2497                 myf = repoman_popen("git ls-files --others")
2498                 myunadded = ["./" + elem[:-1] for elem in myf]
2499                 myf.close()
2500         if vcs == "bzr":
2501                 try:
2502                         with repoman_popen("bzr status -S .") as f:
2503                                 bzrstatus = f.readlines()
2504                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2505                 except SystemExit as e:
2506                         raise  # TODO propagate this
2507                 except:
2508                         err("Error retrieving bzr info; exiting.")
2509         if vcs == "hg":
2510                 with repoman_popen("hg status --no-status --unknown .") as f:
2511                         myunadded = f.readlines()
2512                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2513
2514                 # Mercurial doesn't handle manually deleted files as removed from
2515                 # the repository, so the user need to remove them before commit,
2516                 # using "hg remove [FILES]"
2517                 with repoman_popen("hg status --no-status --deleted .") as f:
2518                         mydeleted = f.readlines()
2519                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2520
2521
2522         myautoadd = []
2523         if myunadded:
2524                 for x in range(len(myunadded)-1, -1, -1):
2525                         xs = myunadded[x].split("/")
2526                         if xs[-1] == "files":
2527                                 print("!!! files dir is not added! Please correct this.")
2528                                 sys.exit(-1)
2529                         elif xs[-1] == "Manifest":
2530                                 # It's a manifest... auto add
2531                                 myautoadd += [myunadded[x]]
2532                                 del myunadded[x]
2533
2534         if myunadded:
2535                 print(red("!!! The following files are in your local tree but are not added to the master"))
2536                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2537                 for x in myunadded:
2538                         print("   ", x)
2539                 print()
2540                 print()
2541                 sys.exit(1)
2542
2543         if vcs == "hg" and mydeleted:
2544                 print(red("!!! The following files are removed manually from your local tree but are not"))
2545                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2546                 for x in mydeleted:
2547                         print("   ", x)
2548                 print()
2549                 print()
2550                 sys.exit(1)
2551
2552         if vcs == "cvs":
2553                 mycvstree = cvstree.getentries("./", recursive=1)
2554                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2555                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2556                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2557                 bin_blob_pattern = re.compile("^-kb$")
2558                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2559                         recursive=1, basedir="./"))
2560
2561         if vcs == "svn":
2562                 with repoman_popen("svn status") as f:
2563                         svnstatus = f.readlines()
2564                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2565                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2566                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2567
2568                 # Subversion expands keywords specified in svn:keywords properties.
2569                 with repoman_popen("svn propget -R svn:keywords") as f:
2570                         props = f.readlines()
2571                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2572                         for prop in props if " - " in prop)
2573
2574         elif vcs == "git":
2575                 with repoman_popen("git diff-index --name-only "
2576                         "--relative --diff-filter=M HEAD") as f:
2577                         mychanged = f.readlines()
2578                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2579
2580                 with repoman_popen("git diff-index --name-only "
2581                         "--relative --diff-filter=A HEAD") as f:
2582                         mynew = f.readlines()
2583                 mynew = ["./" + elem[:-1] for elem in mynew]
2584
2585                 with repoman_popen("git diff-index --name-only "
2586                         "--relative --diff-filter=D HEAD") as f:
2587                         myremoved = f.readlines()
2588                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2589
2590         if vcs == "bzr":
2591                 with repoman_popen("bzr status -S .") as f:
2592                         bzrstatus = f.readlines()
2593                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2594                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2595                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2596                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2597                 # Bazaar expands nothing.
2598
2599         if vcs == "hg":
2600                 with repoman_popen("hg status --no-status --modified .") as f:
2601                         mychanged = f.readlines()
2602                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2603
2604                 with repoman_popen("hg status --no-status --added .") as f:
2605                         mynew = f.readlines()
2606                 mynew = ["./" + elem.rstrip() for elem in mynew]
2607
2608                 with repoman_popen("hg status --no-status --removed .") as f:
2609                         myremoved = f.readlines()
2610                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2611
2612         if vcs:
2613                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2614                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2615                         print()
2616                         print("(Didn't find any changed files...)")
2617                         print()
2618                         sys.exit(1)
2619
2620         # Manifests need to be regenerated after all other commits, so don't commit
2621         # them now even if they have changed.
2622         mymanifests = set()
2623         myupdates = set()
2624         for f in mychanged + mynew:
2625                 if "Manifest" == os.path.basename(f):
2626                         mymanifests.add(f)
2627                 else:
2628                         myupdates.add(f)
2629         myupdates.difference_update(myremoved)
2630         myupdates = list(myupdates)
2631         mymanifests = list(mymanifests)
2632         myheaders = []
2633         mydirty = []
2634
2635         commitmessage = options.commitmsg
2636         if options.commitmsgfile:
2637                 try:
2638                         f = io.open(_unicode_encode(options.commitmsgfile,
2639                         encoding=_encodings['fs'], errors='strict'),
2640                         mode='r', encoding=_encodings['content'], errors='replace')
2641                         commitmessage = f.read()
2642                         f.close()
2643                         del f
2644                 except (IOError, OSError) as e:
2645                         if e.errno == errno.ENOENT:
2646                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2647                         else:
2648                                 raise
2649                 # We've read the content so the file is no longer needed.
2650                 commitmessagefile = None
2651         if not commitmessage or not commitmessage.strip():
2652                 try:
2653                         editor = os.environ.get("EDITOR")
2654                         if editor and utilities.editor_is_executable(editor):
2655                                 commitmessage = utilities.get_commit_message_with_editor(
2656                                         editor, message=qa_output)
2657                         else:
2658                                 commitmessage = utilities.get_commit_message_with_stdin()
2659                 except KeyboardInterrupt:
2660                         exithandler()
2661                 if not commitmessage or not commitmessage.strip():
2662                         print("* no commit message?  aborting commit.")
2663                         sys.exit(1)
2664         commitmessage = commitmessage.rstrip()
2665         changelog_msg = commitmessage
2666         portage_version = getattr(portage, "VERSION", None)
2667         gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
2668         dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
2669         if portage_version is None:
2670                 sys.stderr.write("Failed to insert portage version in message!\n")
2671                 sys.stderr.flush()
2672                 portage_version = "Unknown"
2673
2674         report_options = []
2675         if options.force:
2676                 report_options.append("--force")
2677         if options.ignore_arches:
2678                 report_options.append("--ignore-arches")
2679         if include_arches is not None:
2680                 report_options.append("--include-arches=\"%s\"" %
2681                         " ".join(sorted(include_arches)))
2682
2683         if vcs == "git":
2684                 # Use new footer only for git (see bug #438364).
2685                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2686                 if report_options:
2687                         commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
2688                 if sign_manifests:
2689                         commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
2690                 if dco_sob:
2691                         commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
2692         else:
2693                 unameout = platform.system() + " "
2694                 if platform.system() in ["Darwin", "SunOS"]:
2695                         unameout += platform.processor()
2696                 else:
2697                         unameout += platform.machine()
2698                 commit_footer = "\n\n"
2699                 if dco_sob:
2700                         commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
2701                 commit_footer += "(Portage version: %s/%s/%s" % \
2702                         (portage_version, vcs, unameout)
2703                 if report_options:
2704                         commit_footer += ", RepoMan options: " + " ".join(report_options)
2705                 if sign_manifests:
2706                         commit_footer += ", signed Manifest commit with key %s" % \
2707                                 (gpg_key, )
2708                 else:
2709                         commit_footer += ", unsigned Manifest commit"
2710                 commit_footer += ")"
2711
2712         commitmessage += commit_footer
2713
2714         broken_changelog_manifests = []
2715         if options.echangelog in ('y', 'force'):
2716                 logging.info("checking for unmodified ChangeLog files")
2717                 committer_name = utilities.get_committer_name(env=repoman_settings)
2718                 for x in sorted(vcs_files_to_cps(
2719                         chain(myupdates, mymanifests, myremoved))):
2720                         catdir, pkgdir = x.split("/")
2721                         checkdir = repodir + "/" + x
2722                         checkdir_relative = ""
2723                         if repolevel < 3:
2724                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2725                         if repolevel < 2:
2726                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2727                         checkdir_relative = os.path.join(".", checkdir_relative)
2728
2729                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2730                         changelog_modified = changelog_path in modified_changelogs
2731                         if changelog_modified and options.echangelog != 'force':
2732                                 continue
2733
2734                         # get changes for this package
2735                         cdrlen = len(checkdir_relative)
2736                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2737                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2738                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2739
2740                         # Skip ChangeLog generation if only the Manifest was modified,
2741                         # as discussed in bug #398009.
2742                         nontrivial_cl_files = set()
2743                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2744                         nontrivial_cl_files.difference_update(['Manifest'])
2745                         if not nontrivial_cl_files and options.echangelog != 'force':
2746                                 continue
2747
2748                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2749                                 committer_name, changelog_msg,
2750                                 os.path.join(repodir, 'skel.ChangeLog'),
2751                                 catdir, pkgdir,
2752                                 new=clnew, removed=clremoved, changed=clchanged,
2753                                 pretend=options.pretend)
2754                         if new_changelog is None:
2755                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2756                                         level=logging.ERROR, noiselevel=-1)
2757                                 sys.exit(1)
2758
2759                         # if the ChangeLog was just created, add it to vcs
2760                         if new_changelog:
2761                                 myautoadd.append(changelog_path)
2762                                 # myautoadd is appended to myupdates below
2763                         else:
2764                                 myupdates.append(changelog_path)
2765
2766                         if options.ask and not options.pretend:
2767                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2768                                 repoman_settings["O"] = checkdir
2769                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2770                         else:
2771                                 broken_changelog_manifests.append(x)
2772
2773         if myautoadd:
2774                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2775                 add_cmd = [vcs, "add"]
2776                 add_cmd += myautoadd
2777                 if options.pretend:
2778                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2779                                 noiselevel=-1)
2780                 else:
2781
2782                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2783                                 not os.path.isabs(add_cmd[0]):
2784                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2785                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2786                                 fullname = find_binary(add_cmd[0])
2787                                 if fullname is None:
2788                                         raise portage.exception.CommandNotFound(add_cmd[0])
2789                                 add_cmd[0] = fullname
2790
2791                         add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2792                         retcode = subprocess.call(add_cmd)
2793                         if retcode != os.EX_OK:
2794                                 logging.error(
2795                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2796                                 sys.exit(retcode)
2797
2798                 myupdates += myautoadd
2799
2800         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2801
2802         if vcs not in ('cvs', 'svn'):
2803                 # With git, bzr and hg, there's never any keyword expansion, so
2804                 # there's no need to regenerate manifests and all files will be
2805                 # committed in one big commit at the end.
2806                 print()
2807         elif not repo_config.thin_manifest:
2808                 if vcs == 'cvs':
2809                         headerstring = "'\$(Header|Id).*\$'"
2810                 elif vcs == "svn":
2811                         svn_keywords = dict((k.lower(), k) for k in [
2812                                         "Rev",
2813                                         "Revision",
2814                                         "LastChangedRevision",
2815                                         "Date",
2816                                         "LastChangedDate",
2817                                         "Author",
2818                                         "LastChangedBy",
2819                                         "URL",
2820                                         "HeadURL",
2821                                         "Id",
2822                                         "Header",
2823                         ])
2824
2825                 for myfile in myupdates:
2826
2827                         # for CVS, no_expansion contains files that are excluded from expansion
2828                         if vcs == "cvs":
2829                                 if myfile in no_expansion:
2830                                         continue
2831
2832                         # for SVN, expansion contains files that are included in expansion
2833                         elif vcs == "svn":
2834                                 if myfile not in expansion:
2835                                         continue
2836
2837                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2838                                 enabled_keywords = []
2839                                 for k in expansion[myfile]:
2840                                         keyword = svn_keywords.get(k.lower())
2841                                         if keyword is not None:
2842                                                 enabled_keywords.append(keyword)
2843
2844                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2845
2846                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2847                                 portage._shell_quote(myfile))
2848                         if myout[0] == 0:
2849                                 myheaders.append(myfile)
2850
2851                 print("%s have headers that will change." % green(str(len(myheaders))))
2852                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2853
2854         logging.info("myupdates: %s", myupdates)
2855         logging.info("myheaders: %s", myheaders)
2856
2857         if options.ask and userquery('Commit changes?', True) != 'Yes':
2858                 print("* aborting commit.")
2859                 sys.exit(128 + signal.SIGINT)
2860
2861         # Handle the case where committed files have keywords which
2862         # will change and need a priming commit before the Manifest
2863         # can be committed.
2864         if (myupdates or myremoved) and myheaders:
2865                 myfiles = myupdates + myremoved
2866                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2867                 mymsg = os.fdopen(fd, "wb")
2868                 mymsg.write(_unicode_encode(commitmessage))
2869                 mymsg.close()
2870
2871                 print()
2872                 print(green("Using commit message:"))
2873                 print(green("------------------------------------------------------------------------------"))
2874                 print(commitmessage)
2875                 print(green("------------------------------------------------------------------------------"))
2876                 print()
2877
2878                 # Having a leading ./ prefix on file paths can trigger a bug in
2879                 # the cvs server when committing files to multiple directories,
2880                 # so strip the prefix.
2881                 myfiles = [f.lstrip("./") for f in myfiles]
2882
2883                 commit_cmd = [vcs]
2884                 commit_cmd.extend(vcs_global_opts)
2885                 commit_cmd.append("commit")
2886                 commit_cmd.extend(vcs_local_opts)
2887                 commit_cmd.extend(["-F", commitmessagefile])
2888                 commit_cmd.extend(myfiles)
2889
2890                 try:
2891                         if options.pretend:
2892                                 print("(%s)" % (" ".join(commit_cmd),))
2893                         else:
2894                                 retval = spawn(commit_cmd, env=commit_env)
2895                                 if retval != os.EX_OK:
2896                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2897                                                 "error code: %s\n") % (vcs, retval),
2898                                                 level=logging.ERROR, noiselevel=-1)
2899                                         sys.exit(retval)
2900                 finally:
2901                         try:
2902                                 os.unlink(commitmessagefile)
2903                         except OSError:
2904                                 pass
2905
2906         # Setup the GPG commands
2907         def gpgsign(filename):
2908                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2909                 if gpgcmd is None:
2910                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2911                                 " Is make.globals missing?")
2912                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2913                         "PORTAGE_GPG_KEY" not in repoman_settings:
2914                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2915                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2916                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2917                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2918                                         os.path.expanduser("~/.gnupg")
2919                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2920                                         % repoman_settings["PORTAGE_GPG_DIR"])
2921                         else:
2922                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2923                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2924                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2925                                 raise portage.exception.InvalidLocation(
2926                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2927                                         repoman_settings["PORTAGE_GPG_DIR"])
2928                 gpgvars = {"FILE": filename}
2929                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2930                         v = repoman_settings.get(k)
2931                         if v is not None:
2932                                 gpgvars[k] = v
2933                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2934                 if options.pretend:
2935                         print("(" + gpgcmd + ")")
2936                 else:
2937                         # Encode unicode manually for bug #310789.
2938                         gpgcmd = portage.util.shlex_split(gpgcmd)
2939
2940                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2941                                 not os.path.isabs(gpgcmd[0]):
2942                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2943                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2944                                 fullname = find_binary(gpgcmd[0])
2945                                 if fullname is None:
2946                                         raise portage.exception.CommandNotFound(gpgcmd[0])
2947                                 gpgcmd[0] = fullname
2948
2949                         gpgcmd = [_unicode_encode(arg,
2950                                 encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2951                         rValue = subprocess.call(gpgcmd)
2952                         if rValue == os.EX_OK:
2953                                 os.rename(filename + ".asc", filename)
2954                         else:
2955                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2956
2957         def need_signature(filename):
2958                 try:
2959                         with open(_unicode_encode(filename,
2960                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2961                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2962                 except IOError as e:
2963                         if e.errno in (errno.ENOENT, errno.ESTALE):
2964                                 return False
2965                         raise
2966
2967         # When files are removed and re-added, the cvs server will put /Attic/
2968         # inside the $Header path. This code detects the problem and corrects it
2969         # so that the Manifest will generate correctly. See bug #169500.
2970         # Use binary mode in order to avoid potential character encoding issues.
2971         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2972         attic_str = b'/Attic/'
2973         attic_replace = b'/'
2974         for x in myheaders:
2975                 f = open(_unicode_encode(x,
2976                         encoding=_encodings['fs'], errors='strict'),
2977                         mode='rb')
2978                 mylines = f.readlines()
2979                 f.close()
2980                 modified = False
2981                 for i, line in enumerate(mylines):
2982                         if cvs_header_re.match(line) is not None and \
2983                                 attic_str in line:
2984                                 mylines[i] = line.replace(attic_str, attic_replace)
2985                                 modified = True
2986                 if modified:
2987                         portage.util.write_atomic(x, b''.join(mylines),
2988                                 mode='wb')
2989
2990         if repolevel == 1:
2991                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2992                         "doing the entire repository.\"\n")
2993
2994         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2995
2996                 for x in sorted(vcs_files_to_cps(
2997                         chain(myupdates, myremoved, mymanifests))):
2998                         repoman_settings["O"] = os.path.join(repodir, x)
2999                         digestgen(mysettings=repoman_settings, myportdb=portdb)
3000
3001         elif broken_changelog_manifests:
3002                 for x in broken_changelog_manifests:
3003                         repoman_settings["O"] = os.path.join(repodir, x)
3004                         digestgen(mysettings=repoman_settings, myportdb=portdb)
3005
3006         signed = False
3007         if sign_manifests:
3008                 signed = True
3009                 try:
3010                         for x in sorted(vcs_files_to_cps(
3011                                 chain(myupdates, myremoved, mymanifests))):
3012                                 repoman_settings["O"] = os.path.join(repodir, x)
3013                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
3014                                 if not need_signature(manifest_path):
3015                                         continue
3016                                 gpgsign(manifest_path)
3017                 except portage.exception.PortageException as e:
3018                         portage.writemsg("!!! %s\n" % str(e))
3019                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
3020                         signed = False
3021
3022         if vcs == 'git':
3023                 # It's not safe to use the git commit -a option since there might
3024                 # be some modified files elsewhere in the working tree that the
3025                 # user doesn't want to commit. Therefore, call git update-index
3026                 # in order to ensure that the index is updated with the latest
3027                 # versions of all new and modified files in the relevant portion
3028                 # of the working tree.
3029                 myfiles = mymanifests + myupdates
3030                 myfiles.sort()
3031                 update_index_cmd = ["git", "update-index"]
3032                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
3033                 if options.pretend:
3034                         print("(%s)" % (" ".join(update_index_cmd),))
3035                 else:
3036                         retval = spawn(update_index_cmd, env=os.environ)
3037                         if retval != os.EX_OK:
3038                                 writemsg_level(("!!! Exiting on %s (shell) " + \
3039                                         "error code: %s\n") % (vcs, retval),
3040                                         level=logging.ERROR, noiselevel=-1)
3041                                 sys.exit(retval)
3042
3043         if True:
3044                 myfiles = mymanifests[:]
3045                 # If there are no header (SVN/CVS keywords) changes in
3046                 # the files, this Manifest commit must include the
3047                 # other (yet uncommitted) files.
3048                 if not myheaders:
3049                         myfiles += myupdates
3050                         myfiles += myremoved
3051                 myfiles.sort()
3052
3053                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3054                 mymsg = os.fdopen(fd, "wb")
3055                 mymsg.write(_unicode_encode(commitmessage))
3056                 mymsg.close()
3057
3058                 commit_cmd = []
3059                 if options.pretend and vcs is None:
3060                         # substitute a bogus value for pretend output
3061                         commit_cmd.append("cvs")
3062                 else:
3063                         commit_cmd.append(vcs)
3064                 commit_cmd.extend(vcs_global_opts)
3065                 commit_cmd.append("commit")
3066                 commit_cmd.extend(vcs_local_opts)
3067                 if vcs == "hg":
3068                         commit_cmd.extend(["--logfile", commitmessagefile])
3069                         commit_cmd.extend(myfiles)
3070                 else:
3071                         commit_cmd.extend(["-F", commitmessagefile])
3072                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3073
3074                 try:
3075                         if options.pretend:
3076                                 print("(%s)" % (" ".join(commit_cmd),))
3077                         else:
3078                                 retval = spawn(commit_cmd, env=commit_env)
3079                                 if retval != os.EX_OK:
3080                                         if repo_config.sign_commit and vcs == 'git' and \
3081                                                 not git_supports_gpg_sign():
3082                                                 # Inform user that newer git is needed (bug #403323).
3083                                                 logging.error(
3084                                                         "Git >=1.7.9 is required for signed commits!")
3085
3086                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3087                                                 "error code: %s\n") % (vcs, retval),
3088                                                 level=logging.ERROR, noiselevel=-1)
3089                                         sys.exit(retval)
3090                 finally:
3091                         try:
3092                                 os.unlink(commitmessagefile)
3093                         except OSError:
3094                                 pass
3095
3096         print()
3097         if vcs:
3098                 print("Commit complete.")
3099         else:
3100                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3101         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3102 sys.exit(0)