bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import codecs
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import re
  18 import signal
  19 import stat
  20 import subprocess
  21 import sys
  22 import tempfile
  23 import textwrap
  24 import time
  25 import platform
  26 from itertools import chain
  27 from stat import S_ISDIR
  28
  29 try:
  30         from urllib.parse import urlparse
  31 except ImportError:
  32         from urlparse import urlparse
  33
  34 from os import path as osp
  35 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  36 sys.path.insert(0, pym_path)
  37 import portage
  38 portage._internal_caller = True
  39 portage._disable_legacy_globals()
  40
  41 try:
  42         import xml.etree.ElementTree
  43         from xml.parsers.expat import ExpatError
  44 except (SystemExit, KeyboardInterrupt):
  45         raise
  46 except (ImportError, SystemError, RuntimeError, Exception):
  47         # broken or missing xml support
  48         # http://bugs.python.org/issue14988
  49         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  50         from portage.output import EOutput
  51         out = EOutput()
  52         for line in msg:
  53                 out.eerror(line)
  54         sys.exit(1)
  55
  56 from portage import os
  57 from portage import _encodings
  58 from portage import _unicode_encode
  59 import repoman.checks
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 import portage.repository.config
  69 from portage import cvstree, normalize_path
  70 from portage import util
  71 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  72         ParseError, PermissionDenied)
  73 from portage.dep import Atom
  74 from portage.process import find_binary, spawn
  75 from portage.output import bold, create_color_func, \
  76         green, nocolor, red
  77 from portage.output import ConsoleStyleFile, StyleWriter
  78 from portage.util import writemsg_level
  79 from portage.util._argparse import ArgumentParser
  80 from portage.package.ebuild.digestgen import digestgen
  81 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  82
  83 if sys.hexversion >= 0x3000000:
  84         basestring = str
  85
  86 util.initialize_logger()
  87
  88 # 14 is the length of DESCRIPTION=""
  89 max_desc_len = 100
  90 allowed_filename_chars="a-zA-Z0-9._-+:"
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  93 bad = create_color_func("BAD")
  94
  95 # A sane umask is needed for files that portage creates.
  96 os.umask(0o22)
  97 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  98 # behave incrementally.
  99 repoman_incrementals = tuple(x for x in \
 100         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 101 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 102 repoman_settings = portage.config(config_root=config_root, local_config=False)
 103
 104 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 105         repoman_settings.get('TERM') == 'dumb' or \
 106         not sys.stdout.isatty():
 107         nocolor()
 108
 109 def warn(txt):
 110         print("repoman: " + txt)
 111
 112 def err(txt):
 113         warn(txt)
 114         sys.exit(1)
 115
 116 def exithandler(signum=None, _frame=None):
 117         logging.fatal("Interrupted; exiting...")
 118         if signum is None:
 119                 sys.exit(1)
 120         else:
 121                 sys.exit(128 + signum)
 122
 123 signal.signal(signal.SIGINT, exithandler)
 124
 125 def ParseArgs(argv, qahelp):
 126         """This function uses a customized optionParser to parse command line arguments for repoman
 127         Args:
 128           argv - a sequence of command line arguments
 129                 qahelp - a dict of qa warning to help message
 130         Returns:
 131           (opts, args), just like a call to parser.parse_args()
 132         """
 133
 134         argv = portage._decode_argv(argv)
 135
 136         modes = {
 137                 'commit' : 'Run a scan then commit changes',
 138                 'ci' : 'Run a scan then commit changes',
 139                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 140                 'full' : 'Scan directory tree and print all issues (not a summary)',
 141                 'help' : 'Show this screen',
 142                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 143                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 144                 'scan' : 'Scan directory tree for QA issues'
 145         }
 146
 147         mode_keys = list(modes)
 148         mode_keys.sort()
 149
 150         parser = ArgumentParser(usage="repoman [options] [mode]",
 151                 description="Modes: %s" % " | ".join(mode_keys),
 152                 epilog="For more help consult the man page.")
 153
 154         parser.add_argument('-a', '--ask', dest='ask', action='store_true', default=False,
 155                 help='Request a confirmation before commiting')
 156
 157         parser.add_argument('-m', '--commitmsg', dest='commitmsg',
 158                 help='specify a commit message on the command line')
 159
 160         parser.add_argument('-M', '--commitmsgfile', dest='commitmsgfile',
 161                 help='specify a path to a file that contains a commit message')
 162
 163         parser.add_argument('--digest',
 164                 choices=('y', 'n'), metavar='<y|n>',
 165                 help='Automatically update Manifest digests for modified files')
 166
 167         parser.add_argument('-p', '--pretend', dest='pretend', default=False,
 168                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 169
 170         parser.add_argument('-q', '--quiet', dest="quiet", action="count", default=0,
 171                 help='do not print unnecessary messages')
 172
 173         parser.add_argument(
 174                 '--echangelog', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 175                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 176                 'regardless of modification if \'force\' is specified)')
 177
 178         parser.add_argument('--experimental-inherit', choices=('y', 'n'),
 179                 metavar="<y|n>", default='n',
 180                 help='Enable experimental inherit.missing checks which may misbehave'
 181                         ' when the internal eclass database becomes outdated')
 182
 183         parser.add_argument('-f', '--force', dest='force', default=False, action='store_true',
 184                 help='Commit with QA violations')
 185
 186         parser.add_argument('--vcs', dest='vcs',
 187                 help='Force using specific VCS instead of autodetection')
 188
 189         parser.add_argument('-v', '--verbose', dest="verbosity", action='count',
 190                 help='be very verbose in output', default=0)
 191
 192         parser.add_argument('-V', '--version', dest='version', action='store_true',
 193                 help='show version info')
 194
 195         parser.add_argument('-x', '--xmlparse', dest='xml_parse', action='store_true',
 196                 default=False, help='forces the metadata.xml parse check to be carried out')
 197
 198         parser.add_argument(
 199                 '--if-modified', choices=('y', 'n'), default='n',
 200                 metavar="<y|n>",
 201                 help='only check packages that have uncommitted modifications')
 202
 203         parser.add_argument('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 204                 default=False, help='ignore arch-specific failures (where arch != host)')
 205
 206         parser.add_argument("--ignore-default-opts",
 207                 action="store_true",
 208                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 209
 210         parser.add_argument('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 211                 default=False, help='ignore masked packages (not allowed with commit mode)')
 212
 213         parser.add_argument('--include-arches', dest='include_arches',
 214                 metavar='ARCHES', action='append',
 215                 help='A space separated list of arches used to '
 216                 'filter the selection of profiles for dependency checks')
 217
 218         parser.add_argument('-d', '--include-dev', dest='include_dev', action='store_true',
 219                 default=False, help='include dev profiles in dependency checks')
 220
 221         parser.add_argument('-e', '--include-exp-profiles', choices=('y', 'n'),
 222                 default=False, help='include exp profiles in dependency checks',
 223                 metavar='<y|n>')
 224
 225         parser.add_argument('--unmatched-removal', dest='unmatched_removal', action='store_true',
 226                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 227
 228         parser.add_argument('--without-mask', dest='without_mask', action='store_true',
 229                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 230
 231         parser.add_argument('--mode', dest='mode', choices=mode_keys,
 232                 help='specify which mode repoman will run in (default=full)')
 233
 234         opts, args = parser.parse_known_args(argv[1:])
 235
 236         if not opts.ignore_default_opts:
 237                 default_opts = portage.util.shlex_split(
 238                         repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
 239                 if default_opts:
 240                         opts, args = parser.parse_known_args(default_opts + sys.argv[1:])
 241
 242         if opts.mode == 'help':
 243                 parser.print_help(short=False)
 244
 245         for arg in args:
 246                 if arg in modes:
 247                         if not opts.mode:
 248                                 opts.mode = arg
 249                                 break
 250                 else:
 251                         parser.error("invalid mode: %s" % arg)
 252
 253         if not opts.mode:
 254                 opts.mode = 'full'
 255
 256         if opts.mode == 'ci':
 257                 opts.mode = 'commit'  # backwards compat shortcut
 258
 259         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 260         for val in range(opts.verbosity):
 261                 logger = logging.getLogger()
 262                 logger.setLevel(logger.getEffectiveLevel() - 10)
 263
 264         for val in range(opts.quiet):
 265                 logger = logging.getLogger()
 266                 logger.setLevel(logger.getEffectiveLevel() + 10)
 267
 268         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 269                 if opts.ignore_masked:
 270                         opts.ignore_masked = False
 271                         logging.warn('Commit mode automatically disables --ignore-masked')
 272                 if opts.without_mask:
 273                         opts.without_mask = False
 274                         logging.warn('Commit mode automatically disables --without-mask')
 275
 276         return (opts, args)
 277
 278 qahelp = {
 279         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 280         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 281         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 282         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 283         "changelog.missing": "Missing ChangeLog files",
 284         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 285         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 286         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 287         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 288         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 289         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 290         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 291         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 292         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 293         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 294         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 295         "file.size": "Files in the files directory must be under 20 KiB",
 296         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 297         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 298         "file.UTF8": "File is not UTF8 compliant",
 299         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 300         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 301         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 302         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 303         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 304         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 305         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 306         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 307         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 308         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 309         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 310         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 311         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 312         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 313         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 314         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 315         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 316         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 317         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 318         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 319         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 320         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 321         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 322         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 323         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 324         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 325         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 326         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 327         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 328         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 329         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 330         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 331         "variable.readonly": "Assigning a readonly variable",
 332         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 333         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 334         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 335         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 336         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 337         "IUSE.rubydeprecated": "The ebuild has set a ruby interpreter in USE_RUBY, that is not available as a ruby target anymore",
 338         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 339         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 340         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 341         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 342         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 343         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 344         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 345         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 346         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 347         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 348         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 349         "ebuild.badheader": "This ebuild has a malformed header",
 350         "manifest.bad": "Manifest has missing or incorrect digests",
 351         "metadata.missing": "Missing metadata.xml files",
 352         "metadata.bad": "Bad metadata.xml files",
 353         "metadata.warning": "Warnings in metadata.xml files",
 354         "portage.internal": "The ebuild uses an internal Portage function or variable",
 355         "repo.eapi.banned": "The ebuild uses an EAPI which is banned by the repository's metadata/layout.conf settings",
 356         "repo.eapi.deprecated": "The ebuild uses an EAPI which is deprecated by the repository's metadata/layout.conf settings",
 357         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 358         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 359         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 360         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 361 }
 362
 363 qacats = list(qahelp)
 364 qacats.sort()
 365
 366 qawarnings = set((
 367 "changelog.missing",
 368 "changelog.notadded",
 369 "dependency.unknown",
 370 "digest.assumed",
 371 "digest.unused",
 372 "ebuild.notadded",
 373 "ebuild.nesteddie",
 374 "dependency.badmasked",
 375 "dependency.badindev",
 376 "dependency.badmaskedindev",
 377 "dependency.badtilde",
 378 "DESCRIPTION.toolong",
 379 "EAPI.deprecated",
 380 "HOMEPAGE.virtual",
 381 "LICENSE.deprecated",
 382 "LICENSE.virtual",
 383 "KEYWORDS.dropped",
 384 "KEYWORDS.stupid",
 385 "KEYWORDS.missing",
 386 "PDEPEND.suspect",
 387 "RDEPEND.implicit",
 388 "RDEPEND.suspect",
 389 "virtual.suspect",
 390 "RESTRICT.invalid",
 391 "ebuild.minorsyn",
 392 "ebuild.badheader",
 393 "ebuild.patches",
 394 "file.size",
 395 "inherit.unused",
 396 "inherit.deprecated",
 397 "java.eclassesnotused",
 398 "wxwidgets.eclassnotused",
 399 "metadata.warning",
 400 "portage.internal",
 401 "repo.eapi.deprecated",
 402 "usage.obsolete",
 403 "upstream.workaround",
 404 "LIVEVCS.stable",
 405 "LIVEVCS.unmasked",
 406 "IUSE.rubydeprecated",
 407 ))
 408
 409 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 410
 411 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 412 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 413 allvars.update(Package.metadata_keys)
 414 allvars = sorted(allvars)
 415 commitmessage = None
 416 for x in missingvars:
 417         x += ".missing"
 418         if x not in qacats:
 419                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 420                 qacats.append(x)
 421                 qawarnings.add(x)
 422
 423 valid_restrict = frozenset(["binchecks", "bindist",
 424         "fetch", "installsources", "mirror", "preserve-libs",
 425         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 426
 427 live_eclasses = frozenset([
 428         "bzr",
 429         "cvs",
 430         "darcs",
 431         "git",
 432         "git-2",
 433         "mercurial",
 434         "subversion",
 435         "tla",
 436 ])
 437
 438 suspect_rdepend = frozenset([
 439         "app-arch/cabextract",
 440         "app-arch/rpm2targz",
 441         "app-doc/doxygen",
 442         "dev-lang/nasm",
 443         "dev-lang/swig",
 444         "dev-lang/yasm",
 445         "dev-perl/extutils-pkgconfig",
 446         "dev-util/byacc",
 447         "dev-util/cmake",
 448         "dev-util/ftjam",
 449         "dev-util/gperf",
 450         "dev-util/gtk-doc",
 451         "dev-util/gtk-doc-am",
 452         "dev-util/intltool",
 453         "dev-util/jam",
 454         "dev-util/pkg-config-lite",
 455         "dev-util/pkgconf",
 456         "dev-util/pkgconfig",
 457         "dev-util/pkgconfig-openbsd",
 458         "dev-util/scons",
 459         "dev-util/unifdef",
 460         "dev-util/yacc",
 461         "media-gfx/ebdftopcf",
 462         "sys-apps/help2man",
 463         "sys-devel/autoconf",
 464         "sys-devel/automake",
 465         "sys-devel/bin86",
 466         "sys-devel/bison",
 467         "sys-devel/dev86",
 468         "sys-devel/flex",
 469         "sys-devel/m4",
 470         "sys-devel/pmake",
 471         "virtual/linux-sources",
 472         "virtual/pkgconfig",
 473         "x11-misc/bdftopcf",
 474         "x11-misc/imake",
 475 ])
 476
 477 suspect_virtual = {
 478         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 479         "dev-util/pkgconf":"virtual/pkgconfig",
 480         "dev-util/pkgconfig":"virtual/pkgconfig",
 481         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 482         "dev-libs/libusb":"virtual/libusb",
 483         "dev-libs/libusbx":"virtual/libusb",
 484         "dev-libs/libusb-compat":"virtual/libusb",
 485 }
 486
 487 ruby_deprecated = frozenset([
 488         "ruby_targets_ree18",
 489 ])
 490
 491 metadata_xml_encoding = 'UTF-8'
 492 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 493         (metadata_xml_encoding,)
 494 metadata_doctype_name = 'pkgmetadata'
 495 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 496 # force refetch if the local copy creation time is older than this
 497 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 498
 499 # file.executable
 500 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 501
 502 options, arguments = ParseArgs(sys.argv, qahelp)
 503
 504 if options.version:
 505         print("Portage", portage.VERSION)
 506         sys.exit(0)
 507
 508 if options.experimental_inherit == 'y':
 509         # This is experimental, so it's non-fatal.
 510         qawarnings.add("inherit.missing")
 511         repoman.checks._init(experimental_inherit=True)
 512
 513 # Set this to False when an extraordinary issue (generally
 514 # something other than a QA issue) makes it impossible to
 515 # commit (like if Manifest generation fails).
 516 can_force = True
 517
 518 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 519 if portdir is None:
 520         sys.exit(1)
 521
 522 myreporoot = os.path.basename(portdir_overlay)
 523 myreporoot += mydir[len(portdir_overlay):]
 524
 525 if options.vcs:
 526         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 527                 vcs = options.vcs
 528         else:
 529                 vcs = None
 530 else:
 531         vcses = utilities.FindVCS()
 532         if len(vcses) > 1:
 533                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 534                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 535                 sys.exit(1)
 536         elif vcses:
 537                 vcs = vcses[0]
 538         else:
 539                 vcs = None
 540
 541 if options.if_modified == "y" and vcs is None:
 542         logging.info("Not in a version controlled repository; "
 543                 "disabling --if-modified.")
 544         options.if_modified = "n"
 545
 546 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 547 vcs_preserves_mtime = vcs in ('cvs',)
 548
 549 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 550 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 551 if vcs_global_opts is None:
 552         if vcs in ('cvs', 'svn'):
 553                 vcs_global_opts = "-q"
 554         else:
 555                 vcs_global_opts = ""
 556 vcs_global_opts = vcs_global_opts.split()
 557
 558 if options.mode == 'commit' and not options.pretend and not vcs:
 559         logging.info("Not in a version controlled repository; enabling pretend mode.")
 560         options.pretend = True
 561
 562 # Ensure that current repository is in the list of enabled repositories.
 563 repodir = os.path.realpath(portdir_overlay)
 564 try:
 565         repoman_settings.repositories.get_repo_for_location(repodir)
 566 except KeyError:
 567         repo_name = portage.repository.config.RepoConfig._read_valid_repo_name(portdir_overlay)[0]
 568         layout_conf_data = portage.repository.config.parse_layout_conf(portdir_overlay)[0]
 569         if layout_conf_data['repo-name']:
 570                 repo_name = layout_conf_data['repo-name']
 571         tmp_conf_file = io.StringIO(textwrap.dedent("""
 572                 [%s]
 573                 location = %s
 574                 """) % (repo_name, portdir_overlay))
 575         # Ensure that the repository corresponding to $PWD overrides a
 576         # repository of the same name referenced by the existing PORTDIR
 577         # or PORTDIR_OVERLAY settings.
 578         repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 579                 (repoman_settings.get('PORTDIR_OVERLAY', ''),
 580                 portage._shell_quote(portdir_overlay))
 581         repositories = portage.repository.config.load_repository_config(repoman_settings, extra_files=[tmp_conf_file])
 582         # We have to call the config constructor again so that attributes
 583         # dependent on config.repositories are initialized correctly.
 584         repoman_settings = portage.config(config_root=config_root, local_config=False, repositories=repositories)
 585
 586 root = repoman_settings['EROOT']
 587 trees = {
 588         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 589 }
 590 portdb = trees[root]['porttree'].dbapi
 591
 592 # Constrain dependency resolution to the master(s)
 593 # that are specified in layout.conf.
 594 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 595 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 596 portdir = portdb.porttrees[0]
 597 commit_env = os.environ.copy()
 598 # list() is for iteration on a copy.
 599 for repo in list(repoman_settings.repositories):
 600         # all paths are canonical
 601         if repo.location not in repo_config.eclass_db.porttrees:
 602                 del repoman_settings.repositories[repo.name]
 603
 604 if repo_config.allow_provide_virtual:
 605         qawarnings.add("virtual.oldstyle")
 606
 607 if repo_config.sign_commit:
 608         if vcs == 'git':
 609                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 610                 # the commit arguments. If key_id is unspecified, then it must be
 611                 # configured by `git config user.signingkey key_id`.
 612                 vcs_local_opts.append("--gpg-sign")
 613                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 614                         # Pass GNUPGHOME to git for bug #462362.
 615                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 616
 617                 # Pass GPG_TTY to git for bug #477728.
 618                 try:
 619                         commit_env["GPG_TTY"] = os.ttyname(sys.stdin.fileno())
 620                 except OSError:
 621                         pass
 622
 623 # In order to disable manifest signatures, repos may set
 624 # "sign-manifests = false" in metadata/layout.conf. This
 625 # can be used to prevent merge conflicts like those that
 626 # thin-manifests is designed to prevent.
 627 sign_manifests = "sign" in repoman_settings.features and \
 628         repo_config.sign_manifest
 629
 630 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 631         options.mode in ("commit",) and not sign_manifests:
 632         msg = ("The '%s' repository has manifest signatures enabled, "
 633         "but FEATURES=sign is currently disabled. In order to avoid this "
 634         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 635         "repositories can disable manifest signatures by setting "
 636         "'sign-manifests = false' in metadata/layout.conf.") % \
 637         (repo_config.name,)
 638         for line in textwrap.wrap(msg, 60):
 639                 logging.warn(line)
 640
 641 if sign_manifests and options.mode in ("commit",) and \
 642         repoman_settings.get("PORTAGE_GPG_KEY") and \
 643         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 644         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 645         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 646                 repoman_settings["PORTAGE_GPG_KEY"])
 647         sys.exit(1)
 648
 649 manifest_hashes = repo_config.manifest_hashes
 650 if manifest_hashes is None:
 651         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 652
 653 if options.mode in ("commit", "fix", "manifest"):
 654         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 655                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 656                 "metadata/layout.conf does not contain the '%s' hash which "
 657                 "is required by this portage version. You will have to "
 658                 "upgrade portage if you want to generate valid manifests for "
 659                 "this repository.") % \
 660                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 661                 for line in textwrap.wrap(msg, 70):
 662                         logging.error(line)
 663                 sys.exit(1)
 664
 665         unsupported_hashes = manifest_hashes.difference(
 666                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 667         if unsupported_hashes:
 668                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 669                 "metadata/layout.conf contains one or more hash types '%s' "
 670                 "which are not supported by this portage version. You will "
 671                 "have to upgrade portage if you want to generate valid "
 672                 "manifests for this repository.") % \
 673                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 674                 for line in textwrap.wrap(msg, 70):
 675                         logging.error(line)
 676                 sys.exit(1)
 677
 678 if options.echangelog is None and repo_config.update_changelog:
 679         options.echangelog = 'y'
 680
 681 if vcs is None:
 682         options.echangelog = 'n'
 683
 684 # The --echangelog option causes automatic ChangeLog generation,
 685 # which invalidates changelog.ebuildadded and changelog.missing
 686 # checks.
 687 # Note: Some don't use ChangeLogs in distributed SCMs.
 688 # It will be generated on server side from scm log,
 689 # before package moves to the rsync server.
 690 # This is needed because they try to avoid merge collisions.
 691 # Gentoo's Council decided to always use the ChangeLog file.
 692 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 693 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 694
 695 if 'digest' in repoman_settings.features and options.digest != 'n':
 696         options.digest = 'y'
 697
 698 logging.debug("vcs: %s" % (vcs,))
 699 logging.debug("repo config: %s" % (repo_config,))
 700 logging.debug("options: %s" % (options,))
 701
 702 # It's confusing if these warnings are displayed without the user
 703 # being told which profile they come from, so disable them.
 704 env = os.environ.copy()
 705 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 706
 707 categories = []
 708 for path in repo_config.eclass_db.porttrees:
 709         categories.extend(portage.util.grabfile(
 710                 os.path.join(path, 'profiles', 'categories')))
 711 repoman_settings.categories = frozenset(
 712         portage.util.stack_lists([categories], incremental=1))
 713 categories = repoman_settings.categories
 714
 715 portdb.settings = repoman_settings
 716 root_config = RootConfig(repoman_settings, trees[root], None)
 717 # We really only need to cache the metadata that's necessary for visibility
 718 # filtering. Anything else can be discarded to reduce memory consumption.
 719 portdb._aux_cache_keys.clear()
 720 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 721
 722 reposplit = myreporoot.split(os.path.sep)
 723 repolevel = len(reposplit)
 724
 725 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 726 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 727 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 728 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 729         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 730         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 731         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 732         print(red("***"))
 733         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 734
 735 # Make startdir relative to the canonical repodir, so that we can pass
 736 # it to digestgen and it won't have to be canonicalized again.
 737 if repolevel == 1:
 738         startdir = repodir
 739 else:
 740         startdir = normalize_path(mydir)
 741         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 742
 743 def caterror(mycat):
 744         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 745
 746 def repoman_getstatusoutput(cmd):
 747         """
 748         Implements an interface similar to getstatusoutput(), but with
 749         customized unicode handling (see bug #310789) and without the shell.
 750         """
 751         args = portage.util.shlex_split(cmd)
 752
 753         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 754                 not os.path.isabs(args[0]):
 755                 # Python 3.1 _execvp throws TypeError for non-absolute executable
 756                 # path passed as bytes (see http://bugs.python.org/issue8513).
 757                 fullname = find_binary(args[0])
 758                 if fullname is None:
 759                         raise portage.exception.CommandNotFound(args[0])
 760                 args[0] = fullname
 761
 762         encoding = _encodings['fs']
 763         args = [_unicode_encode(x,
 764                 encoding=encoding, errors='strict') for x in args]
 765         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 766                 stderr=subprocess.STDOUT)
 767         output = portage._unicode_decode(proc.communicate()[0],
 768                 encoding=encoding, errors='strict')
 769         if output and output[-1] == "\n":
 770                 # getstatusoutput strips one newline
 771                 output = output[:-1]
 772         return (proc.wait(), output)
 773
 774 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 775         """
 776         Implements an interface similar to os.popen(), but with customized
 777         unicode handling (see bug #310789) and without the shell.
 778         """
 779
 780         __slots__ = ('_proc', '_stdout')
 781
 782         def __init__(self, cmd):
 783                 args = portage.util.shlex_split(cmd)
 784
 785                 if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 786                         not os.path.isabs(args[0]):
 787                         # Python 3.1 _execvp throws TypeError for non-absolute executable
 788                         # path passed as bytes (see http://bugs.python.org/issue8513).
 789                         fullname = find_binary(args[0])
 790                         if fullname is None:
 791                                 raise portage.exception.CommandNotFound(args[0])
 792                         args[0] = fullname
 793
 794                 encoding = _encodings['fs']
 795                 args = [_unicode_encode(x,
 796                         encoding=encoding, errors='strict') for x in args]
 797                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 798                 object.__setattr__(self, '_proc', proc)
 799                 object.__setattr__(self, '_stdout',
 800                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 801
 802         def _get_target(self):
 803                 return object.__getattribute__(self, '_stdout')
 804
 805         __enter__ = _get_target
 806
 807         def __exit__(self, exc_type, exc_value, traceback):
 808                 proc = object.__getattribute__(self, '_proc')
 809                 proc.wait()
 810                 proc.stdout.close()
 811
 812 class ProfileDesc(object):
 813         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 814         def __init__(self, arch, status, sub_path, tree_path):
 815                 self.arch = arch
 816                 self.status = status
 817                 if sub_path:
 818                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 819                 self.sub_path = sub_path
 820                 self.tree_path = tree_path
 821                 if tree_path:
 822                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 823                 else:
 824                         self.abs_path = tree_path
 825
 826         def __str__(self):
 827                 if self.sub_path:
 828                         return self.sub_path
 829                 return 'empty profile'
 830
 831 profile_list = []
 832 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 833
 834 # get lists of valid keywords, licenses, and use
 835 kwlist = set()
 836 liclist = set()
 837 uselist = set()
 838 global_pmasklines = []
 839
 840 for path in portdb.porttrees:
 841         try:
 842                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 843         except OSError:
 844                 pass
 845         kwlist.update(portage.grabfile(os.path.join(path,
 846                 "profiles", "arch.list")))
 847
 848         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 849         for x in use_desc:
 850                 x = x.split()
 851                 if x:
 852                         uselist.add(x[0])
 853
 854         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 855         try:
 856                 expand_list = os.listdir(expand_desc_dir)
 857         except OSError:
 858                 pass
 859         else:
 860                 for fn in expand_list:
 861                         if not fn[-5:] == '.desc':
 862                                 continue
 863                         use_prefix = fn[:-5].lower() + '_'
 864                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 865                                 x = x.split()
 866                                 if x:
 867                                         uselist.add(use_prefix + x[0])
 868
 869         global_pmasklines.append(portage.util.grabfile_package(
 870                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 871
 872         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 873         try:
 874                 desc_file = io.open(_unicode_encode(desc_path,
 875                         encoding=_encodings['fs'], errors='strict'),
 876                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 877         except EnvironmentError:
 878                 pass
 879         else:
 880                 for i, x in enumerate(desc_file):
 881                         if x[0] == "#":
 882                                 continue
 883                         arch = x.split()
 884                         if len(arch) == 0:
 885                                 continue
 886                         if len(arch) != 3:
 887                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 888                                         desc_path + " line %d" % (i + 1, ))
 889                         elif arch[0] not in kwlist:
 890                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 891                                         desc_path + " line %d" % (i + 1, ))
 892                         elif arch[2] not in valid_profile_types:
 893                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 894                                         desc_path + " line %d" % (i + 1, ))
 895                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 896                         if not os.path.isdir(profile_desc.abs_path):
 897                                 logging.error(
 898                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 899                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 900                                 continue
 901                         if os.path.exists(
 902                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 903                                 continue
 904                         profile_list.append(profile_desc)
 905                 desc_file.close()
 906
 907 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 908 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 909
 910 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 911 global_pmaskdict = {}
 912 for x in global_pmasklines:
 913         global_pmaskdict.setdefault(x.cp, []).append(x)
 914 del global_pmasklines
 915
 916 def has_global_mask(pkg):
 917         mask_atoms = global_pmaskdict.get(pkg.cp)
 918         if mask_atoms:
 919                 pkg_list = [pkg]
 920                 for x in mask_atoms:
 921                         if portage.dep.match_from_list(x, pkg_list):
 922                                 return x
 923         return None
 924
 925 # Ensure that profile sub_path attributes are unique. Process in reverse order
 926 # so that profiles with duplicate sub_path from overlays will override
 927 # profiles with the same sub_path from parent repos.
 928 profiles = {}
 929 profile_list.reverse()
 930 profile_sub_paths = set()
 931 for prof in profile_list:
 932         if prof.sub_path in profile_sub_paths:
 933                 continue
 934         profile_sub_paths.add(prof.sub_path)
 935         profiles.setdefault(prof.arch, []).append(prof)
 936
 937 # Use an empty profile for checking dependencies of
 938 # packages that have empty KEYWORDS.
 939 prof = ProfileDesc('**', 'stable', '', '')
 940 profiles.setdefault(prof.arch, []).append(prof)
 941
 942 for x in repoman_settings.archlist():
 943         if x[0] == "~":
 944                 continue
 945         if x not in profiles:
 946                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 947                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 948                 print(red("up with the " + x + " team."))
 949                 print()
 950
 951 liclist_deprecated = set()
 952 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 953         liclist_deprecated.update(
 954                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 955
 956 if not liclist:
 957         logging.fatal("Couldn't find licenses?")
 958         sys.exit(1)
 959
 960 if not kwlist:
 961         logging.fatal("Couldn't read KEYWORDS from arch.list")
 962         sys.exit(1)
 963
 964 if not uselist:
 965         logging.fatal("Couldn't find use.desc?")
 966         sys.exit(1)
 967
 968 scanlist = []
 969 if repolevel == 2:
 970         # we are inside a category directory
 971         catdir = reposplit[-1]
 972         if catdir not in categories:
 973                 caterror(catdir)
 974         mydirlist = os.listdir(startdir)
 975         for x in mydirlist:
 976                 if x == "CVS" or x.startswith("."):
 977                         continue
 978                 if os.path.isdir(startdir + "/" + x):
 979                         scanlist.append(catdir + "/" + x)
 980         repo_subdir = catdir + os.sep
 981 elif repolevel == 1:
 982         for x in categories:
 983                 if not os.path.isdir(startdir + "/" + x):
 984                         continue
 985                 for y in os.listdir(startdir + "/" + x):
 986                         if y == "CVS" or y.startswith("."):
 987                                 continue
 988                         if os.path.isdir(startdir + "/" + x + "/" + y):
 989                                 scanlist.append(x + "/" + y)
 990         repo_subdir = ""
 991 elif repolevel == 3:
 992         catdir = reposplit[-2]
 993         if catdir not in categories:
 994                 caterror(catdir)
 995         scanlist.append(catdir + "/" + reposplit[-1])
 996         repo_subdir = scanlist[-1] + os.sep
 997 else:
 998         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 999                 ' from the current working directory'
1000         logging.critical(msg)
1001         sys.exit(1)
1002
1003 repo_subdir_len = len(repo_subdir)
1004 scanlist.sort()
1005
1006 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
1007
1008 def vcs_files_to_cps(vcs_file_iter):
1009         """
1010         Iterate over the given modified file paths returned from the vcs,
1011         and return a frozenset containing category/pn strings for each
1012         modified package.
1013         """
1014
1015         modified_cps = []
1016
1017         if repolevel == 3:
1018                 if reposplit[-2] in categories and \
1019                         next(vcs_file_iter, None) is not None:
1020                         modified_cps.append("/".join(reposplit[-2:]))
1021
1022         elif repolevel == 2:
1023                 category = reposplit[-1]
1024                 if category in categories:
1025                         for filename in vcs_file_iter:
1026                                 f_split = filename.split(os.sep)
1027                                 # ['.', pn, ...]
1028                                 if len(f_split) > 2:
1029                                         modified_cps.append(category + "/" + f_split[1])
1030
1031         else:
1032                 # repolevel == 1
1033                 for filename in vcs_file_iter:
1034                         f_split = filename.split(os.sep)
1035                         # ['.', category, pn, ...]
1036                         if len(f_split) > 3 and f_split[1] in categories:
1037                                 modified_cps.append("/".join(f_split[1:3]))
1038
1039         return frozenset(modified_cps)
1040
1041 def git_supports_gpg_sign():
1042         status, cmd_output = \
1043                 repoman_getstatusoutput("git --version")
1044         cmd_output = cmd_output.split()
1045         if cmd_output:
1046                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1047                 if version is not None:
1048                         version = [int(x) for x in version.groups()]
1049                         if version[0] > 1 or \
1050                                 (version[0] == 1 and version[1] > 7) or \
1051                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1052                                 return True
1053         return False
1054
1055 def dev_keywords(profiles):
1056         """
1057         Create a set of KEYWORDS values that exist in 'dev'
1058         profiles. These are used
1059         to trigger a message notifying the user when they might
1060         want to add the --include-dev option.
1061         """
1062         type_arch_map = {}
1063         for arch, arch_profiles in profiles.items():
1064                 for prof in arch_profiles:
1065                         arch_set = type_arch_map.get(prof.status)
1066                         if arch_set is None:
1067                                 arch_set = set()
1068                                 type_arch_map[prof.status] = arch_set
1069                         arch_set.add(arch)
1070
1071         dev_keywords = type_arch_map.get('dev', set())
1072         dev_keywords.update(['~' + arch for arch in dev_keywords])
1073         return frozenset(dev_keywords)
1074
1075 dev_keywords = dev_keywords(profiles)
1076
1077 stats = {}
1078 fails = {}
1079
1080 for x in qacats:
1081         stats[x] = 0
1082         fails[x] = []
1083
1084 xmllint_capable = False
1085 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1086
1087 def fetch_metadata_dtd():
1088         """
1089         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1090         metadata_dtd_ctime_interval.
1091         @rtype: bool
1092         @return: True if successful, otherwise False
1093         """
1094
1095         must_fetch = True
1096         metadata_dtd_st = None
1097         current_time = int(time.time())
1098         try:
1099                 metadata_dtd_st = os.stat(metadata_dtd)
1100         except EnvironmentError as e:
1101                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1102                         raise
1103                 del e
1104         else:
1105                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1106                 if abs(current_time - metadata_dtd_st.st_ctime) \
1107                         < metadata_dtd_ctime_interval:
1108                         must_fetch = False
1109
1110         if must_fetch:
1111                 print()
1112                 print(green("***") + " the local copy of metadata.dtd " + \
1113                         "needs to be refetched, doing that now")
1114                 print()
1115                 parsed_url = urlparse(metadata_dtd_uri)
1116                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
1117                 fcmd = repoman_settings.get(setting)
1118                 if not fcmd:
1119                         fcmd = repoman_settings.get('FETCHCOMMAND')
1120                         if not fcmd:
1121                                 logging.error("FETCHCOMMAND is unset")
1122                                 return False
1123
1124                 destdir = repoman_settings["DISTDIR"]
1125                 fd, metadata_dtd_tmp = tempfile.mkstemp(
1126                         prefix='metadata.dtd.', dir=destdir)
1127                 os.close(fd)
1128
1129                 try:
1130                         if not portage.getbinpkg.file_get(metadata_dtd_uri,
1131                                 destdir, fcmd=fcmd,
1132                                 filename=os.path.basename(metadata_dtd_tmp)):
1133                                 logging.error("failed to fetch metadata.dtd from '%s'" %
1134                                         metadata_dtd_uri)
1135                                 return False
1136
1137                         try:
1138                                 portage.util.apply_secpass_permissions(metadata_dtd_tmp,
1139                                         gid=portage.data.portage_gid, mode=0o664, mask=0o2)
1140                         except portage.exception.PortageException:
1141                                 pass
1142
1143                         os.rename(metadata_dtd_tmp, metadata_dtd)
1144                 finally:
1145                         try:
1146                                 os.unlink(metadata_dtd_tmp)
1147                         except OSError:
1148                                 pass
1149
1150         return True
1151
1152 if options.mode == "manifest":
1153         pass
1154 elif not find_binary('xmllint'):
1155         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1156         if options.xml_parse or repolevel == 3:
1157                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1158                 sys.exit(1)
1159 else:
1160         if not fetch_metadata_dtd():
1161                 sys.exit(1)
1162         # this can be problematic if xmllint changes their output
1163         xmllint_capable = True
1164
1165 if options.mode == 'commit' and vcs:
1166         utilities.detect_vcs_conflicts(options, vcs)
1167
1168 if options.mode == "manifest":
1169         pass
1170 elif options.pretend:
1171         print(green("\nRepoMan does a once-over of the neighborhood..."))
1172 else:
1173         print(green("\nRepoMan scours the neighborhood..."))
1174
1175 new_ebuilds = set()
1176 modified_ebuilds = set()
1177 modified_changelogs = set()
1178 mychanged = []
1179 mynew = []
1180 myremoved = []
1181
1182 if vcs == "cvs":
1183         mycvstree = cvstree.getentries("./", recursive=1)
1184         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1185         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1186         if options.if_modified == "y":
1187                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1188
1189 elif vcs == "svn":
1190         with repoman_popen("svn status") as f:
1191                 svnstatus = f.readlines()
1192         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1193         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1194         if options.if_modified == "y":
1195                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1196
1197 elif vcs == "git":
1198         with repoman_popen("git diff-index --name-only "
1199                 "--relative --diff-filter=M HEAD") as f:
1200                 mychanged = f.readlines()
1201         mychanged = ["./" + elem[:-1] for elem in mychanged]
1202
1203         with repoman_popen("git diff-index --name-only "
1204                 "--relative --diff-filter=A HEAD") as f:
1205                 mynew = f.readlines()
1206         mynew = ["./" + elem[:-1] for elem in mynew]
1207         if options.if_modified == "y":
1208                 with repoman_popen("git diff-index --name-only "
1209                         "--relative --diff-filter=D HEAD") as f:
1210                         myremoved = f.readlines()
1211                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1212
1213 elif vcs == "bzr":
1214         with repoman_popen("bzr status -S .") as f:
1215                 bzrstatus = f.readlines()
1216         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1217         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1218         if options.if_modified == "y":
1219                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1220
1221 elif vcs == "hg":
1222         with repoman_popen("hg status --no-status --modified .") as f:
1223                 mychanged = f.readlines()
1224         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1225         with repoman_popen("hg status --no-status --added .") as f:
1226                 mynew = f.readlines()
1227         mynew = ["./" + elem.rstrip() for elem in mynew]
1228         if options.if_modified == "y":
1229                 with repoman_popen("hg status --no-status --removed .") as f:
1230                         myremoved = f.readlines()
1231                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1232
1233 if vcs:
1234         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1235         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1236         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1237                 if os.path.basename(x) == "ChangeLog")
1238
1239 def vcs_new_changed(relative_path):
1240         for x in chain(mychanged, mynew):
1241                 if x == relative_path:
1242                         return True
1243         return False
1244
1245 have_pmasked = False
1246 have_dev_keywords = False
1247 dofail = 0
1248
1249 # NOTE: match-all caches are not shared due to potential
1250 # differences between profiles in _get_implicit_iuse.
1251 arch_caches = {}
1252 arch_xmatch_caches = {}
1253 shared_xmatch_caches = {"cp-list":{}}
1254
1255 include_arches = None
1256 if options.include_arches:
1257         include_arches = set()
1258         include_arches.update(*[x.split() for x in options.include_arches])
1259
1260 # Disable the "ebuild.notadded" check when not in commit mode and
1261 # running `svn status` in every package dir will be too expensive.
1262
1263 check_ebuild_notadded = not \
1264         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1265
1266 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1267 thirdpartymirrors = {}
1268 for k, v in repoman_settings.thirdpartymirrors().items():
1269         for v in v:
1270                 if not v.endswith("/"):
1271                         v += "/"
1272                 thirdpartymirrors[v] = k
1273
1274 class _XMLParser(xml.etree.ElementTree.XMLParser):
1275
1276         def __init__(self, data, **kwargs):
1277                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1278                 self._portage_data = data
1279                 if hasattr(self, 'parser'):
1280                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1281                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1282                         self._base_StartDoctypeDeclHandler = \
1283                                 self.parser.StartDoctypeDeclHandler
1284                         self.parser.StartDoctypeDeclHandler = \
1285                                 self._portage_StartDoctypeDeclHandler
1286
1287         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1288                 if self._base_XmlDeclHandler is not None:
1289                         self._base_XmlDeclHandler(version, encoding, standalone)
1290                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1291
1292         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1293                 has_internal_subset):
1294                 if self._base_StartDoctypeDeclHandler is not None:
1295                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1296                                 has_internal_subset)
1297                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1298
1299 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1300         """
1301         Implements doctype() as required to avoid deprecation warnings with
1302         >=python-2.7.
1303         """
1304         def doctype(self, name, pubid, system):
1305                 pass
1306
1307 try:
1308         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1309 except (EnvironmentError, ParseError, PermissionDenied) as e:
1310         err(str(e))
1311 except FileNotFound:
1312         # TODO: Download as we do for metadata.dtd, but add a way to
1313         # disable for non-gentoo repoman users who may not have herds.
1314         herd_base = None
1315
1316 effective_scanlist = scanlist
1317 if options.if_modified == "y":
1318         effective_scanlist = sorted(vcs_files_to_cps(
1319                 chain(mychanged, mynew, myremoved)))
1320
1321 for x in effective_scanlist:
1322         # ebuilds and digests added to cvs respectively.
1323         logging.info("checking package %s" % x)
1324         # save memory by discarding xmatch caches from previous package(s)
1325         arch_xmatch_caches.clear()
1326         eadded = []
1327         catdir, pkgdir = x.split("/")
1328         checkdir = repodir + "/" + x
1329         checkdir_relative = ""
1330         if repolevel < 3:
1331                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1332         if repolevel < 2:
1333                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1334         checkdir_relative = os.path.join(".", checkdir_relative)
1335         generated_manifest = False
1336
1337         if options.mode == "manifest" or \
1338           (options.mode != 'manifest-check' and options.digest == 'y') or \
1339           options.mode in ('commit', 'fix') and not options.pretend:
1340                 auto_assumed = set()
1341                 fetchlist_dict = portage.FetchlistDict(checkdir,
1342                         repoman_settings, portdb)
1343                 if options.mode == 'manifest' and options.force:
1344                         portage._doebuild_manifest_exempt_depend += 1
1345                         try:
1346                                 distdir = repoman_settings['DISTDIR']
1347                                 mf = repoman_settings.repositories.get_repo_for_location(
1348                                         os.path.dirname(os.path.dirname(checkdir)))
1349                                 mf = mf.load_manifest(checkdir, distdir,
1350                                         fetchlist_dict=fetchlist_dict)
1351                                 mf.create(requiredDistfiles=None,
1352                                         assumeDistHashesAlways=True)
1353                                 for distfiles in fetchlist_dict.values():
1354                                         for distfile in distfiles:
1355                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1356                                                         mf.fhashdict['DIST'].pop(distfile, None)
1357                                                 else:
1358                                                         auto_assumed.add(distfile)
1359                                 mf.write()
1360                         finally:
1361                                 portage._doebuild_manifest_exempt_depend -= 1
1362
1363                 repoman_settings["O"] = checkdir
1364                 try:
1365                         generated_manifest = digestgen(
1366                                 mysettings=repoman_settings, myportdb=portdb)
1367                 except portage.exception.PermissionDenied as e:
1368                         generated_manifest = False
1369                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1370                                 level=logging.ERROR, noiselevel=-1)
1371
1372                 if not generated_manifest:
1373                         print("Unable to generate manifest.")
1374                         dofail = 1
1375
1376                 if options.mode == "manifest":
1377                         if not dofail and options.force and auto_assumed and \
1378                                 'assume-digests' in repoman_settings.features:
1379                                 # Show which digests were assumed despite the --force option
1380                                 # being given. This output will already have been shown by
1381                                 # digestgen() if assume-digests is not enabled, so only show
1382                                 # it here if assume-digests is enabled.
1383                                 pkgs = list(fetchlist_dict)
1384                                 pkgs.sort()
1385                                 portage.writemsg_stdout("  digest.assumed" + \
1386                                         portage.output.colorize("WARN",
1387                                         str(len(auto_assumed)).rjust(18)) + "\n")
1388                                 for cpv in pkgs:
1389                                         fetchmap = fetchlist_dict[cpv]
1390                                         pf = portage.catsplit(cpv)[1]
1391                                         for distfile in sorted(fetchmap):
1392                                                 if distfile in auto_assumed:
1393                                                         portage.writemsg_stdout(
1394                                                                 "   %s::%s\n" % (pf, distfile))
1395                         continue
1396                 elif dofail:
1397                         sys.exit(1)
1398
1399         if not generated_manifest:
1400                 repoman_settings['O'] = checkdir
1401                 repoman_settings['PORTAGE_QUIET'] = '1'
1402                 if not portage.digestcheck([], repoman_settings, strict=1):
1403                         stats["manifest.bad"] += 1
1404                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1405                 repoman_settings.pop('PORTAGE_QUIET', None)
1406
1407         if options.mode == 'manifest-check':
1408                 continue
1409
1410         checkdirlist = os.listdir(checkdir)
1411         ebuildlist = []
1412         pkgs = {}
1413         allvalid = True
1414         for y in checkdirlist:
1415                 if (y in no_exec or y.endswith(".ebuild")) and \
1416                                 stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1417                         stats["file.executable"] += 1
1418                         fails["file.executable"].append(os.path.join(checkdir, y))
1419                 if y.endswith(".ebuild"):
1420                         pf = y[:-7]
1421                         ebuildlist.append(pf)
1422                         cpv = "%s/%s" % (catdir, pf)
1423                         try:
1424                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1425                         except KeyError:
1426                                 allvalid = False
1427                                 stats["ebuild.syntax"] += 1
1428                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1429                                 continue
1430                         except IOError:
1431                                 allvalid = False
1432                                 stats["ebuild.output"] += 1
1433                                 fails["ebuild.output"].append(os.path.join(x, y))
1434                                 continue
1435                         if not portage.eapi_is_supported(myaux["EAPI"]):
1436                                 allvalid = False
1437                                 stats["EAPI.unsupported"] += 1
1438                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1439                                 continue
1440                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1441                                 root_config=root_config, type_name="ebuild")
1442
1443         slot_keywords = {}
1444
1445         if len(pkgs) != len(ebuildlist):
1446                 # If we can't access all the metadata then it's totally unsafe to
1447                 # commit since there's no way to generate a correct Manifest.
1448                 # Do not try to do any more QA checks on this package since missing
1449                 # metadata leads to false positives for several checks, and false
1450                 # positives confuse users.
1451                 can_force = False
1452                 continue
1453
1454         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1455         ebuildlist = sorted(pkgs.values())
1456         ebuildlist = [pkg.pf for pkg in ebuildlist]
1457
1458         for y in checkdirlist:
1459                 index = repo_config.find_invalid_path_char(y)
1460                 if index != -1:
1461                         y_relative = os.path.join(checkdir_relative, y)
1462                         if vcs is not None and not vcs_new_changed(y_relative):
1463                                 # If the file isn't in the VCS new or changed set, then
1464                                 # assume that it's an irrelevant temporary file (Manifest
1465                                 # entries are not generated for file names containing
1466                                 # prohibited characters). See bug #406877.
1467                                 index = -1
1468                 if index != -1:
1469                         stats["file.name"] += 1
1470                         fails["file.name"].append("%s/%s: char '%s'" % \
1471                                 (checkdir, y, y[index]))
1472
1473                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1474                         continue
1475                 f = None
1476                 try:
1477                         line = 1
1478                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1479                                 encoding=_encodings['fs'], errors='strict'),
1480                                 mode='r', encoding=_encodings['repo.content'])
1481                         for l in f:
1482                                 line += 1
1483                 except UnicodeDecodeError as ue:
1484                         stats["file.UTF8"] += 1
1485                         s = ue.object[:ue.start]
1486                         l2 = s.count("\n")
1487                         line += l2
1488                         if l2 != 0:
1489                                 s = s[s.rfind("\n") + 1:]
1490                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1491                 finally:
1492                         if f is not None:
1493                                 f.close()
1494
1495         if vcs in ("git", "hg") and check_ebuild_notadded:
1496                 if vcs == "git":
1497                         myf = repoman_popen("git ls-files --others %s" % \
1498                                 (portage._shell_quote(checkdir_relative),))
1499                 if vcs == "hg":
1500                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1501                                 (portage._shell_quote(checkdir_relative),))
1502                 for l in myf:
1503                         if l[:-1][-7:] == ".ebuild":
1504                                 stats["ebuild.notadded"] += 1
1505                                 fails["ebuild.notadded"].append(
1506                                         os.path.join(x, os.path.basename(l[:-1])))
1507                 myf.close()
1508
1509         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1510                 try:
1511                         if vcs == "cvs":
1512                                 myf = open(checkdir + "/CVS/Entries", "r")
1513                         if vcs == "svn":
1514                                 myf = repoman_popen("svn status --depth=files --verbose " +
1515                                         portage._shell_quote(checkdir))
1516                         if vcs == "bzr":
1517                                 myf = repoman_popen("bzr ls -v --kind=file " +
1518                                         portage._shell_quote(checkdir))
1519                         myl = myf.readlines()
1520                         myf.close()
1521                         for l in myl:
1522                                 if vcs == "cvs":
1523                                         if l[0] != "/":
1524                                                 continue
1525                                         splitl = l[1:].split("/")
1526                                         if not len(splitl):
1527                                                 continue
1528                                         if splitl[0][-7:] == ".ebuild":
1529                                                 eadded.append(splitl[0][:-7])
1530                                 if vcs == "svn":
1531                                         if l[:1] == "?":
1532                                                 continue
1533                                         if l[:7] == '      >':
1534                                                 # tree conflict, new in subversion 1.6
1535                                                 continue
1536                                         l = l.split()[-1]
1537                                         if l[-7:] == ".ebuild":
1538                                                 eadded.append(os.path.basename(l[:-7]))
1539                                 if vcs == "bzr":
1540                                         if l[1:2] == "?":
1541                                                 continue
1542                                         l = l.split()[-1]
1543                                         if l[-7:] == ".ebuild":
1544                                                 eadded.append(os.path.basename(l[:-7]))
1545                         if vcs == "svn":
1546                                 myf = repoman_popen("svn status " +
1547                                         portage._shell_quote(checkdir))
1548                                 myl = myf.readlines()
1549                                 myf.close()
1550                                 for l in myl:
1551                                         if l[0] == "A":
1552                                                 l = l.rstrip().split(' ')[-1]
1553                                                 if l[-7:] == ".ebuild":
1554                                                         eadded.append(os.path.basename(l[:-7]))
1555                 except IOError:
1556                         if vcs == "cvs":
1557                                 stats["CVS/Entries.IO_error"] += 1
1558                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1559                         else:
1560                                 raise
1561                         continue
1562
1563         mf = repoman_settings.repositories.get_repo_for_location(
1564                 os.path.dirname(os.path.dirname(checkdir)))
1565         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1566         mydigests = mf.getTypeDigests("DIST")
1567
1568         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1569         myfiles_all = []
1570         src_uri_error = False
1571         for mykey in fetchlist_dict:
1572                 try:
1573                         myfiles_all.extend(fetchlist_dict[mykey])
1574                 except portage.exception.InvalidDependString as e:
1575                         src_uri_error = True
1576                         try:
1577                                 portdb.aux_get(mykey, ["SRC_URI"])
1578                         except KeyError:
1579                                 # This will be reported as an "ebuild.syntax" error.
1580                                 pass
1581                         else:
1582                                 stats["SRC_URI.syntax"] += 1
1583                                 fails["SRC_URI.syntax"].append(
1584                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1585         del fetchlist_dict
1586         if not src_uri_error:
1587                 # This test can produce false positives if SRC_URI could not
1588                 # be parsed for one or more ebuilds. There's no point in
1589                 # producing a false error here since the root cause will
1590                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1591                 # or "ebuild.sytax".
1592                 myfiles_all = set(myfiles_all)
1593                 for entry in mydigests:
1594                         if entry not in myfiles_all:
1595                                 stats["digest.unused"] += 1
1596                                 fails["digest.unused"].append(checkdir + "::" + entry)
1597                 for entry in myfiles_all:
1598                         if entry not in mydigests:
1599                                 stats["digest.missing"] += 1
1600                                 fails["digest.missing"].append(checkdir + "::" + entry)
1601         del myfiles_all
1602
1603         if os.path.exists(checkdir + "/files"):
1604                 filesdirlist = os.listdir(checkdir + "/files")
1605
1606                 # recurse through files directory
1607                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1608                 while filesdirlist:
1609                         y = filesdirlist.pop(0)
1610                         relative_path = os.path.join(x, "files", y)
1611                         full_path = os.path.join(repodir, relative_path)
1612                         try:
1613                                 mystat = os.stat(full_path)
1614                         except OSError as oe:
1615                                 if oe.errno == 2:
1616                                         # don't worry about it.  it likely was removed via fix above.
1617                                         continue
1618                                 else:
1619                                         raise oe
1620                         if S_ISDIR(mystat.st_mode):
1621                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1622                                 if y == "CVS" or y == ".svn":
1623                                         continue
1624                                 for z in os.listdir(checkdir + "/files/" + y):
1625                                         if z == "CVS" or z == ".svn":
1626                                                 continue
1627                                         filesdirlist.append(y + "/" + z)
1628                         # Current policy is no files over 20 KiB, these are the checks. File size between
1629                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1630                         elif mystat.st_size > 61440:
1631                                 stats["file.size.fatal"] += 1
1632                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1633                         elif mystat.st_size > 20480:
1634                                 stats["file.size"] += 1
1635                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1636
1637                         index = repo_config.find_invalid_path_char(y)
1638                         if index != -1:
1639                                 y_relative = os.path.join(checkdir_relative, "files", y)
1640                                 if vcs is not None and not vcs_new_changed(y_relative):
1641                                         # If the file isn't in the VCS new or changed set, then
1642                                         # assume that it's an irrelevant temporary file (Manifest
1643                                         # entries are not generated for file names containing
1644                                         # prohibited characters). See bug #406877.
1645                                         index = -1
1646                         if index != -1:
1647                                 stats["file.name"] += 1
1648                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1649                                         (checkdir, y, y[index]))
1650         del mydigests
1651
1652         if check_changelog and "ChangeLog" not in checkdirlist:
1653                 stats["changelog.missing"] += 1
1654                 fails["changelog.missing"].append(x + "/ChangeLog")
1655
1656         musedict = {}
1657         # metadata.xml file check
1658         if "metadata.xml" not in checkdirlist:
1659                 stats["metadata.missing"] += 1
1660                 fails["metadata.missing"].append(x + "/metadata.xml")
1661         # metadata.xml parse check
1662         else:
1663                 metadata_bad = False
1664                 xml_info = {}
1665                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1666
1667                 # read metadata.xml into memory
1668                 try:
1669                         _metadata_xml = xml.etree.ElementTree.parse(
1670                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1671                                 encoding=_encodings['fs'], errors='strict'),
1672                                 parser=xml_parser)
1673                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1674                         metadata_bad = True
1675                         stats["metadata.bad"] += 1
1676                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1677                         del e
1678                 else:
1679                         if not hasattr(xml_parser, 'parser') or \
1680                                 sys.hexversion < 0x2070000 or \
1681                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1682                                 # doctype is not parsed with python 2.6 or 3.1
1683                                 pass
1684                         else:
1685                                 if "XML_DECLARATION" not in xml_info:
1686                                         stats["metadata.bad"] += 1
1687                                         fails["metadata.bad"].append("%s/metadata.xml: "
1688                                                 "xml declaration is missing on first line, "
1689                                                 "should be '%s'" % (x, metadata_xml_declaration))
1690                                 else:
1691                                         xml_version, xml_encoding, xml_standalone = \
1692                                                 xml_info["XML_DECLARATION"]
1693                                         if xml_encoding is None or \
1694                                                 xml_encoding.upper() != metadata_xml_encoding:
1695                                                 stats["metadata.bad"] += 1
1696                                                 if xml_encoding is None:
1697                                                         encoding_problem = "but it is undefined"
1698                                                 else:
1699                                                         encoding_problem = "not '%s'" % xml_encoding
1700                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1701                                                         "xml declaration encoding should be '%s', %s" %
1702                                                         (x, metadata_xml_encoding, encoding_problem))
1703
1704                                 if "DOCTYPE" not in xml_info:
1705                                         metadata_bad = True
1706                                         stats["metadata.bad"] += 1
1707                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1708                                                 "DOCTYPE is missing"))
1709                                 else:
1710                                         doctype_name, doctype_system, doctype_pubid = \
1711                                                 xml_info["DOCTYPE"]
1712                                         if doctype_system != metadata_dtd_uri:
1713                                                 stats["metadata.bad"] += 1
1714                                                 if doctype_system is None:
1715                                                         system_problem = "but it is undefined"
1716                                                 else:
1717                                                         system_problem = "not '%s'" % doctype_system
1718                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1719                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1720                                                         (x, metadata_dtd_uri, system_problem))
1721
1722                                         if doctype_name != metadata_doctype_name:
1723                                                 stats["metadata.bad"] += 1
1724                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1725                                                         "DOCTYPE: name should be '%s', not '%s'" %
1726                                                         (x, metadata_doctype_name, doctype_name))
1727
1728                         # load USE flags from metadata.xml
1729                         try:
1730                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1731                         except portage.exception.ParseError as e:
1732                                 metadata_bad = True
1733                                 stats["metadata.bad"] += 1
1734                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1735                         else:
1736                                 for atom in chain(*musedict.values()):
1737                                         if atom is None:
1738                                                 continue
1739                                         try:
1740                                                 atom = Atom(atom)
1741                                         except InvalidAtom as e:
1742                                                 stats["metadata.bad"] += 1
1743                                                 fails["metadata.bad"].append(
1744                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1745                                         else:
1746                                                 if atom.cp != x:
1747                                                         stats["metadata.bad"] += 1
1748                                                         fails["metadata.bad"].append(
1749                                                                 ("%s/metadata.xml: Atom contains "
1750                                                                 "unexpected cat/pn: %s") % (x, atom))
1751
1752                         # Run other metadata.xml checkers
1753                         try:
1754                                 utilities.check_metadata(_metadata_xml, herd_base)
1755                         except (utilities.UnknownHerdsError, ) as e:
1756                                 metadata_bad = True
1757                                 stats["metadata.bad"] += 1
1758                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1759                                 del e
1760
1761                 # Only carry out if in package directory or check forced
1762                 if xmllint_capable and not metadata_bad:
1763                         # xmlint can produce garbage output even on success, so only dump
1764                         # the ouput when it fails.
1765                         st, out = repoman_getstatusoutput(
1766                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1767                                  (portage._shell_quote(metadata_dtd),
1768                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1769                         if st != os.EX_OK:
1770                                 print(red("!!!") + " metadata.xml is invalid:")
1771                                 for z in out.splitlines():
1772                                         print(red("!!! ") + z)
1773                                 stats["metadata.bad"] += 1
1774                                 fails["metadata.bad"].append(x + "/metadata.xml")
1775
1776                 del metadata_bad
1777         muselist = frozenset(musedict)
1778
1779         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1780         changelog_modified = changelog_path in modified_changelogs
1781
1782         # detect unused local USE-descriptions
1783         used_useflags = set()
1784
1785         for y in ebuildlist:
1786                 relative_path = os.path.join(x, y + ".ebuild")
1787                 full_path = os.path.join(repodir, relative_path)
1788                 ebuild_path = y + ".ebuild"
1789                 if repolevel < 3:
1790                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1791                 if repolevel < 2:
1792                         ebuild_path = os.path.join(catdir, ebuild_path)
1793                 ebuild_path = os.path.join(".", ebuild_path)
1794                 if check_changelog and not changelog_modified \
1795                         and ebuild_path in new_ebuilds:
1796                         stats['changelog.ebuildadded'] += 1
1797                         fails['changelog.ebuildadded'].append(relative_path)
1798
1799                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1800                         # ebuild not added to vcs
1801                         stats["ebuild.notadded"] += 1
1802                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1803                 myesplit = portage.pkgsplit(y)
1804                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1805                             or pv_toolong_re.search(myesplit[1]) \
1806                             or pv_toolong_re.search(myesplit[2]):
1807                         stats["ebuild.invalidname"] += 1
1808                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1809                         continue
1810                 elif myesplit[0] != pkgdir:
1811                         print(pkgdir, myesplit[0])
1812                         stats["ebuild.namenomatch"] += 1
1813                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1814                         continue
1815
1816                 pkg = pkgs[y]
1817
1818                 if pkg.invalid:
1819                         allvalid = False
1820                         for k, msgs in pkg.invalid.items():
1821                                 for msg in msgs:
1822                                         stats[k] += 1
1823                                         fails[k].append("%s: %s" % (relative_path, msg))
1824                         continue
1825
1826                 myaux = pkg._metadata
1827                 eapi = myaux["EAPI"]
1828                 inherited = pkg.inherited
1829                 live_ebuild = live_eclasses.intersection(inherited)
1830
1831                 if repo_config.eapi_is_banned(eapi):
1832                         stats["repo.eapi.banned"] += 1
1833                         fails["repo.eapi.banned"].append(
1834                                 "%s: %s" % (relative_path, eapi))
1835
1836                 elif repo_config.eapi_is_deprecated(eapi):
1837                         stats["repo.eapi.deprecated"] += 1
1838                         fails["repo.eapi.deprecated"].append(
1839                                 "%s: %s" % (relative_path, eapi))
1840
1841                 for k, v in myaux.items():
1842                         if not isinstance(v, basestring):
1843                                 continue
1844                         m = non_ascii_re.search(v)
1845                         if m is not None:
1846                                 stats["variable.invalidchar"] += 1
1847                                 fails["variable.invalidchar"].append(
1848                                         ("%s: %s variable contains non-ASCII " + \
1849                                         "character at position %s") % \
1850                                         (relative_path, k, m.start() + 1))
1851
1852                 if not src_uri_error:
1853                         # Check that URIs don't reference a server from thirdpartymirrors.
1854                         for uri in portage.dep.use_reduce( \
1855                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1856                                 contains_mirror = False
1857                                 for mirror, mirror_alias in thirdpartymirrors.items():
1858                                         if uri.startswith(mirror):
1859                                                 contains_mirror = True
1860                                                 break
1861                                 if not contains_mirror:
1862                                         continue
1863
1864                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1865                                 stats["SRC_URI.mirror"] += 1
1866                                 fails["SRC_URI.mirror"].append(
1867                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1868                                         (relative_path, mirror, new_uri))
1869
1870                 if myaux.get("PROVIDE"):
1871                         stats["virtual.oldstyle"] += 1
1872                         fails["virtual.oldstyle"].append(relative_path)
1873
1874                 for pos, missing_var in enumerate(missingvars):
1875                         if not myaux.get(missing_var):
1876                                 if catdir == "virtual" and \
1877                                         missing_var in ("HOMEPAGE", "LICENSE"):
1878                                         continue
1879                                 if live_ebuild and missing_var == "KEYWORDS":
1880                                         continue
1881                                 myqakey = missingvars[pos] + ".missing"
1882                                 stats[myqakey] += 1
1883                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1884
1885                 if catdir == "virtual":
1886                         for var in ("HOMEPAGE", "LICENSE"):
1887                                 if myaux.get(var):
1888                                         myqakey = var + ".virtual"
1889                                         stats[myqakey] += 1
1890                                         fails[myqakey].append(relative_path)
1891
1892                 # 14 is the length of DESCRIPTION=""
1893                 if len(myaux['DESCRIPTION']) > max_desc_len:
1894                         stats['DESCRIPTION.toolong'] += 1
1895                         fails['DESCRIPTION.toolong'].append(
1896                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1897                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1898
1899                 keywords = myaux["KEYWORDS"].split()
1900                 stable_keywords = []
1901                 for keyword in keywords:
1902                         if not keyword.startswith("~") and \
1903                                 not keyword.startswith("-"):
1904                                 stable_keywords.append(keyword)
1905                 if stable_keywords:
1906                         if ebuild_path in new_ebuilds and catdir != "virtual":
1907                                 stable_keywords.sort()
1908                                 stats["KEYWORDS.stable"] += 1
1909                                 fails["KEYWORDS.stable"].append(
1910                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1911                                                 " ".join(stable_keywords))
1912
1913                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1914                         if not kw.startswith("-"))
1915
1916                 previous_keywords = slot_keywords.get(pkg.slot)
1917                 if previous_keywords is None:
1918                         slot_keywords[pkg.slot] = set()
1919                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1920                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1921                         if dropped_keywords:
1922                                 stats["KEYWORDS.dropped"] += 1
1923                                 fails["KEYWORDS.dropped"].append(
1924                                         relative_path + ": %s" % \
1925                                         " ".join(sorted(dropped_keywords)))
1926
1927                 slot_keywords[pkg.slot].update(ebuild_archs)
1928
1929                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1930                 if "-*" in keywords:
1931                         haskeyword = False
1932                         for kw in keywords:
1933                                 if kw[0] == "~":
1934                                         kw = kw[1:]
1935                                 if kw in kwlist:
1936                                         haskeyword = True
1937                         if not haskeyword:
1938                                 stats["KEYWORDS.stupid"] += 1
1939                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1940
1941                 """
1942                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1943                 not be allowed to be marked stable
1944                 """
1945                 if live_ebuild and repo_config.name == "gentoo":
1946                         bad_stable_keywords = []
1947                         for keyword in keywords:
1948                                 if not keyword.startswith("~") and \
1949                                         not keyword.startswith("-"):
1950                                         bad_stable_keywords.append(keyword)
1951                                 del keyword
1952                         if bad_stable_keywords:
1953                                 stats["LIVEVCS.stable"] += 1
1954                                 fails["LIVEVCS.stable"].append(
1955                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1956                                                 bad_stable_keywords)
1957                         del bad_stable_keywords
1958
1959                         if keywords and not has_global_mask(pkg):
1960                                 stats["LIVEVCS.unmasked"] += 1
1961                                 fails["LIVEVCS.unmasked"].append(relative_path)
1962
1963                 if options.ignore_arches:
1964                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1965                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1966                 else:
1967                         arches = set()
1968                         for keyword in keywords:
1969                                 if keyword[0] == "-":
1970                                         continue
1971                                 elif keyword[0] == "~":
1972                                         arch = keyword[1:]
1973                                         if arch == "*":
1974                                                 for expanded_arch in profiles:
1975                                                         if expanded_arch == "**":
1976                                                                 continue
1977                                                         arches.add((keyword, expanded_arch,
1978                                                                 (expanded_arch, "~" + expanded_arch)))
1979                                         else:
1980                                                 arches.add((keyword, arch, (arch, keyword)))
1981                                 else:
1982                                         if keyword == "*":
1983                                                 for expanded_arch in profiles:
1984                                                         if expanded_arch == "**":
1985                                                                 continue
1986                                                         arches.add((keyword, expanded_arch,
1987                                                                 (expanded_arch,)))
1988                                         else:
1989                                                 arches.add((keyword, keyword, (keyword,)))
1990                         if not arches:
1991                                 # Use an empty profile for checking dependencies of
1992                                 # packages that have empty KEYWORDS.
1993                                 arches.add(('**', '**', ('**',)))
1994
1995                 unknown_pkgs = set()
1996                 baddepsyntax = False
1997                 badlicsyntax = False
1998                 badprovsyntax = False
1999                 catpkg = catdir + "/" + y
2000
2001                 inherited_java_eclass = "java-pkg-2" in inherited or \
2002                         "java-pkg-opt-2" in inherited
2003                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
2004                 operator_tokens = set(["||", "(", ")"])
2005                 type_list, badsyntax = [], []
2006                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
2007                         mydepstr = myaux[mytype]
2008
2009                         buildtime = mytype in Package._buildtime_keys
2010                         runtime = mytype in Package._runtime_keys
2011                         token_class = None
2012                         if mytype.endswith("DEPEND"):
2013                                 token_class = portage.dep.Atom
2014
2015                         try:
2016                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
2017                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
2018                         except portage.exception.InvalidDependString as e:
2019                                 atoms = None
2020                                 badsyntax.append(str(e))
2021
2022                         if atoms and mytype.endswith("DEPEND"):
2023                                 if runtime and \
2024                                         "test?" in mydepstr.split():
2025                                         stats[mytype + '.suspect'] += 1
2026                                         fails[mytype + '.suspect'].append(relative_path + \
2027                                                 ": 'test?' USE conditional in %s" % mytype)
2028
2029                                 for atom in atoms:
2030                                         if atom == "||":
2031                                                 continue
2032
2033                                         # Skip dependency.unknown for blockers, so that we
2034                                         # don't encourage people to remove necessary blockers,
2035                                         # as discussed in bug #382407.
2036                                         if atom.blocker is None and \
2037                                                 not portdb.xmatch("match-all", atom) and \
2038                                                 not atom.cp.startswith("virtual/"):
2039                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2040
2041                                         is_blocker = atom.blocker
2042
2043                                         if catdir != "virtual":
2044                                                 if not is_blocker and \
2045                                                         atom.cp in suspect_virtual:
2046                                                         stats['virtual.suspect'] += 1
2047                                                         fails['virtual.suspect'].append(
2048                                                                 relative_path +
2049                                                                 ": %s: consider using '%s' instead of '%s'" %
2050                                                                 (mytype, suspect_virtual[atom.cp], atom))
2051
2052                                         if buildtime and \
2053                                                 not is_blocker and \
2054                                                 not inherited_java_eclass and \
2055                                                 atom.cp == "virtual/jdk":
2056                                                 stats['java.eclassesnotused'] += 1
2057                                                 fails['java.eclassesnotused'].append(relative_path)
2058                                         elif buildtime and \
2059                                                 not is_blocker and \
2060                                                 not inherited_wxwidgets_eclass and \
2061                                                 atom.cp == "x11-libs/wxGTK":
2062                                                 stats['wxwidgets.eclassnotused'] += 1
2063                                                 fails['wxwidgets.eclassnotused'].append(
2064                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2065                                                         " without inheriting wxwidgets.eclass") % mytype)
2066                                         elif runtime:
2067                                                 if not is_blocker and \
2068                                                         atom.cp in suspect_rdepend:
2069                                                         stats[mytype + '.suspect'] += 1
2070                                                         fails[mytype + '.suspect'].append(
2071                                                                 relative_path + ": '%s'" % atom)
2072
2073                                         if atom.operator == "~" and \
2074                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2075                                                 qacat = 'dependency.badtilde'
2076                                                 stats[qacat] += 1
2077                                                 fails[qacat].append(
2078                                                         (relative_path + ": %s uses the ~ operator"
2079                                                          " with a non-zero revision:" + \
2080                                                          " '%s'") % (mytype, atom))
2081
2082                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2083
2084                 for m, b in zip(type_list, badsyntax):
2085                         if m.endswith("DEPEND"):
2086                                 qacat = "dependency.syntax"
2087                         else:
2088                                 qacat = m + ".syntax"
2089                         stats[qacat] += 1
2090                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2091
2092                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2093                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2094                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2095                 badlicsyntax = badlicsyntax > 0
2096                 badprovsyntax = badprovsyntax > 0
2097
2098                 # uselist checks - global
2099                 myuse = []
2100                 default_use = []
2101                 for myflag in myaux["IUSE"].split():
2102                         flag_name = myflag.lstrip("+-")
2103                         used_useflags.add(flag_name)
2104                         if myflag != flag_name:
2105                                 default_use.append(myflag)
2106                         if flag_name not in uselist:
2107                                 myuse.append(flag_name)
2108
2109                 # uselist checks - metadata
2110                 for mypos in range(len(myuse)-1, -1, -1):
2111                         if myuse[mypos] and (myuse[mypos] in muselist):
2112                                 del myuse[mypos]
2113
2114                 if default_use and not eapi_has_iuse_defaults(eapi):
2115                         for myflag in default_use:
2116                                 stats['EAPI.incompatible'] += 1
2117                                 fails['EAPI.incompatible'].append(
2118                                         (relative_path + ": IUSE defaults" + \
2119                                         " not supported with EAPI='%s':" + \
2120                                         " '%s'") % (eapi, myflag))
2121
2122                 for mypos in range(len(myuse)):
2123                         stats["IUSE.invalid"] += 1
2124                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2125
2126                 # Check for outdated RUBY targets
2127                 if "ruby-ng" in inherited or "ruby-fakegem" in inherited or "ruby" in inherited:
2128                         ruby_intersection = pkg.iuse.all.intersection(ruby_deprecated)
2129                         if ruby_intersection:
2130                                 for myruby in ruby_intersection:
2131                                         stats["IUSE.rubydeprecated"] += 1
2132                                         fails["IUSE.rubydeprecated"].append(
2133                                                 (relative_path + ": Deprecated ruby target: %s") % myruby)
2134
2135                 # license checks
2136                 if not badlicsyntax:
2137                         # Parse the LICENSE variable, remove USE conditions and
2138                         # flatten it.
2139                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2140                         # Check each entry to ensure that it exists in PORTDIR's
2141                         # license directory.
2142                         for lic in licenses:
2143                                 # Need to check for "||" manually as no portage
2144                                 # function will remove it without removing values.
2145                                 if lic not in liclist and lic != "||":
2146                                         stats["LICENSE.invalid"] += 1
2147                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2148                                 elif lic in liclist_deprecated:
2149                                         stats["LICENSE.deprecated"] += 1
2150                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2151
2152                 # keyword checks
2153                 myuse = myaux["KEYWORDS"].split()
2154                 for mykey in myuse:
2155                         if mykey not in ("-*", "*", "~*"):
2156                                 myskey = mykey
2157                                 if myskey[:1] == "-":
2158                                         myskey = myskey[1:]
2159                                 if myskey[:1] == "~":
2160                                         myskey = myskey[1:]
2161                                 if myskey not in kwlist:
2162                                         stats["KEYWORDS.invalid"] += 1
2163                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2164                                 elif myskey not in profiles:
2165                                         stats["KEYWORDS.invalid"] += 1
2166                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2167
2168                 # restrict checks
2169                 myrestrict = None
2170                 try:
2171                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2172                 except portage.exception.InvalidDependString as e:
2173                         stats["RESTRICT.syntax"] += 1
2174                         fails["RESTRICT.syntax"].append(
2175                                 "%s: RESTRICT: %s" % (relative_path, e))
2176                         del e
2177                 if myrestrict:
2178                         myrestrict = set(myrestrict)
2179                         mybadrestrict = myrestrict.difference(valid_restrict)
2180                         if mybadrestrict:
2181                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2182                                 for mybad in mybadrestrict:
2183                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2184                 # REQUIRED_USE check
2185                 required_use = myaux["REQUIRED_USE"]
2186                 if required_use:
2187                         if not eapi_has_required_use(eapi):
2188                                 stats['EAPI.incompatible'] += 1
2189                                 fails['EAPI.incompatible'].append(
2190                                         relative_path + ": REQUIRED_USE" + \
2191                                         " not supported with EAPI='%s'" % (eapi,))
2192                         try:
2193                                 portage.dep.check_required_use(required_use, (),
2194                                         pkg.iuse.is_valid_flag, eapi=eapi)
2195                         except portage.exception.InvalidDependString as e:
2196                                 stats["REQUIRED_USE.syntax"] += 1
2197                                 fails["REQUIRED_USE.syntax"].append(
2198                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2199                                 del e
2200
2201                 # Syntax Checks
2202                 relative_path = os.path.join(x, y + ".ebuild")
2203                 full_path = os.path.join(repodir, relative_path)
2204                 if not vcs_preserves_mtime:
2205                         if ebuild_path not in new_ebuilds and \
2206                                 ebuild_path not in modified_ebuilds:
2207                                 pkg.mtime = None
2208                 try:
2209                         # All ebuilds should have utf_8 encoding.
2210                         f = io.open(_unicode_encode(full_path,
2211                                 encoding=_encodings['fs'], errors='strict'),
2212                                 mode='r', encoding=_encodings['repo.content'])
2213                         try:
2214                                 for check_name, e in run_checks(f, pkg):
2215                                         stats[check_name] += 1
2216                                         fails[check_name].append(relative_path + ': %s' % e)
2217                         finally:
2218                                 f.close()
2219                 except UnicodeDecodeError:
2220                         # A file.UTF8 failure will have already been recorded above.
2221                         pass
2222
2223                 if options.force:
2224                         # The dep_check() calls are the most expensive QA test. If --force
2225                         # is enabled, there's no point in wasting time on these since the
2226                         # user is intent on forcing the commit anyway.
2227                         continue
2228
2229                 relevant_profiles = []
2230                 for keyword, arch, groups in arches:
2231                         if arch not in profiles:
2232                                 # A missing profile will create an error further down
2233                                 # during the KEYWORDS verification.
2234                                 continue
2235
2236                         if include_arches is not None:
2237                                 if arch not in include_arches:
2238                                         continue
2239
2240                         relevant_profiles.extend((keyword, groups, prof)
2241                                 for prof in profiles[arch])
2242
2243                 def sort_key(item):
2244                         return item[2].sub_path
2245
2246                 relevant_profiles.sort(key=sort_key)
2247
2248                 for keyword, groups, prof in relevant_profiles:
2249
2250                         if not (prof.status == "stable" or \
2251                                 (prof.status == "dev" and options.include_dev) or \
2252                                 (prof.status == "exp" and options.include_exp_profiles == 'y')):
2253                                 continue
2254
2255                         dep_settings = arch_caches.get(prof.sub_path)
2256                         if dep_settings is None:
2257                                 dep_settings = portage.config(
2258                                         config_profile_path=prof.abs_path,
2259                                         config_incrementals=repoman_incrementals,
2260                                         config_root=config_root,
2261                                         local_config=False,
2262                                         _unmatched_removal=options.unmatched_removal,
2263                                         env=env, repositories=repoman_settings.repositories)
2264                                 dep_settings.categories = repoman_settings.categories
2265                                 if options.without_mask:
2266                                         dep_settings._mask_manager_obj = \
2267                                                 copy.deepcopy(dep_settings._mask_manager)
2268                                         dep_settings._mask_manager._pmaskdict.clear()
2269                                 arch_caches[prof.sub_path] = dep_settings
2270
2271                         xmatch_cache_key = (prof.sub_path, tuple(groups))
2272                         xcache = arch_xmatch_caches.get(xmatch_cache_key)
2273                         if xcache is None:
2274                                 portdb.melt()
2275                                 portdb.freeze()
2276                                 xcache = portdb.xcache
2277                                 xcache.update(shared_xmatch_caches)
2278                                 arch_xmatch_caches[xmatch_cache_key] = xcache
2279
2280                         trees[root]["porttree"].settings = dep_settings
2281                         portdb.settings = dep_settings
2282                         portdb.xcache = xcache
2283
2284                         dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2285                         # just in case, prevent config.reset() from nuking these.
2286                         dep_settings.backup_changes("ACCEPT_KEYWORDS")
2287
2288                         # This attribute is used in dbapi._match_use() to apply
2289                         # use.stable.{mask,force} settings based on the stable
2290                         # status of the parent package. This is required in order
2291                         # for USE deps of unstable packages to be resolved correctly,
2292                         # since otherwise use.stable.{mask,force} settings of
2293                         # dependencies may conflict (see bug #456342).
2294                         dep_settings._parent_stable = dep_settings._isStable(pkg)
2295
2296                         # Handle package.use*.{force,mask) calculation, for use
2297                         # in dep_check.
2298                         dep_settings.useforce = dep_settings._use_manager.getUseForce(
2299                                 pkg, stable=dep_settings._parent_stable)
2300                         dep_settings.usemask = dep_settings._use_manager.getUseMask(
2301                                 pkg, stable=dep_settings._parent_stable)
2302
2303                         if not baddepsyntax:
2304                                 ismasked = not ebuild_archs or \
2305                                         pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2306                                 if ismasked:
2307                                         if not have_pmasked:
2308                                                 have_pmasked = bool(dep_settings._getMaskAtom(
2309                                                         pkg.cpv, pkg._metadata))
2310                                         if options.ignore_masked:
2311                                                 continue
2312                                         # we are testing deps for a masked package; give it some lee-way
2313                                         suffix = "masked"
2314                                         matchmode = "minimum-all"
2315                                 else:
2316                                         suffix = ""
2317                                         matchmode = "minimum-visible"
2318
2319                                 if not have_dev_keywords:
2320                                         have_dev_keywords = \
2321                                                 bool(dev_keywords.intersection(keywords))
2322
2323                                 if prof.status == "dev":
2324                                         suffix = suffix + "indev"
2325
2326                                 for mytype in Package._dep_keys:
2327
2328                                         mykey = "dependency.bad" + suffix
2329                                         myvalue = myaux[mytype]
2330                                         if not myvalue:
2331                                                 continue
2332
2333                                         success, atoms = portage.dep_check(myvalue, portdb,
2334                                                 dep_settings, use="all", mode=matchmode,
2335                                                 trees=trees)
2336
2337                                         if success:
2338                                                 if atoms:
2339
2340                                                         # Don't bother with dependency.unknown for
2341                                                         # cases in which *DEPEND.bad is triggered.
2342                                                         for atom in atoms:
2343                                                                 # dep_check returns all blockers and they
2344                                                                 # aren't counted for *DEPEND.bad, so we
2345                                                                 # ignore them here.
2346                                                                 if not atom.blocker:
2347                                                                         unknown_pkgs.discard(
2348                                                                                 (mytype, atom.unevaluated_atom))
2349
2350                                                         if not prof.sub_path:
2351                                                                 # old-style virtuals currently aren't
2352                                                                 # resolvable with empty profile, since
2353                                                                 # 'virtuals' mappings are unavailable
2354                                                                 # (it would be expensive to search
2355                                                                 # for PROVIDE in all ebuilds)
2356                                                                 atoms = [atom for atom in atoms if not \
2357                                                                         (atom.cp.startswith('virtual/') and \
2358                                                                         not portdb.cp_list(atom.cp))]
2359
2360                                                         # we have some unsolvable deps
2361                                                         # remove ! deps, which always show up as unsatisfiable
2362                                                         atoms = [str(atom.unevaluated_atom) \
2363                                                                 for atom in atoms if not atom.blocker]
2364
2365                                                         # if we emptied out our list, continue:
2366                                                         if not atoms:
2367                                                                 continue
2368                                                         stats[mykey] += 1
2369                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2370                                                                 (relative_path, mytype, keyword,
2371                                                                 prof, repr(atoms)))
2372                                         else:
2373                                                 stats[mykey] += 1
2374                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2375                                                         (relative_path, mytype, keyword,
2376                                                         prof, repr(atoms)))
2377
2378                 if not baddepsyntax and unknown_pkgs:
2379                         type_map = {}
2380                         for mytype, atom in unknown_pkgs:
2381                                 type_map.setdefault(mytype, set()).add(atom)
2382                         for mytype, atoms in type_map.items():
2383                                 stats["dependency.unknown"] += 1
2384                                 fails["dependency.unknown"].append("%s: %s: %s" %
2385                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2386
2387         # check if there are unused local USE-descriptions in metadata.xml
2388         # (unless there are any invalids, to avoid noise)
2389         if allvalid:
2390                 for myflag in muselist.difference(used_useflags):
2391                         stats["metadata.warning"] += 1
2392                         fails["metadata.warning"].append(
2393                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2394                                 (x, myflag))
2395
2396 if options.if_modified == "y" and len(effective_scanlist) < 1:
2397         logging.warn("--if-modified is enabled, but no modified packages were found!")
2398
2399 if options.mode == "manifest":
2400         sys.exit(dofail)
2401
2402 # dofail will be set to 1 if we have failed in at least one non-warning category
2403 dofail = 0
2404 # dowarn will be set to 1 if we tripped any warnings
2405 dowarn = 0
2406 # dofull will be set if we should print a "repoman full" informational message
2407 dofull = options.mode != 'full'
2408
2409 for x in qacats:
2410         if not stats[x]:
2411                 continue
2412         dowarn = 1
2413         if x not in qawarnings:
2414                 dofail = 1
2415
2416 if dofail or \
2417         (dowarn and not (options.quiet or options.mode == "scan")):
2418         dofull = 0
2419
2420 # Save QA output so that it can be conveniently displayed
2421 # in $EDITOR while the user creates a commit message.
2422 # Otherwise, the user would not be able to see this output
2423 # once the editor has taken over the screen.
2424 qa_output = io.StringIO()
2425 style_file = ConsoleStyleFile(sys.stdout)
2426 if options.mode == 'commit' and \
2427         (not commitmessage or not commitmessage.strip()):
2428         style_file.write_listener = qa_output
2429 console_writer = StyleWriter(file=style_file, maxcol=9999)
2430 console_writer.style_listener = style_file.new_styles
2431
2432 f = formatter.AbstractFormatter(console_writer)
2433
2434 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2435
2436 style_file.flush()
2437 del console_writer, f, style_file
2438 qa_output = qa_output.getvalue()
2439 qa_output = qa_output.splitlines(True)
2440
2441 suggest_ignore_masked = False
2442 suggest_include_dev = False
2443
2444 if have_pmasked and not (options.without_mask or options.ignore_masked):
2445         suggest_ignore_masked = True
2446 if have_dev_keywords and not options.include_dev:
2447         suggest_include_dev = True
2448
2449 if suggest_ignore_masked or suggest_include_dev:
2450         print()
2451         if suggest_ignore_masked:
2452                 print(bold("Note: use --without-mask to check " + \
2453                         "KEYWORDS on dependencies of masked packages"))
2454
2455         if suggest_include_dev:
2456                 print(bold("Note: use --include-dev (-d) to check " + \
2457                         "dependencies for 'dev' profiles"))
2458         print()
2459
2460 if options.mode != 'commit':
2461         if dofull:
2462                 print(bold("Note: type \"repoman full\" for a complete listing."))
2463         if dowarn and not dofail:
2464                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2465         elif not dofail:
2466                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2467         elif dofail:
2468                 print(bad("Please fix these important QA issues first."))
2469                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2470                 sys.exit(1)
2471 else:
2472         if dofail and can_force and options.force and not options.pretend:
2473                 print(green("RepoMan sez:") + \
2474                         " \"You want to commit even with these QA issues?\n" + \
2475                         "              I'll take it this time, but I'm not happy.\"\n")
2476         elif dofail:
2477                 if options.force and not can_force:
2478                         print(bad("The --force option has been disabled due to extraordinary issues."))
2479                 print(bad("Please fix these important QA issues first."))
2480                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2481                 sys.exit(1)
2482
2483         if options.pretend:
2484                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2485
2486         myunadded = []
2487         if vcs == "cvs":
2488                 try:
2489                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2490                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2491                 except SystemExit as e:
2492                         raise  # TODO propagate this
2493                 except:
2494                         err("Error retrieving CVS tree; exiting.")
2495         if vcs == "svn":
2496                 try:
2497                         with repoman_popen("svn status --no-ignore") as f:
2498                                 svnstatus = f.readlines()
2499                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2500                 except SystemExit as e:
2501                         raise  # TODO propagate this
2502                 except:
2503                         err("Error retrieving SVN info; exiting.")
2504         if vcs == "git":
2505                 # get list of files not under version control or missing
2506                 myf = repoman_popen("git ls-files --others")
2507                 myunadded = ["./" + elem[:-1] for elem in myf]
2508                 myf.close()
2509         if vcs == "bzr":
2510                 try:
2511                         with repoman_popen("bzr status -S .") as f:
2512                                 bzrstatus = f.readlines()
2513                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2514                 except SystemExit as e:
2515                         raise  # TODO propagate this
2516                 except:
2517                         err("Error retrieving bzr info; exiting.")
2518         if vcs == "hg":
2519                 with repoman_popen("hg status --no-status --unknown .") as f:
2520                         myunadded = f.readlines()
2521                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2522
2523                 # Mercurial doesn't handle manually deleted files as removed from
2524                 # the repository, so the user need to remove them before commit,
2525                 # using "hg remove [FILES]"
2526                 with repoman_popen("hg status --no-status --deleted .") as f:
2527                         mydeleted = f.readlines()
2528                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2529
2530
2531         myautoadd = []
2532         if myunadded:
2533                 for x in range(len(myunadded)-1, -1, -1):
2534                         xs = myunadded[x].split("/")
2535                         if xs[-1] == "files":
2536                                 print("!!! files dir is not added! Please correct this.")
2537                                 sys.exit(-1)
2538                         elif xs[-1] == "Manifest":
2539                                 # It's a manifest... auto add
2540                                 myautoadd += [myunadded[x]]
2541                                 del myunadded[x]
2542
2543         if myunadded:
2544                 print(red("!!! The following files are in your local tree but are not added to the master"))
2545                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2546                 for x in myunadded:
2547                         print("   ", x)
2548                 print()
2549                 print()
2550                 sys.exit(1)
2551
2552         if vcs == "hg" and mydeleted:
2553                 print(red("!!! The following files are removed manually from your local tree but are not"))
2554                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2555                 for x in mydeleted:
2556                         print("   ", x)
2557                 print()
2558                 print()
2559                 sys.exit(1)
2560
2561         if vcs == "cvs":
2562                 mycvstree = cvstree.getentries("./", recursive=1)
2563                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2564                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2565                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2566                 bin_blob_pattern = re.compile("^-kb$")
2567                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2568                         recursive=1, basedir="./"))
2569
2570         if vcs == "svn":
2571                 with repoman_popen("svn status") as f:
2572                         svnstatus = f.readlines()
2573                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2574                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2575                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2576
2577                 # Subversion expands keywords specified in svn:keywords properties.
2578                 with repoman_popen("svn propget -R svn:keywords") as f:
2579                         props = f.readlines()
2580                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2581                         for prop in props if " - " in prop)
2582
2583         elif vcs == "git":
2584                 with repoman_popen("git diff-index --name-only "
2585                         "--relative --diff-filter=M HEAD") as f:
2586                         mychanged = f.readlines()
2587                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2588
2589                 with repoman_popen("git diff-index --name-only "
2590                         "--relative --diff-filter=A HEAD") as f:
2591                         mynew = f.readlines()
2592                 mynew = ["./" + elem[:-1] for elem in mynew]
2593
2594                 with repoman_popen("git diff-index --name-only "
2595                         "--relative --diff-filter=D HEAD") as f:
2596                         myremoved = f.readlines()
2597                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2598
2599         if vcs == "bzr":
2600                 with repoman_popen("bzr status -S .") as f:
2601                         bzrstatus = f.readlines()
2602                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2603                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2604                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2605                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2606                 # Bazaar expands nothing.
2607
2608         if vcs == "hg":
2609                 with repoman_popen("hg status --no-status --modified .") as f:
2610                         mychanged = f.readlines()
2611                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2612
2613                 with repoman_popen("hg status --no-status --added .") as f:
2614                         mynew = f.readlines()
2615                 mynew = ["./" + elem.rstrip() for elem in mynew]
2616
2617                 with repoman_popen("hg status --no-status --removed .") as f:
2618                         myremoved = f.readlines()
2619                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2620
2621         if vcs:
2622                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2623                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2624                         print()
2625                         print("(Didn't find any changed files...)")
2626                         print()
2627                         sys.exit(1)
2628
2629         # Manifests need to be regenerated after all other commits, so don't commit
2630         # them now even if they have changed.
2631         mymanifests = set()
2632         myupdates = set()
2633         for f in mychanged + mynew:
2634                 if "Manifest" == os.path.basename(f):
2635                         mymanifests.add(f)
2636                 else:
2637                         myupdates.add(f)
2638         myupdates.difference_update(myremoved)
2639         myupdates = list(myupdates)
2640         mymanifests = list(mymanifests)
2641         myheaders = []
2642         mydirty = []
2643
2644         commitmessage = options.commitmsg
2645         if options.commitmsgfile:
2646                 try:
2647                         f = io.open(_unicode_encode(options.commitmsgfile,
2648                         encoding=_encodings['fs'], errors='strict'),
2649                         mode='r', encoding=_encodings['content'], errors='replace')
2650                         commitmessage = f.read()
2651                         f.close()
2652                         del f
2653                 except (IOError, OSError) as e:
2654                         if e.errno == errno.ENOENT:
2655                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2656                         else:
2657                                 raise
2658                 # We've read the content so the file is no longer needed.
2659                 commitmessagefile = None
2660         if not commitmessage or not commitmessage.strip():
2661                 try:
2662                         editor = os.environ.get("EDITOR")
2663                         if editor and utilities.editor_is_executable(editor):
2664                                 commitmessage = utilities.get_commit_message_with_editor(
2665                                         editor, message=qa_output)
2666                         else:
2667                                 commitmessage = utilities.get_commit_message_with_stdin()
2668                 except KeyboardInterrupt:
2669                         exithandler()
2670                 if not commitmessage or not commitmessage.strip():
2671                         print("* no commit message?  aborting commit.")
2672                         sys.exit(1)
2673         commitmessage = commitmessage.rstrip()
2674         changelog_msg = commitmessage
2675         portage_version = getattr(portage, "VERSION", None)
2676         gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
2677         dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
2678         if portage_version is None:
2679                 sys.stderr.write("Failed to insert portage version in message!\n")
2680                 sys.stderr.flush()
2681                 portage_version = "Unknown"
2682
2683         report_options = []
2684         if options.force:
2685                 report_options.append("--force")
2686         if options.ignore_arches:
2687                 report_options.append("--ignore-arches")
2688         if include_arches is not None:
2689                 report_options.append("--include-arches=\"%s\"" %
2690                         " ".join(sorted(include_arches)))
2691
2692         if vcs == "git":
2693                 # Use new footer only for git (see bug #438364).
2694                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2695                 if report_options:
2696                         commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
2697                 if sign_manifests:
2698                         commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
2699                 if dco_sob:
2700                         commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
2701         else:
2702                 unameout = platform.system() + " "
2703                 if platform.system() in ["Darwin", "SunOS"]:
2704                         unameout += platform.processor()
2705                 else:
2706                         unameout += platform.machine()
2707                 commit_footer = "\n\n"
2708                 if dco_sob:
2709                         commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
2710                 commit_footer += "(Portage version: %s/%s/%s" % \
2711                         (portage_version, vcs, unameout)
2712                 if report_options:
2713                         commit_footer += ", RepoMan options: " + " ".join(report_options)
2714                 if sign_manifests:
2715                         commit_footer += ", signed Manifest commit with key %s" % \
2716                                 (gpg_key, )
2717                 else:
2718                         commit_footer += ", unsigned Manifest commit"
2719                 commit_footer += ")"
2720
2721         commitmessage += commit_footer
2722
2723         broken_changelog_manifests = []
2724         if options.echangelog in ('y', 'force'):
2725                 logging.info("checking for unmodified ChangeLog files")
2726                 committer_name = utilities.get_committer_name(env=repoman_settings)
2727                 for x in sorted(vcs_files_to_cps(
2728                         chain(myupdates, mymanifests, myremoved))):
2729                         catdir, pkgdir = x.split("/")
2730                         checkdir = repodir + "/" + x
2731                         checkdir_relative = ""
2732                         if repolevel < 3:
2733                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2734                         if repolevel < 2:
2735                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2736                         checkdir_relative = os.path.join(".", checkdir_relative)
2737
2738                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2739                         changelog_modified = changelog_path in modified_changelogs
2740                         if changelog_modified and options.echangelog != 'force':
2741                                 continue
2742
2743                         # get changes for this package
2744                         cdrlen = len(checkdir_relative)
2745                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2746                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2747                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2748
2749                         # Skip ChangeLog generation if only the Manifest was modified,
2750                         # as discussed in bug #398009.
2751                         nontrivial_cl_files = set()
2752                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2753                         nontrivial_cl_files.difference_update(['Manifest'])
2754                         if not nontrivial_cl_files and options.echangelog != 'force':
2755                                 continue
2756
2757                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2758                                 committer_name, changelog_msg,
2759                                 os.path.join(repodir, 'skel.ChangeLog'),
2760                                 catdir, pkgdir,
2761                                 new=clnew, removed=clremoved, changed=clchanged,
2762                                 pretend=options.pretend)
2763                         if new_changelog is None:
2764                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2765                                         level=logging.ERROR, noiselevel=-1)
2766                                 sys.exit(1)
2767
2768                         # if the ChangeLog was just created, add it to vcs
2769                         if new_changelog:
2770                                 myautoadd.append(changelog_path)
2771                                 # myautoadd is appended to myupdates below
2772                         else:
2773                                 myupdates.append(changelog_path)
2774
2775                         if options.ask and not options.pretend:
2776                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2777                                 repoman_settings["O"] = checkdir
2778                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2779                         else:
2780                                 broken_changelog_manifests.append(x)
2781
2782         if myautoadd:
2783                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2784                 add_cmd = [vcs, "add"]
2785                 add_cmd += myautoadd
2786                 if options.pretend:
2787                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2788                                 noiselevel=-1)
2789                 else:
2790
2791                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2792                                 not os.path.isabs(add_cmd[0]):
2793                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2794                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2795                                 fullname = find_binary(add_cmd[0])
2796                                 if fullname is None:
2797                                         raise portage.exception.CommandNotFound(add_cmd[0])
2798                                 add_cmd[0] = fullname
2799
2800                         add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2801                         retcode = subprocess.call(add_cmd)
2802                         if retcode != os.EX_OK:
2803                                 logging.error(
2804                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2805                                 sys.exit(retcode)
2806
2807                 myupdates += myautoadd
2808
2809         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2810
2811         if vcs not in ('cvs', 'svn'):
2812                 # With git, bzr and hg, there's never any keyword expansion, so
2813                 # there's no need to regenerate manifests and all files will be
2814                 # committed in one big commit at the end.
2815                 print()
2816         elif not repo_config.thin_manifest:
2817                 if vcs == 'cvs':
2818                         headerstring = "'\$(Header|Id).*\$'"
2819                 elif vcs == "svn":
2820                         svn_keywords = dict((k.lower(), k) for k in [
2821                                         "Rev",
2822                                         "Revision",
2823                                         "LastChangedRevision",
2824                                         "Date",
2825                                         "LastChangedDate",
2826                                         "Author",
2827                                         "LastChangedBy",
2828                                         "URL",
2829                                         "HeadURL",
2830                                         "Id",
2831                                         "Header",
2832                         ])
2833
2834                 for myfile in myupdates:
2835
2836                         # for CVS, no_expansion contains files that are excluded from expansion
2837                         if vcs == "cvs":
2838                                 if myfile in no_expansion:
2839                                         continue
2840
2841                         # for SVN, expansion contains files that are included in expansion
2842                         elif vcs == "svn":
2843                                 if myfile not in expansion:
2844                                         continue
2845
2846                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2847                                 enabled_keywords = []
2848                                 for k in expansion[myfile]:
2849                                         keyword = svn_keywords.get(k.lower())
2850                                         if keyword is not None:
2851                                                 enabled_keywords.append(keyword)
2852
2853                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2854
2855                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2856                                 portage._shell_quote(myfile))
2857                         if myout[0] == 0:
2858                                 myheaders.append(myfile)
2859
2860                 print("%s have headers that will change." % green(str(len(myheaders))))
2861                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2862
2863         logging.info("myupdates: %s", myupdates)
2864         logging.info("myheaders: %s", myheaders)
2865
2866         if options.ask and userquery('Commit changes?', True) != 'Yes':
2867                 print("* aborting commit.")
2868                 sys.exit(128 + signal.SIGINT)
2869
2870         # Handle the case where committed files have keywords which
2871         # will change and need a priming commit before the Manifest
2872         # can be committed.
2873         if (myupdates or myremoved) and myheaders:
2874                 myfiles = myupdates + myremoved
2875                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2876                 mymsg = os.fdopen(fd, "wb")
2877                 mymsg.write(_unicode_encode(commitmessage))
2878                 mymsg.close()
2879
2880                 print()
2881                 print(green("Using commit message:"))
2882                 print(green("------------------------------------------------------------------------------"))
2883                 print(commitmessage)
2884                 print(green("------------------------------------------------------------------------------"))
2885                 print()
2886
2887                 # Having a leading ./ prefix on file paths can trigger a bug in
2888                 # the cvs server when committing files to multiple directories,
2889                 # so strip the prefix.
2890                 myfiles = [f.lstrip("./") for f in myfiles]
2891
2892                 commit_cmd = [vcs]
2893                 commit_cmd.extend(vcs_global_opts)
2894                 commit_cmd.append("commit")
2895                 commit_cmd.extend(vcs_local_opts)
2896                 commit_cmd.extend(["-F", commitmessagefile])
2897                 commit_cmd.extend(myfiles)
2898
2899                 try:
2900                         if options.pretend:
2901                                 print("(%s)" % (" ".join(commit_cmd),))
2902                         else:
2903                                 retval = spawn(commit_cmd, env=commit_env)
2904                                 if retval != os.EX_OK:
2905                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2906                                                 "error code: %s\n") % (vcs, retval),
2907                                                 level=logging.ERROR, noiselevel=-1)
2908                                         sys.exit(retval)
2909                 finally:
2910                         try:
2911                                 os.unlink(commitmessagefile)
2912                         except OSError:
2913                                 pass
2914
2915         # Setup the GPG commands
2916         def gpgsign(filename):
2917                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2918                 if gpgcmd is None:
2919                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2920                                 " Is make.globals missing?")
2921                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2922                         "PORTAGE_GPG_KEY" not in repoman_settings:
2923                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2924                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2925                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2926                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2927                                         os.path.expanduser("~/.gnupg")
2928                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2929                                         % repoman_settings["PORTAGE_GPG_DIR"])
2930                         else:
2931                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2932                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2933                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2934                                 raise portage.exception.InvalidLocation(
2935                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2936                                         repoman_settings["PORTAGE_GPG_DIR"])
2937                 gpgvars = {"FILE": filename}
2938                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2939                         v = repoman_settings.get(k)
2940                         if v is not None:
2941                                 gpgvars[k] = v
2942                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2943                 if options.pretend:
2944                         print("(" + gpgcmd + ")")
2945                 else:
2946                         # Encode unicode manually for bug #310789.
2947                         gpgcmd = portage.util.shlex_split(gpgcmd)
2948
2949                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2950                                 not os.path.isabs(gpgcmd[0]):
2951                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2952                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2953                                 fullname = find_binary(gpgcmd[0])
2954                                 if fullname is None:
2955                                         raise portage.exception.CommandNotFound(gpgcmd[0])
2956                                 gpgcmd[0] = fullname
2957
2958                         gpgcmd = [_unicode_encode(arg,
2959                                 encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2960                         rValue = subprocess.call(gpgcmd)
2961                         if rValue == os.EX_OK:
2962                                 os.rename(filename + ".asc", filename)
2963                         else:
2964                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2965
2966         def need_signature(filename):
2967                 try:
2968                         with open(_unicode_encode(filename,
2969                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2970                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2971                 except IOError as e:
2972                         if e.errno in (errno.ENOENT, errno.ESTALE):
2973                                 return False
2974                         raise
2975
2976         # When files are removed and re-added, the cvs server will put /Attic/
2977         # inside the $Header path. This code detects the problem and corrects it
2978         # so that the Manifest will generate correctly. See bug #169500.
2979         # Use binary mode in order to avoid potential character encoding issues.
2980         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2981         attic_str = b'/Attic/'
2982         attic_replace = b'/'
2983         for x in myheaders:
2984                 f = open(_unicode_encode(x,
2985                         encoding=_encodings['fs'], errors='strict'),
2986                         mode='rb')
2987                 mylines = f.readlines()
2988                 f.close()
2989                 modified = False
2990                 for i, line in enumerate(mylines):
2991                         if cvs_header_re.match(line) is not None and \
2992                                 attic_str in line:
2993                                 mylines[i] = line.replace(attic_str, attic_replace)
2994                                 modified = True
2995                 if modified:
2996                         portage.util.write_atomic(x, b''.join(mylines),
2997                                 mode='wb')
2998
2999         if repolevel == 1:
3000                 print(green("RepoMan sez:"), "\"You're rather crazy... "
3001                         "doing the entire repository.\"\n")
3002
3003         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
3004
3005                 for x in sorted(vcs_files_to_cps(
3006                         chain(myupdates, myremoved, mymanifests))):
3007                         repoman_settings["O"] = os.path.join(repodir, x)
3008                         digestgen(mysettings=repoman_settings, myportdb=portdb)
3009
3010         elif broken_changelog_manifests:
3011                 for x in broken_changelog_manifests:
3012                         repoman_settings["O"] = os.path.join(repodir, x)
3013                         digestgen(mysettings=repoman_settings, myportdb=portdb)
3014
3015         signed = False
3016         if sign_manifests:
3017                 signed = True
3018                 try:
3019                         for x in sorted(vcs_files_to_cps(
3020                                 chain(myupdates, myremoved, mymanifests))):
3021                                 repoman_settings["O"] = os.path.join(repodir, x)
3022                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
3023                                 if not need_signature(manifest_path):
3024                                         continue
3025                                 gpgsign(manifest_path)
3026                 except portage.exception.PortageException as e:
3027                         portage.writemsg("!!! %s\n" % str(e))
3028                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
3029                         signed = False
3030
3031         if vcs == 'git':
3032                 # It's not safe to use the git commit -a option since there might
3033                 # be some modified files elsewhere in the working tree that the
3034                 # user doesn't want to commit. Therefore, call git update-index
3035                 # in order to ensure that the index is updated with the latest
3036                 # versions of all new and modified files in the relevant portion
3037                 # of the working tree.
3038                 myfiles = mymanifests + myupdates
3039                 myfiles.sort()
3040                 update_index_cmd = ["git", "update-index"]
3041                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
3042                 if options.pretend:
3043                         print("(%s)" % (" ".join(update_index_cmd),))
3044                 else:
3045                         retval = spawn(update_index_cmd, env=os.environ)
3046                         if retval != os.EX_OK:
3047                                 writemsg_level(("!!! Exiting on %s (shell) " + \
3048                                         "error code: %s\n") % (vcs, retval),
3049                                         level=logging.ERROR, noiselevel=-1)
3050                                 sys.exit(retval)
3051
3052         if True:
3053                 myfiles = mymanifests[:]
3054                 # If there are no header (SVN/CVS keywords) changes in
3055                 # the files, this Manifest commit must include the
3056                 # other (yet uncommitted) files.
3057                 if not myheaders:
3058                         myfiles += myupdates
3059                         myfiles += myremoved
3060                 myfiles.sort()
3061
3062                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3063                 mymsg = os.fdopen(fd, "wb")
3064                 mymsg.write(_unicode_encode(commitmessage))
3065                 mymsg.close()
3066
3067                 commit_cmd = []
3068                 if options.pretend and vcs is None:
3069                         # substitute a bogus value for pretend output
3070                         commit_cmd.append("cvs")
3071                 else:
3072                         commit_cmd.append(vcs)
3073                 commit_cmd.extend(vcs_global_opts)
3074                 commit_cmd.append("commit")
3075                 commit_cmd.extend(vcs_local_opts)
3076                 if vcs == "hg":
3077                         commit_cmd.extend(["--logfile", commitmessagefile])
3078                         commit_cmd.extend(myfiles)
3079                 else:
3080                         commit_cmd.extend(["-F", commitmessagefile])
3081                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3082
3083                 try:
3084                         if options.pretend:
3085                                 print("(%s)" % (" ".join(commit_cmd),))
3086                         else:
3087                                 retval = spawn(commit_cmd, env=commit_env)
3088                                 if retval != os.EX_OK:
3089                                         if repo_config.sign_commit and vcs == 'git' and \
3090                                                 not git_supports_gpg_sign():
3091                                                 # Inform user that newer git is needed (bug #403323).
3092                                                 logging.error(
3093                                                         "Git >=1.7.9 is required for signed commits!")
3094
3095                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3096                                                 "error code: %s\n") % (vcs, retval),
3097                                                 level=logging.ERROR, noiselevel=-1)
3098                                         sys.exit(retval)
3099                 finally:
3100                         try:
3101                                 os.unlink(commitmessagefile)
3102                         except OSError:
3103                                 pass
3104
3105         print()
3106         if vcs:
3107                 print("Commit complete.")
3108         else:
3109                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3110         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3111 sys.exit(0)