bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import codecs
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27 from itertools import chain
  28 from stat import S_ISDIR
  29
  30 try:
  31         from urllib.parse import urlparse
  32 except ImportError:
  33         from urlparse import urlparse
  34
  35 from os import path as osp
  36 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  37 sys.path.insert(0, pym_path)
  38 import portage
  39 portage._internal_caller = True
  40 portage._disable_legacy_globals()
  41
  42 try:
  43         import xml.etree.ElementTree
  44         from xml.parsers.expat import ExpatError
  45 except (SystemExit, KeyboardInterrupt):
  46         raise
  47 except (ImportError, SystemError, RuntimeError, Exception):
  48         # broken or missing xml support
  49         # http://bugs.python.org/issue14988
  50         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  51         from portage.output import EOutput
  52         out = EOutput()
  53         for line in msg:
  54                 out.eerror(line)
  55         sys.exit(1)
  56
  57 from portage import os
  58 from portage import _encodings
  59 from portage import _unicode_encode
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 from portage import cvstree, normalize_path
  69 from portage import util
  70 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  71         ParseError, PermissionDenied)
  72 from portage.dep import Atom
  73 from portage.process import find_binary, spawn
  74 from portage.output import bold, create_color_func, \
  75         green, nocolor, red
  76 from portage.output import ConsoleStyleFile, StyleWriter
  77 from portage.util import writemsg_level
  78 from portage.package.ebuild.digestgen import digestgen
  79 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  80
  81 if sys.hexversion >= 0x3000000:
  82         basestring = str
  83
  84 util.initialize_logger()
  85
  86 # 14 is the length of DESCRIPTION=""
  87 max_desc_len = 100
  88 allowed_filename_chars="a-zA-Z0-9._-+:"
  89 pv_toolong_re = re.compile(r'[0-9]{19,}')
  90 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  91 bad = create_color_func("BAD")
  92
  93 # A sane umask is needed for files that portage creates.
  94 os.umask(0o22)
  95 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  96 # behave incrementally.
  97 repoman_incrementals = tuple(x for x in \
  98         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
  99 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 100 repoman_settings = portage.config(config_root=config_root, local_config=False)
 101
 102 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 103         repoman_settings.get('TERM') == 'dumb' or \
 104         not sys.stdout.isatty():
 105         nocolor()
 106
 107 def warn(txt):
 108         print("repoman: " + txt)
 109
 110 def err(txt):
 111         warn(txt)
 112         sys.exit(1)
 113
 114 def exithandler(signum=None, frame=None):
 115         logging.fatal("Interrupted; exiting...")
 116         if signum is None:
 117                 sys.exit(1)
 118         else:
 119                 sys.exit(128 + signum)
 120
 121 signal.signal(signal.SIGINT, exithandler)
 122
 123 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 124         """Repoman needs its own HelpFormatter for now, because the default ones
 125         murder the help text."""
 126
 127         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 128                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 129
 130         def format_description(self, description):
 131                 return description
 132
 133 class RepomanOptionParser(optparse.OptionParser):
 134         """Add the on_tail function, ruby has it, optionParser should too
 135         """
 136
 137         def __init__(self, *args, **kwargs):
 138                 optparse.OptionParser.__init__(self, *args, **kwargs)
 139                 self.tail = ""
 140
 141         def on_tail(self, description):
 142                 self.tail += description
 143
 144         def format_help(self, formatter=None):
 145                 result = optparse.OptionParser.format_help(self, formatter)
 146                 result += self.tail
 147                 return result
 148
 149
 150 def ParseArgs(argv, qahelp):
 151         """This function uses a customized optionParser to parse command line arguments for repoman
 152         Args:
 153           argv - a sequence of command line arguments
 154                 qahelp - a dict of qa warning to help message
 155         Returns:
 156           (opts, args), just like a call to parser.parse_args()
 157         """
 158
 159         if argv and isinstance(argv[0], bytes):
 160                 argv = [portage._unicode_decode(x) for x in argv]
 161
 162         modes = {
 163                 'commit' : 'Run a scan then commit changes',
 164                 'ci' : 'Run a scan then commit changes',
 165                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 166                 'full' : 'Scan directory tree and print all issues (not a summary)',
 167                 'help' : 'Show this screen',
 168                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 169                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 170                 'scan' : 'Scan directory tree for QA issues'
 171         }
 172
 173         mode_keys = list(modes)
 174         mode_keys.sort()
 175
 176         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 177         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 178         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 179         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 180         parser.description += "\nmodes: " + " | ".join(map(green, mode_keys))
 181
 182         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 183                 help='Request a confirmation before commiting')
 184
 185         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 186                 help='specify a commit message on the command line')
 187
 188         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 189                 help='specify a path to a file that contains a commit message')
 190
 191         parser.add_option('--digest',
 192                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 193                 help='Automatically update Manifest digests for modified files')
 194
 195         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 196                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 197
 198         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 199                 help='do not print unnecessary messages')
 200
 201         parser.add_option(
 202                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 203                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 204                 'regardless of modification if \'force\' is specified)')
 205
 206         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 207                 help='Commit with QA violations')
 208
 209         parser.add_option('--vcs', dest='vcs',
 210                 help='Force using specific VCS instead of autodetection')
 211
 212         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 213                 help='be very verbose in output', default=0)
 214
 215         parser.add_option('-V', '--version', dest='version', action='store_true',
 216                 help='show version info')
 217
 218         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 219                 default=False, help='forces the metadata.xml parse check to be carried out')
 220
 221         parser.add_option(
 222                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 223                 metavar="<y|n>",
 224                 help='only check packages that have uncommitted modifications')
 225
 226         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 227                 default=False, help='ignore arch-specific failures (where arch != host)')
 228
 229         parser.add_option("--ignore-default-opts",
 230                 action="store_true",
 231                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 232
 233         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 234                 default=False, help='ignore masked packages (not allowed with commit mode)')
 235
 236         parser.add_option('--include-arches', dest='include_arches',
 237                 metavar='ARCHES', action='append',
 238                 help='A space separated list of arches used to '
 239                 'filter the selection of profiles for dependency checks')
 240
 241         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 242                 default=False, help='include dev profiles in dependency checks')
 243
 244         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 245                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 246
 247         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 248                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 249
 250         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 251                 help='specify which mode repoman will run in (default=full)')
 252
 253         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 254
 255         for k in mode_keys:
 256                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 257
 258         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 259
 260         sorted_qa = list(qahelp)
 261         sorted_qa.sort()
 262         for k in sorted_qa:
 263                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 264
 265         opts, args = parser.parse_args(argv[1:])
 266
 267         if not opts.ignore_default_opts:
 268                 default_opts = portage.util.shlex_split(
 269                         repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
 270                 if default_opts:
 271                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 272
 273         if opts.mode == 'help':
 274                 parser.print_help(short=False)
 275
 276         for arg in args:
 277                 if arg in modes:
 278                         if not opts.mode:
 279                                 opts.mode = arg
 280                                 break
 281                 else:
 282                         parser.error("invalid mode: %s" % arg)
 283
 284         if not opts.mode:
 285                 opts.mode = 'full'
 286
 287         if opts.mode == 'ci':
 288                 opts.mode = 'commit'  # backwards compat shortcut
 289
 290         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 291                 if opts.ignore_masked:
 292                         parser.error('Commit mode and --ignore-masked are not compatible')
 293                 if opts.without_mask:
 294                         parser.error('Commit mode and --without-mask are not compatible')
 295
 296         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 297         for val in range(opts.verbosity):
 298                 logger = logging.getLogger()
 299                 logger.setLevel(logger.getEffectiveLevel() - 10)
 300
 301         for val in range(opts.quiet):
 302                 logger = logging.getLogger()
 303                 logger.setLevel(logger.getEffectiveLevel() + 10)
 304
 305         return (opts, args)
 306
 307 qahelp = {
 308         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 309         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 310         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 311         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 312         "changelog.missing": "Missing ChangeLog files",
 313         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 314         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 315         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 316         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 317         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 318         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 319         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 320         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 321         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 322         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 323         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 324         "file.size": "Files in the files directory must be under 20 KiB",
 325         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 326         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 327         "file.UTF8": "File is not UTF8 compliant",
 328         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 329         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 330         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 331         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 332         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 333         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 334         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 335         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 336         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 337         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 338         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 339         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 340         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 341         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 342         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 343         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 344         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 345         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 346         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 347         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 348         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 349         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 350         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 351         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 352         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 353         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 354         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 355         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 356         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 357         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 358         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 359         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 360         "variable.readonly": "Assigning a readonly variable",
 361         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 362         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 363         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 364         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 365         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 366         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 367         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 368         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 369         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 370         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 371         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 372         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 373         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 374         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 375         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 376         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 377         "ebuild.badheader": "This ebuild has a malformed header",
 378         "manifest.bad": "Manifest has missing or incorrect digests",
 379         "metadata.missing": "Missing metadata.xml files",
 380         "metadata.bad": "Bad metadata.xml files",
 381         "metadata.warning": "Warnings in metadata.xml files",
 382         "portage.internal": "The ebuild uses an internal Portage function or variable",
 383         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 384         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 385         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 386         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 387 }
 388
 389 qacats = list(qahelp)
 390 qacats.sort()
 391
 392 qawarnings = set((
 393 "changelog.missing",
 394 "changelog.notadded",
 395 "dependency.unknown",
 396 "digest.assumed",
 397 "digest.unused",
 398 "ebuild.notadded",
 399 "ebuild.nesteddie",
 400 "dependency.badmasked",
 401 "dependency.badindev",
 402 "dependency.badmaskedindev",
 403 "dependency.badtilde",
 404 "DESCRIPTION.toolong",
 405 "EAPI.deprecated",
 406 "HOMEPAGE.virtual",
 407 "LICENSE.deprecated",
 408 "LICENSE.virtual",
 409 "KEYWORDS.dropped",
 410 "KEYWORDS.stupid",
 411 "KEYWORDS.missing",
 412 "PDEPEND.suspect",
 413 "RDEPEND.implicit",
 414 "RDEPEND.suspect",
 415 "virtual.suspect",
 416 "RESTRICT.invalid",
 417 "ebuild.minorsyn",
 418 "ebuild.badheader",
 419 "ebuild.patches",
 420 "file.size",
 421 "inherit.unused",
 422 "inherit.deprecated",
 423 "java.eclassesnotused",
 424 "wxwidgets.eclassnotused",
 425 "metadata.warning",
 426 "portage.internal",
 427 "usage.obsolete",
 428 "upstream.workaround",
 429 "LIVEVCS.stable",
 430 "LIVEVCS.unmasked",
 431 ))
 432
 433 if portage.const._ENABLE_INHERIT_CHECK:
 434         # This is experimental, so it's non-fatal.
 435         qawarnings.add("inherit.missing")
 436
 437 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 438
 439 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 440 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 441 allvars.update(Package.metadata_keys)
 442 allvars = sorted(allvars)
 443 commitmessage = None
 444 for x in missingvars:
 445         x += ".missing"
 446         if x not in qacats:
 447                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 448                 qacats.append(x)
 449                 qawarnings.add(x)
 450
 451 valid_restrict = frozenset(["binchecks", "bindist",
 452         "fetch", "installsources", "mirror", "preserve-libs",
 453         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 454
 455 live_eclasses = frozenset([
 456         "bzr",
 457         "cvs",
 458         "darcs",
 459         "git",
 460         "git-2",
 461         "mercurial",
 462         "subversion",
 463         "tla",
 464 ])
 465
 466 suspect_rdepend = frozenset([
 467         "app-arch/cabextract",
 468         "app-arch/rpm2targz",
 469         "app-doc/doxygen",
 470         "dev-lang/nasm",
 471         "dev-lang/swig",
 472         "dev-lang/yasm",
 473         "dev-perl/extutils-pkgconfig",
 474         "dev-util/byacc",
 475         "dev-util/cmake",
 476         "dev-util/ftjam",
 477         "dev-util/gperf",
 478         "dev-util/gtk-doc",
 479         "dev-util/gtk-doc-am",
 480         "dev-util/intltool",
 481         "dev-util/jam",
 482         "dev-util/pkg-config-lite",
 483         "dev-util/pkgconf",
 484         "dev-util/pkgconfig",
 485         "dev-util/pkgconfig-openbsd",
 486         "dev-util/scons",
 487         "dev-util/unifdef",
 488         "dev-util/yacc",
 489         "media-gfx/ebdftopcf",
 490         "sys-apps/help2man",
 491         "sys-devel/autoconf",
 492         "sys-devel/automake",
 493         "sys-devel/bin86",
 494         "sys-devel/bison",
 495         "sys-devel/dev86",
 496         "sys-devel/flex",
 497         "sys-devel/m4",
 498         "sys-devel/pmake",
 499         "virtual/linux-sources",
 500         "virtual/pkgconfig",
 501         "x11-misc/bdftopcf",
 502         "x11-misc/imake",
 503 ])
 504
 505 suspect_virtual = {
 506         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 507         "dev-util/pkgconf":"virtual/pkgconfig",
 508         "dev-util/pkgconfig":"virtual/pkgconfig",
 509         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 510         "dev-libs/libusb":"virtual/libusb",
 511         "dev-libs/libusbx":"virtual/libusb",
 512         "dev-libs/libusb-compat":"virtual/libusb",
 513 }
 514
 515 metadata_xml_encoding = 'UTF-8'
 516 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 517         (metadata_xml_encoding,)
 518 metadata_doctype_name = 'pkgmetadata'
 519 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 520 # force refetch if the local copy creation time is older than this
 521 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 522
 523 # file.executable
 524 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 525
 526 options, arguments = ParseArgs(sys.argv, qahelp)
 527
 528 if options.version:
 529         print("Portage", portage.VERSION)
 530         sys.exit(0)
 531
 532 # Set this to False when an extraordinary issue (generally
 533 # something other than a QA issue) makes it impossible to
 534 # commit (like if Manifest generation fails).
 535 can_force = True
 536
 537 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 538 if portdir is None:
 539         sys.exit(1)
 540
 541 myreporoot = os.path.basename(portdir_overlay)
 542 myreporoot += mydir[len(portdir_overlay):]
 543
 544 if options.vcs:
 545         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 546                 vcs = options.vcs
 547         else:
 548                 vcs = None
 549 else:
 550         vcses = utilities.FindVCS()
 551         if len(vcses) > 1:
 552                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 553                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 554                 sys.exit(1)
 555         elif vcses:
 556                 vcs = vcses[0]
 557         else:
 558                 vcs = None
 559
 560 if options.if_modified == "y" and vcs is None:
 561         logging.info("Not in a version controlled repository; "
 562                 "disabling --if-modified.")
 563         options.if_modified = "n"
 564
 565 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 566 vcs_preserves_mtime = vcs in ('cvs',)
 567
 568 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 569 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 570 if vcs_global_opts is None:
 571         if vcs in ('cvs', 'svn'):
 572                 vcs_global_opts = "-q"
 573         else:
 574                 vcs_global_opts = ""
 575 vcs_global_opts = vcs_global_opts.split()
 576
 577 if options.mode == 'commit' and not options.pretend and not vcs:
 578         logging.info("Not in a version controlled repository; enabling pretend mode.")
 579         options.pretend = True
 580
 581 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 582 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 583         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 584         portage._shell_quote(portdir_overlay))
 585 # We have to call the config constructor again so
 586 # that config.repositories is initialized correctly.
 587 repoman_settings = portage.config(config_root=config_root, local_config=False,
 588         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 589
 590 root = repoman_settings['EROOT']
 591 trees = {
 592         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 593 }
 594 portdb = trees[root]['porttree'].dbapi
 595
 596 # Constrain dependency resolution to the master(s)
 597 # that are specified in layout.conf.
 598 repodir = os.path.realpath(portdir_overlay)
 599 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 600 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 601 portdir = portdb.porttrees[0]
 602 commit_env = os.environ.copy()
 603
 604 if repo_config.allow_provide_virtual:
 605         qawarnings.add("virtual.oldstyle")
 606
 607 if repo_config.sign_commit:
 608         if vcs == 'git':
 609                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 610                 # the commit arguments. If key_id is unspecified, then it must be
 611                 # configured by `git config user.signingkey key_id`.
 612                 vcs_local_opts.append("--gpg-sign")
 613                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 614                         # Pass GNUPGHOME to git for bug #462362.
 615                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 616
 617 # In order to disable manifest signatures, repos may set
 618 # "sign-manifests = false" in metadata/layout.conf. This
 619 # can be used to prevent merge conflicts like those that
 620 # thin-manifests is designed to prevent.
 621 sign_manifests = "sign" in repoman_settings.features and \
 622         repo_config.sign_manifest
 623
 624 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 625         options.mode in ("commit",) and not sign_manifests:
 626         msg = ("The '%s' repository has manifest signatures enabled, "
 627         "but FEATURES=sign is currently disabled. In order to avoid this "
 628         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 629         "repositories can disable manifest signatures by setting "
 630         "'sign-manifests = false' in metadata/layout.conf.") % \
 631         (repo_config.name,)
 632         for line in textwrap.wrap(msg, 60):
 633                 logging.warn(line)
 634
 635 if sign_manifests and options.mode in ("commit",) and \
 636         repoman_settings.get("PORTAGE_GPG_KEY") and \
 637         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 638         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 639         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 640                 repoman_settings["PORTAGE_GPG_KEY"])
 641         sys.exit(1)
 642
 643 manifest_hashes = repo_config.manifest_hashes
 644 if manifest_hashes is None:
 645         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 646
 647 if options.mode in ("commit", "fix", "manifest"):
 648         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 649                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 650                 "metadata/layout.conf does not contain the '%s' hash which "
 651                 "is required by this portage version. You will have to "
 652                 "upgrade portage if you want to generate valid manifests for "
 653                 "this repository.") % \
 654                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 655                 for line in textwrap.wrap(msg, 70):
 656                         logging.error(line)
 657                 sys.exit(1)
 658
 659         unsupported_hashes = manifest_hashes.difference(
 660                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 661         if unsupported_hashes:
 662                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 663                 "metadata/layout.conf contains one or more hash types '%s' "
 664                 "which are not supported by this portage version. You will "
 665                 "have to upgrade portage if you want to generate valid "
 666                 "manifests for this repository.") % \
 667                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 668                 for line in textwrap.wrap(msg, 70):
 669                         logging.error(line)
 670                 sys.exit(1)
 671
 672 if options.echangelog is None and repo_config.update_changelog:
 673         options.echangelog = 'y'
 674
 675 if vcs is None:
 676         options.echangelog = 'n'
 677
 678 # The --echangelog option causes automatic ChangeLog generation,
 679 # which invalidates changelog.ebuildadded and changelog.missing
 680 # checks.
 681 # Note: Some don't use ChangeLogs in distributed SCMs.
 682 # It will be generated on server side from scm log,
 683 # before package moves to the rsync server.
 684 # This is needed because they try to avoid merge collisions.
 685 # Gentoo's Council decided to always use the ChangeLog file.
 686 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 687 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 688
 689 if 'digest' in repoman_settings.features and options.digest != 'n':
 690         options.digest = 'y'
 691
 692 logging.debug("vcs: %s" % (vcs,))
 693 logging.debug("repo config: %s" % (repo_config,))
 694 logging.debug("options: %s" % (options,))
 695
 696 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 697 # profile-specific config constructor calls.
 698 env = os.environ.copy()
 699 env['PORTDIR'] = portdir
 700 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 701
 702 logging.info('Setting paths:')
 703 logging.info('PORTDIR = "' + portdir + '"')
 704 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 705
 706 # It's confusing if these warnings are displayed without the user
 707 # being told which profile they come from, so disable them.
 708 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 709
 710 categories = []
 711 for path in repo_config.eclass_db.porttrees:
 712         categories.extend(portage.util.grabfile(
 713                 os.path.join(path, 'profiles', 'categories')))
 714 repoman_settings.categories = frozenset(
 715         portage.util.stack_lists([categories], incremental=1))
 716 categories = repoman_settings.categories
 717
 718 portdb.settings = repoman_settings
 719 root_config = RootConfig(repoman_settings, trees[root], None)
 720 # We really only need to cache the metadata that's necessary for visibility
 721 # filtering. Anything else can be discarded to reduce memory consumption.
 722 portdb._aux_cache_keys.clear()
 723 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 724
 725 reposplit = myreporoot.split(os.path.sep)
 726 repolevel = len(reposplit)
 727
 728 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 729 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 730 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 731 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 732         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 733         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 734         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 735         print(red("***"))
 736         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 737
 738 # Make startdir relative to the canonical repodir, so that we can pass
 739 # it to digestgen and it won't have to be canonicalized again.
 740 if repolevel == 1:
 741         startdir = repodir
 742 else:
 743         startdir = normalize_path(mydir)
 744         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 745
 746 def caterror(mycat):
 747         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 748
 749 def repoman_getstatusoutput(cmd):
 750         """
 751         Implements an interface similar to getstatusoutput(), but with
 752         customized unicode handling (see bug #310789) and without the shell.
 753         """
 754         args = portage.util.shlex_split(cmd)
 755         encoding = _encodings['fs']
 756         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 757                 # Python 3.1 does not support bytes in Popen args.
 758                 args = [_unicode_encode(x,
 759                         encoding=encoding, errors='strict') for x in args]
 760         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 761                 stderr=subprocess.STDOUT)
 762         output = portage._unicode_decode(proc.communicate()[0],
 763                 encoding=encoding, errors='strict')
 764         if output and output[-1] == "\n":
 765                 # getstatusoutput strips one newline
 766                 output = output[:-1]
 767         return (proc.wait(), output)
 768
 769 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 770         """
 771         Implements an interface similar to os.popen(), but with customized
 772         unicode handling (see bug #310789) and without the shell.
 773         """
 774
 775         __slots__ = ('_proc', '_stdout')
 776
 777         def __init__(self, cmd):
 778                 args = portage.util.shlex_split(cmd)
 779                 encoding = _encodings['fs']
 780                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 781                         # Python 3.1 does not support bytes in Popen args.
 782                         args = [_unicode_encode(x,
 783                                 encoding=encoding, errors='strict') for x in args]
 784                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 785                 object.__setattr__(self, '_proc', proc)
 786                 object.__setattr__(self, '_stdout',
 787                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 788
 789         def _get_target(self):
 790                 return object.__getattribute__(self, '_stdout')
 791
 792         __enter__ = _get_target
 793
 794         def __exit__(self, exc_type, exc_value, traceback):
 795                 proc = object.__getattribute__(self, '_proc')
 796                 proc.wait()
 797                 proc.stdout.close()
 798
 799 class ProfileDesc(object):
 800         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 801         def __init__(self, arch, status, sub_path, tree_path):
 802                 self.arch = arch
 803                 self.status = status
 804                 if sub_path:
 805                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 806                 self.sub_path = sub_path
 807                 self.tree_path = tree_path
 808                 if tree_path:
 809                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 810                 else:
 811                         self.abs_path = tree_path
 812
 813         def __str__(self):
 814                 if self.sub_path:
 815                         return self.sub_path
 816                 return 'empty profile'
 817
 818 profile_list = []
 819 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 820
 821 # get lists of valid keywords, licenses, and use
 822 kwlist = set()
 823 liclist = set()
 824 uselist = set()
 825 global_pmasklines = []
 826
 827 for path in portdb.porttrees:
 828         try:
 829                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 830         except OSError:
 831                 pass
 832         kwlist.update(portage.grabfile(os.path.join(path,
 833                 "profiles", "arch.list")))
 834
 835         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 836         for x in use_desc:
 837                 x = x.split()
 838                 if x:
 839                         uselist.add(x[0])
 840
 841         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 842         try:
 843                 expand_list = os.listdir(expand_desc_dir)
 844         except OSError:
 845                 pass
 846         else:
 847                 for fn in expand_list:
 848                         if not fn[-5:] == '.desc':
 849                                 continue
 850                         use_prefix = fn[:-5].lower() + '_'
 851                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 852                                 x = x.split()
 853                                 if x:
 854                                         uselist.add(use_prefix + x[0])
 855
 856         global_pmasklines.append(portage.util.grabfile_package(
 857                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 858
 859         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 860         try:
 861                 desc_file = io.open(_unicode_encode(desc_path,
 862                         encoding=_encodings['fs'], errors='strict'),
 863                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 864         except EnvironmentError:
 865                 pass
 866         else:
 867                 for i, x in enumerate(desc_file):
 868                         if x[0] == "#":
 869                                 continue
 870                         arch = x.split()
 871                         if len(arch) == 0:
 872                                 continue
 873                         if len(arch) != 3:
 874                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 875                                         desc_path + " line %d" % (i + 1, ))
 876                         elif arch[0] not in kwlist:
 877                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 878                                         desc_path + " line %d" % (i + 1, ))
 879                         elif arch[2] not in valid_profile_types:
 880                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 881                                         desc_path + " line %d" % (i + 1, ))
 882                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 883                         if not os.path.isdir(profile_desc.abs_path):
 884                                 logging.error(
 885                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 886                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 887                                 continue
 888                         if os.path.exists(
 889                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 890                                 continue
 891                         profile_list.append(profile_desc)
 892                 desc_file.close()
 893
 894 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 895 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 896
 897 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 898 global_pmaskdict = {}
 899 for x in global_pmasklines:
 900         global_pmaskdict.setdefault(x.cp, []).append(x)
 901 del global_pmasklines
 902
 903 def has_global_mask(pkg):
 904         mask_atoms = global_pmaskdict.get(pkg.cp)
 905         if mask_atoms:
 906                 pkg_list = [pkg]
 907                 for x in mask_atoms:
 908                         if portage.dep.match_from_list(x, pkg_list):
 909                                 return x
 910         return None
 911
 912 # Ensure that profile sub_path attributes are unique. Process in reverse order
 913 # so that profiles with duplicate sub_path from overlays will override
 914 # profiles with the same sub_path from parent repos.
 915 profiles = {}
 916 profile_list.reverse()
 917 profile_sub_paths = set()
 918 for prof in profile_list:
 919         if prof.sub_path in profile_sub_paths:
 920                 continue
 921         profile_sub_paths.add(prof.sub_path)
 922         profiles.setdefault(prof.arch, []).append(prof)
 923
 924 # Use an empty profile for checking dependencies of
 925 # packages that have empty KEYWORDS.
 926 prof = ProfileDesc('**', 'stable', '', '')
 927 profiles.setdefault(prof.arch, []).append(prof)
 928
 929 for x in repoman_settings.archlist():
 930         if x[0] == "~":
 931                 continue
 932         if x not in profiles:
 933                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 934                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 935                 print(red("up with the " + x + " team."))
 936                 print()
 937
 938 liclist_deprecated = set()
 939 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 940         liclist_deprecated.update(
 941                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 942
 943 if not liclist:
 944         logging.fatal("Couldn't find licenses?")
 945         sys.exit(1)
 946
 947 if not kwlist:
 948         logging.fatal("Couldn't read KEYWORDS from arch.list")
 949         sys.exit(1)
 950
 951 if not uselist:
 952         logging.fatal("Couldn't find use.desc?")
 953         sys.exit(1)
 954
 955 scanlist = []
 956 if repolevel == 2:
 957         # we are inside a category directory
 958         catdir = reposplit[-1]
 959         if catdir not in categories:
 960                 caterror(catdir)
 961         mydirlist = os.listdir(startdir)
 962         for x in mydirlist:
 963                 if x == "CVS" or x.startswith("."):
 964                         continue
 965                 if os.path.isdir(startdir + "/" + x):
 966                         scanlist.append(catdir + "/" + x)
 967         repo_subdir = catdir + os.sep
 968 elif repolevel == 1:
 969         for x in categories:
 970                 if not os.path.isdir(startdir + "/" + x):
 971                         continue
 972                 for y in os.listdir(startdir + "/" + x):
 973                         if y == "CVS" or y.startswith("."):
 974                                 continue
 975                         if os.path.isdir(startdir + "/" + x + "/" + y):
 976                                 scanlist.append(x + "/" + y)
 977         repo_subdir = ""
 978 elif repolevel == 3:
 979         catdir = reposplit[-2]
 980         if catdir not in categories:
 981                 caterror(catdir)
 982         scanlist.append(catdir + "/" + reposplit[-1])
 983         repo_subdir = scanlist[-1] + os.sep
 984 else:
 985         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 986                 ' from the current working directory'
 987         logging.critical(msg)
 988         sys.exit(1)
 989
 990 repo_subdir_len = len(repo_subdir)
 991 scanlist.sort()
 992
 993 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 994
 995 def vcs_files_to_cps(vcs_file_iter):
 996         """
 997         Iterate over the given modified file paths returned from the vcs,
 998         and return a frozenset containing category/pn strings for each
 999         modified package.
1000         """
1001
1002         modified_cps = []
1003
1004         if repolevel == 3:
1005                 if reposplit[-2] in categories and \
1006                         next(vcs_file_iter, None) is not None:
1007                         modified_cps.append("/".join(reposplit[-2:]))
1008
1009         elif repolevel == 2:
1010                 category = reposplit[-1]
1011                 if category in categories:
1012                         for filename in vcs_file_iter:
1013                                 f_split = filename.split(os.sep)
1014                                 # ['.', pn, ...]
1015                                 if len(f_split) > 2:
1016                                         modified_cps.append(category + "/" + f_split[1])
1017
1018         else:
1019                 # repolevel == 1
1020                 for filename in vcs_file_iter:
1021                         f_split = filename.split(os.sep)
1022                         # ['.', category, pn, ...]
1023                         if len(f_split) > 3 and f_split[1] in categories:
1024                                 modified_cps.append("/".join(f_split[1:3]))
1025
1026         return frozenset(modified_cps)
1027
1028 def git_supports_gpg_sign():
1029         status, cmd_output = \
1030                 repoman_getstatusoutput("git --version")
1031         cmd_output = cmd_output.split()
1032         if cmd_output:
1033                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1034                 if version is not None:
1035                         version = [int(x) for x in version.groups()]
1036                         if version[0] > 1 or \
1037                                 (version[0] == 1 and version[1] > 7) or \
1038                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1039                                 return True
1040         return False
1041
1042 def dev_keywords(profiles):
1043         """
1044         Create a set of KEYWORDS values that exist in 'dev'
1045         profiles. These are used
1046         to trigger a message notifying the user when they might
1047         want to add the --include-dev option.
1048         """
1049         type_arch_map = {}
1050         for arch, arch_profiles in profiles.items():
1051                 for prof in arch_profiles:
1052                         arch_set = type_arch_map.get(prof.status)
1053                         if arch_set is None:
1054                                 arch_set = set()
1055                                 type_arch_map[prof.status] = arch_set
1056                         arch_set.add(arch)
1057
1058         dev_keywords = type_arch_map.get('dev', set())
1059         dev_keywords.update(['~' + arch for arch in dev_keywords])
1060         return frozenset(dev_keywords)
1061
1062 dev_keywords = dev_keywords(profiles)
1063
1064 stats = {}
1065 fails = {}
1066
1067 for x in qacats:
1068         stats[x] = 0
1069         fails[x] = []
1070
1071 xmllint_capable = False
1072 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1073
1074 def fetch_metadata_dtd():
1075         """
1076         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1077         metadata_dtd_ctime_interval.
1078         @rtype: bool
1079         @return: True if successful, otherwise False
1080         """
1081
1082         must_fetch = True
1083         metadata_dtd_st = None
1084         current_time = int(time.time())
1085         try:
1086                 metadata_dtd_st = os.stat(metadata_dtd)
1087         except EnvironmentError as e:
1088                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1089                         raise
1090                 del e
1091         else:
1092                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1093                 if abs(current_time - metadata_dtd_st.st_ctime) \
1094                         < metadata_dtd_ctime_interval:
1095                         must_fetch = False
1096
1097         if must_fetch:
1098                 print()
1099                 print(green("***") + " the local copy of metadata.dtd " + \
1100                         "needs to be refetched, doing that now")
1101                 print()
1102                 parsed_url = urlparse(metadata_dtd_uri)
1103                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
1104                 fcmd = repoman_settings.get(setting)
1105                 if not fcmd:
1106                         fcmd = repoman_settings.get('FETCHCOMMAND')
1107                         if not fcmd:
1108                                 logging.error("FETCHCOMMAND is unset")
1109                                 return False
1110
1111                 destdir = repoman_settings["DISTDIR"]
1112                 fd, metadata_dtd_tmp = tempfile.mkstemp(
1113                         prefix='metadata.dtd.', dir=destdir)
1114                 os.close(fd)
1115
1116                 try:
1117                         if not portage.getbinpkg.file_get(metadata_dtd_uri,
1118                                 destdir, fcmd=fcmd,
1119                                 filename=os.path.basename(metadata_dtd_tmp)):
1120                                 logging.error("failed to fetch metadata.dtd from '%s'" %
1121                                         metadata_dtd_uri)
1122                                 return False
1123
1124                         try:
1125                                 portage.util.apply_secpass_permissions(metadata_dtd_tmp,
1126                                         gid=portage.data.portage_gid, mode=0o664, mask=0o2)
1127                         except portage.exception.PortageException:
1128                                 pass
1129
1130                         os.rename(metadata_dtd_tmp, metadata_dtd)
1131                 finally:
1132                         try:
1133                                 os.unlink(metadata_dtd_tmp)
1134                         except OSError:
1135                                 pass
1136
1137         return True
1138
1139 if options.mode == "manifest":
1140         pass
1141 elif not find_binary('xmllint'):
1142         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1143         if options.xml_parse or repolevel == 3:
1144                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1145                 sys.exit(1)
1146 else:
1147         if not fetch_metadata_dtd():
1148                 sys.exit(1)
1149         # this can be problematic if xmllint changes their output
1150         xmllint_capable = True
1151
1152 if options.mode == 'commit' and vcs:
1153         utilities.detect_vcs_conflicts(options, vcs)
1154
1155 if options.mode == "manifest":
1156         pass
1157 elif options.pretend:
1158         print(green("\nRepoMan does a once-over of the neighborhood..."))
1159 else:
1160         print(green("\nRepoMan scours the neighborhood..."))
1161
1162 new_ebuilds = set()
1163 modified_ebuilds = set()
1164 modified_changelogs = set()
1165 mychanged = []
1166 mynew = []
1167 myremoved = []
1168
1169 if vcs == "cvs":
1170         mycvstree = cvstree.getentries("./", recursive=1)
1171         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1172         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1173         if options.if_modified == "y":
1174                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1175
1176 elif vcs == "svn":
1177         with repoman_popen("svn status") as f:
1178                 svnstatus = f.readlines()
1179         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1180         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1181         if options.if_modified == "y":
1182                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1183
1184 elif vcs == "git":
1185         with repoman_popen("git diff-index --name-only "
1186                 "--relative --diff-filter=M HEAD") as f:
1187                 mychanged = f.readlines()
1188         mychanged = ["./" + elem[:-1] for elem in mychanged]
1189
1190         with repoman_popen("git diff-index --name-only "
1191                 "--relative --diff-filter=A HEAD") as f:
1192                 mynew = f.readlines()
1193         mynew = ["./" + elem[:-1] for elem in mynew]
1194         if options.if_modified == "y":
1195                 with repoman_popen("git diff-index --name-only "
1196                         "--relative --diff-filter=D HEAD") as f:
1197                         myremoved = f.readlines()
1198                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1199
1200 elif vcs == "bzr":
1201         with repoman_popen("bzr status -S .") as f:
1202                 bzrstatus = f.readlines()
1203         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1204         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1205         if options.if_modified == "y":
1206                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1207
1208 elif vcs == "hg":
1209         with repoman_popen("hg status --no-status --modified .") as f:
1210                 mychanged = f.readlines()
1211         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1212         with repoman_popen("hg status --no-status --added .") as f:
1213                 mynew = f.readlines()
1214         mynew = ["./" + elem.rstrip() for elem in mynew]
1215         if options.if_modified == "y":
1216                 with repoman_popen("hg status --no-status --removed .") as f:
1217                         myremoved = f.readlines()
1218                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1219
1220 if vcs:
1221         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1222         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1223         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1224                 if os.path.basename(x) == "ChangeLog")
1225
1226 def vcs_new_changed(relative_path):
1227         for x in chain(mychanged, mynew):
1228                 if x == relative_path:
1229                         return True
1230         return False
1231
1232 have_pmasked = False
1233 have_dev_keywords = False
1234 dofail = 0
1235
1236 # NOTE: match-all caches are not shared due to potential
1237 # differences between profiles in _get_implicit_iuse.
1238 arch_caches = {}
1239 arch_xmatch_caches = {}
1240 shared_xmatch_caches = {"cp-list":{}}
1241
1242 include_arches = None
1243 if options.include_arches:
1244         include_arches = set()
1245         include_arches.update(*[x.split() for x in options.include_arches])
1246
1247 # Disable the "ebuild.notadded" check when not in commit mode and
1248 # running `svn status` in every package dir will be too expensive.
1249
1250 check_ebuild_notadded = not \
1251         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1252
1253 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1254 thirdpartymirrors = {}
1255 for k, v in repoman_settings.thirdpartymirrors().items():
1256         for v in v:
1257                 if not v.endswith("/"):
1258                         v += "/"
1259                 thirdpartymirrors[v] = k
1260
1261 class _XMLParser(xml.etree.ElementTree.XMLParser):
1262
1263         def __init__(self, data, **kwargs):
1264                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1265                 self._portage_data = data
1266                 if hasattr(self, 'parser'):
1267                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1268                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1269                         self._base_StartDoctypeDeclHandler = \
1270                                 self.parser.StartDoctypeDeclHandler
1271                         self.parser.StartDoctypeDeclHandler = \
1272                                 self._portage_StartDoctypeDeclHandler
1273
1274         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1275                 if self._base_XmlDeclHandler is not None:
1276                         self._base_XmlDeclHandler(version, encoding, standalone)
1277                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1278
1279         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1280                 has_internal_subset):
1281                 if self._base_StartDoctypeDeclHandler is not None:
1282                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1283                                 has_internal_subset)
1284                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1285
1286 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1287         """
1288         Implements doctype() as required to avoid deprecation warnings with
1289         >=python-2.7.
1290         """
1291         def doctype(self, name, pubid, system):
1292                 pass
1293
1294 try:
1295         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1296 except (EnvironmentError, ParseError, PermissionDenied) as e:
1297         err(str(e))
1298 except FileNotFound:
1299         # TODO: Download as we do for metadata.dtd, but add a way to
1300         # disable for non-gentoo repoman users who may not have herds.
1301         herd_base = None
1302
1303 effective_scanlist = scanlist
1304 if options.if_modified == "y":
1305         effective_scanlist = sorted(vcs_files_to_cps(
1306                 chain(mychanged, mynew, myremoved)))
1307
1308 for x in effective_scanlist:
1309         #ebuilds and digests added to cvs respectively.
1310         logging.info("checking package %s" % x)
1311         # save memory by discarding xmatch caches from previous package(s)
1312         arch_xmatch_caches.clear()
1313         eadded = []
1314         catdir, pkgdir = x.split("/")
1315         checkdir = repodir + "/" + x
1316         checkdir_relative = ""
1317         if repolevel < 3:
1318                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1319         if repolevel < 2:
1320                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1321         checkdir_relative = os.path.join(".", checkdir_relative)
1322         generated_manifest = False
1323
1324         if options.mode == "manifest" or \
1325           (options.mode != 'manifest-check' and options.digest == 'y') or \
1326           options.mode in ('commit', 'fix') and not options.pretend:
1327                 auto_assumed = set()
1328                 fetchlist_dict = portage.FetchlistDict(checkdir,
1329                         repoman_settings, portdb)
1330                 if options.mode == 'manifest' and options.force:
1331                         portage._doebuild_manifest_exempt_depend += 1
1332                         try:
1333                                 distdir = repoman_settings['DISTDIR']
1334                                 mf = repoman_settings.repositories.get_repo_for_location(
1335                                         os.path.dirname(os.path.dirname(checkdir)))
1336                                 mf = mf.load_manifest(checkdir, distdir,
1337                                         fetchlist_dict=fetchlist_dict)
1338                                 mf.create(requiredDistfiles=None,
1339                                         assumeDistHashesAlways=True)
1340                                 for distfiles in fetchlist_dict.values():
1341                                         for distfile in distfiles:
1342                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1343                                                         mf.fhashdict['DIST'].pop(distfile, None)
1344                                                 else:
1345                                                         auto_assumed.add(distfile)
1346                                 mf.write()
1347                         finally:
1348                                 portage._doebuild_manifest_exempt_depend -= 1
1349
1350                 repoman_settings["O"] = checkdir
1351                 try:
1352                         generated_manifest = digestgen(
1353                                 mysettings=repoman_settings, myportdb=portdb)
1354                 except portage.exception.PermissionDenied as e:
1355                         generated_manifest = False
1356                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1357                                 level=logging.ERROR, noiselevel=-1)
1358
1359                 if not generated_manifest:
1360                         print("Unable to generate manifest.")
1361                         dofail = 1
1362
1363                 if options.mode == "manifest":
1364                         if not dofail and options.force and auto_assumed and \
1365                                 'assume-digests' in repoman_settings.features:
1366                                 # Show which digests were assumed despite the --force option
1367                                 # being given. This output will already have been shown by
1368                                 # digestgen() if assume-digests is not enabled, so only show
1369                                 # it here if assume-digests is enabled.
1370                                 pkgs = list(fetchlist_dict)
1371                                 pkgs.sort()
1372                                 portage.writemsg_stdout("  digest.assumed" + \
1373                                         portage.output.colorize("WARN",
1374                                         str(len(auto_assumed)).rjust(18)) + "\n")
1375                                 for cpv in pkgs:
1376                                         fetchmap = fetchlist_dict[cpv]
1377                                         pf = portage.catsplit(cpv)[1]
1378                                         for distfile in sorted(fetchmap):
1379                                                 if distfile in auto_assumed:
1380                                                         portage.writemsg_stdout(
1381                                                                 "   %s::%s\n" % (pf, distfile))
1382                         continue
1383                 elif dofail:
1384                         sys.exit(1)
1385
1386         if not generated_manifest:
1387                 repoman_settings['O'] = checkdir
1388                 repoman_settings['PORTAGE_QUIET'] = '1'
1389                 if not portage.digestcheck([], repoman_settings, strict=1):
1390                         stats["manifest.bad"] += 1
1391                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1392                 repoman_settings.pop('PORTAGE_QUIET', None)
1393
1394         if options.mode == 'manifest-check':
1395                 continue
1396
1397         checkdirlist = os.listdir(checkdir)
1398         ebuildlist = []
1399         pkgs = {}
1400         allvalid = True
1401         for y in checkdirlist:
1402                 if (y in no_exec or y.endswith(".ebuild")) and \
1403                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1404                                 stats["file.executable"] += 1
1405                                 fails["file.executable"].append(os.path.join(checkdir, y))
1406                 if y.endswith(".ebuild"):
1407                         pf = y[:-7]
1408                         ebuildlist.append(pf)
1409                         cpv = "%s/%s" % (catdir, pf)
1410                         try:
1411                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1412                         except KeyError:
1413                                 allvalid = False
1414                                 stats["ebuild.syntax"] += 1
1415                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1416                                 continue
1417                         except IOError:
1418                                 allvalid = False
1419                                 stats["ebuild.output"] += 1
1420                                 fails["ebuild.output"].append(os.path.join(x, y))
1421                                 continue
1422                         if not portage.eapi_is_supported(myaux["EAPI"]):
1423                                 allvalid = False
1424                                 stats["EAPI.unsupported"] += 1
1425                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1426                                 continue
1427                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1428                                 root_config=root_config, type_name="ebuild")
1429
1430         slot_keywords = {}
1431
1432         if len(pkgs) != len(ebuildlist):
1433                 # If we can't access all the metadata then it's totally unsafe to
1434                 # commit since there's no way to generate a correct Manifest.
1435                 # Do not try to do any more QA checks on this package since missing
1436                 # metadata leads to false positives for several checks, and false
1437                 # positives confuse users.
1438                 can_force = False
1439                 continue
1440
1441         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1442         ebuildlist = sorted(pkgs.values())
1443         ebuildlist = [pkg.pf for pkg in ebuildlist]
1444
1445         for y in checkdirlist:
1446                 index = repo_config.find_invalid_path_char(y)
1447                 if index != -1:
1448                         y_relative = os.path.join(checkdir_relative, y)
1449                         if vcs is not None and not vcs_new_changed(y_relative):
1450                                 # If the file isn't in the VCS new or changed set, then
1451                                 # assume that it's an irrelevant temporary file (Manifest
1452                                 # entries are not generated for file names containing
1453                                 # prohibited characters). See bug #406877.
1454                                 index = -1
1455                 if index != -1:
1456                         stats["file.name"] += 1
1457                         fails["file.name"].append("%s/%s: char '%s'" % \
1458                                 (checkdir, y, y[index]))
1459
1460                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1461                         continue
1462                 f = None
1463                 try:
1464                         line = 1
1465                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1466                                 encoding=_encodings['fs'], errors='strict'),
1467                                 mode='r', encoding=_encodings['repo.content'])
1468                         for l in f:
1469                                 line += 1
1470                 except UnicodeDecodeError as ue:
1471                         stats["file.UTF8"] += 1
1472                         s = ue.object[:ue.start]
1473                         l2 = s.count("\n")
1474                         line += l2
1475                         if l2 != 0:
1476                                 s = s[s.rfind("\n") + 1:]
1477                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1478                 finally:
1479                         if f is not None:
1480                                 f.close()
1481
1482         if vcs in ("git", "hg") and check_ebuild_notadded:
1483                 if vcs == "git":
1484                         myf = repoman_popen("git ls-files --others %s" % \
1485                                 (portage._shell_quote(checkdir_relative),))
1486                 if vcs == "hg":
1487                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1488                                 (portage._shell_quote(checkdir_relative),))
1489                 for l in myf:
1490                         if l[:-1][-7:] == ".ebuild":
1491                                 stats["ebuild.notadded"] += 1
1492                                 fails["ebuild.notadded"].append(
1493                                         os.path.join(x, os.path.basename(l[:-1])))
1494                 myf.close()
1495
1496         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1497                 try:
1498                         if vcs == "cvs":
1499                                 myf = open(checkdir + "/CVS/Entries", "r")
1500                         if vcs == "svn":
1501                                 myf = repoman_popen("svn status --depth=files --verbose " +
1502                                         portage._shell_quote(checkdir))
1503                         if vcs == "bzr":
1504                                 myf = repoman_popen("bzr ls -v --kind=file " +
1505                                         portage._shell_quote(checkdir))
1506                         myl = myf.readlines()
1507                         myf.close()
1508                         for l in myl:
1509                                 if vcs == "cvs":
1510                                         if l[0] != "/":
1511                                                 continue
1512                                         splitl = l[1:].split("/")
1513                                         if not len(splitl):
1514                                                 continue
1515                                         if splitl[0][-7:] == ".ebuild":
1516                                                 eadded.append(splitl[0][:-7])
1517                                 if vcs == "svn":
1518                                         if l[:1] == "?":
1519                                                 continue
1520                                         if l[:7] == '      >':
1521                                                 # tree conflict, new in subversion 1.6
1522                                                 continue
1523                                         l = l.split()[-1]
1524                                         if l[-7:] == ".ebuild":
1525                                                 eadded.append(os.path.basename(l[:-7]))
1526                                 if vcs == "bzr":
1527                                         if l[1:2] == "?":
1528                                                 continue
1529                                         l = l.split()[-1]
1530                                         if l[-7:] == ".ebuild":
1531                                                 eadded.append(os.path.basename(l[:-7]))
1532                         if vcs == "svn":
1533                                 myf = repoman_popen("svn status " +
1534                                         portage._shell_quote(checkdir))
1535                                 myl = myf.readlines()
1536                                 myf.close()
1537                                 for l in myl:
1538                                         if l[0] == "A":
1539                                                 l = l.rstrip().split(' ')[-1]
1540                                                 if l[-7:] == ".ebuild":
1541                                                         eadded.append(os.path.basename(l[:-7]))
1542                 except IOError:
1543                         if vcs == "cvs":
1544                                 stats["CVS/Entries.IO_error"] += 1
1545                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1546                         else:
1547                                 raise
1548                         continue
1549
1550         mf = repoman_settings.repositories.get_repo_for_location(
1551                 os.path.dirname(os.path.dirname(checkdir)))
1552         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1553         mydigests = mf.getTypeDigests("DIST")
1554
1555         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1556         myfiles_all = []
1557         src_uri_error = False
1558         for mykey in fetchlist_dict:
1559                 try:
1560                         myfiles_all.extend(fetchlist_dict[mykey])
1561                 except portage.exception.InvalidDependString as e:
1562                         src_uri_error = True
1563                         try:
1564                                 portdb.aux_get(mykey, ["SRC_URI"])
1565                         except KeyError:
1566                                 # This will be reported as an "ebuild.syntax" error.
1567                                 pass
1568                         else:
1569                                 stats["SRC_URI.syntax"] += 1
1570                                 fails["SRC_URI.syntax"].append(
1571                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1572         del fetchlist_dict
1573         if not src_uri_error:
1574                 # This test can produce false positives if SRC_URI could not
1575                 # be parsed for one or more ebuilds. There's no point in
1576                 # producing a false error here since the root cause will
1577                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1578                 # or "ebuild.sytax".
1579                 myfiles_all = set(myfiles_all)
1580                 for entry in mydigests:
1581                         if entry not in myfiles_all:
1582                                 stats["digest.unused"] += 1
1583                                 fails["digest.unused"].append(checkdir + "::" + entry)
1584                 for entry in myfiles_all:
1585                         if entry not in mydigests:
1586                                 stats["digest.missing"] += 1
1587                                 fails["digest.missing"].append(checkdir + "::" + entry)
1588         del myfiles_all
1589
1590         if os.path.exists(checkdir + "/files"):
1591                 filesdirlist = os.listdir(checkdir + "/files")
1592
1593                 # recurse through files directory
1594                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1595                 while filesdirlist:
1596                         y = filesdirlist.pop(0)
1597                         relative_path = os.path.join(x, "files", y)
1598                         full_path = os.path.join(repodir, relative_path)
1599                         try:
1600                                 mystat = os.stat(full_path)
1601                         except OSError as oe:
1602                                 if oe.errno == 2:
1603                                         # don't worry about it.  it likely was removed via fix above.
1604                                         continue
1605                                 else:
1606                                         raise oe
1607                         if S_ISDIR(mystat.st_mode):
1608                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1609                                 if y == "CVS" or y == ".svn":
1610                                         continue
1611                                 for z in os.listdir(checkdir + "/files/" + y):
1612                                         if z == "CVS" or z == ".svn":
1613                                                 continue
1614                                         filesdirlist.append(y + "/" + z)
1615                         # Current policy is no files over 20 KiB, these are the checks. File size between
1616                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1617                         elif mystat.st_size > 61440:
1618                                 stats["file.size.fatal"] += 1
1619                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1620                         elif mystat.st_size > 20480:
1621                                 stats["file.size"] += 1
1622                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1623
1624                         index = repo_config.find_invalid_path_char(y)
1625                         if index != -1:
1626                                 y_relative = os.path.join(checkdir_relative, "files", y)
1627                                 if vcs is not None and not vcs_new_changed(y_relative):
1628                                         # If the file isn't in the VCS new or changed set, then
1629                                         # assume that it's an irrelevant temporary file (Manifest
1630                                         # entries are not generated for file names containing
1631                                         # prohibited characters). See bug #406877.
1632                                         index = -1
1633                         if index != -1:
1634                                 stats["file.name"] += 1
1635                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1636                                         (checkdir, y, y[index]))
1637         del mydigests
1638
1639         if check_changelog and "ChangeLog" not in checkdirlist:
1640                 stats["changelog.missing"] += 1
1641                 fails["changelog.missing"].append(x + "/ChangeLog")
1642
1643         musedict = {}
1644         # metadata.xml file check
1645         if "metadata.xml" not in checkdirlist:
1646                 stats["metadata.missing"] += 1
1647                 fails["metadata.missing"].append(x + "/metadata.xml")
1648         # metadata.xml parse check
1649         else:
1650                 metadata_bad = False
1651                 xml_info = {}
1652                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1653
1654                 # read metadata.xml into memory
1655                 try:
1656                         _metadata_xml = xml.etree.ElementTree.parse(
1657                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1658                                 encoding=_encodings['fs'], errors='strict'),
1659                                 parser=xml_parser)
1660                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1661                         metadata_bad = True
1662                         stats["metadata.bad"] += 1
1663                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1664                         del e
1665                 else:
1666                         if not hasattr(xml_parser, 'parser') or \
1667                                 sys.hexversion < 0x2070000 or \
1668                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1669                                 # doctype is not parsed with python 2.6 or 3.1
1670                                 pass
1671                         else:
1672                                 if "XML_DECLARATION" not in xml_info:
1673                                         stats["metadata.bad"] += 1
1674                                         fails["metadata.bad"].append("%s/metadata.xml: "
1675                                                 "xml declaration is missing on first line, "
1676                                                 "should be '%s'" % (x, metadata_xml_declaration))
1677                                 else:
1678                                         xml_version, xml_encoding, xml_standalone = \
1679                                                 xml_info["XML_DECLARATION"]
1680                                         if xml_encoding is None or \
1681                                                 xml_encoding.upper() != metadata_xml_encoding:
1682                                                 stats["metadata.bad"] += 1
1683                                                 if xml_encoding is None:
1684                                                         encoding_problem = "but it is undefined"
1685                                                 else:
1686                                                         encoding_problem = "not '%s'" % xml_encoding
1687                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1688                                                         "xml declaration encoding should be '%s', %s" %
1689                                                         (x, metadata_xml_encoding, encoding_problem))
1690
1691                                 if "DOCTYPE" not in xml_info:
1692                                         metadata_bad = True
1693                                         stats["metadata.bad"] += 1
1694                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1695                                                 "DOCTYPE is missing"))
1696                                 else:
1697                                         doctype_name, doctype_system, doctype_pubid = \
1698                                                 xml_info["DOCTYPE"]
1699                                         if doctype_system != metadata_dtd_uri:
1700                                                 stats["metadata.bad"] += 1
1701                                                 if doctype_system is None:
1702                                                         system_problem = "but it is undefined"
1703                                                 else:
1704                                                         system_problem = "not '%s'" % doctype_system
1705                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1706                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1707                                                         (x, metadata_dtd_uri, system_problem))
1708
1709                                         if doctype_name != metadata_doctype_name:
1710                                                 stats["metadata.bad"] += 1
1711                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1712                                                         "DOCTYPE: name should be '%s', not '%s'" %
1713                                                         (x, metadata_doctype_name, doctype_name))
1714
1715                         # load USE flags from metadata.xml
1716                         try:
1717                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1718                         except portage.exception.ParseError as e:
1719                                 metadata_bad = True
1720                                 stats["metadata.bad"] += 1
1721                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1722                         else:
1723                                 for atom in chain(*musedict.values()):
1724                                         if atom is None:
1725                                                 continue
1726                                         try:
1727                                                 atom = Atom(atom)
1728                                         except InvalidAtom as e:
1729                                                 stats["metadata.bad"] += 1
1730                                                 fails["metadata.bad"].append(
1731                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1732                                         else:
1733                                                 if atom.cp != x:
1734                                                         stats["metadata.bad"] += 1
1735                                                         fails["metadata.bad"].append(
1736                                                                 ("%s/metadata.xml: Atom contains "
1737                                                                 "unexpected cat/pn: %s") % (x, atom))
1738
1739                         # Run other metadata.xml checkers
1740                         try:
1741                                 utilities.check_metadata(_metadata_xml, herd_base)
1742                         except (utilities.UnknownHerdsError, ) as e:
1743                                 metadata_bad = True
1744                                 stats["metadata.bad"] += 1
1745                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1746                                 del e
1747
1748                 #Only carry out if in package directory or check forced
1749                 if xmllint_capable and not metadata_bad:
1750                         # xmlint can produce garbage output even on success, so only dump
1751                         # the ouput when it fails.
1752                         st, out = repoman_getstatusoutput(
1753                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1754                                  (portage._shell_quote(metadata_dtd),
1755                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1756                         if st != os.EX_OK:
1757                                 print(red("!!!") + " metadata.xml is invalid:")
1758                                 for z in out.splitlines():
1759                                         print(red("!!! ") + z)
1760                                 stats["metadata.bad"] += 1
1761                                 fails["metadata.bad"].append(x + "/metadata.xml")
1762
1763                 del metadata_bad
1764         muselist = frozenset(musedict)
1765
1766         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1767         changelog_modified = changelog_path in modified_changelogs
1768
1769         # detect unused local USE-descriptions
1770         used_useflags = set()
1771
1772         for y in ebuildlist:
1773                 relative_path = os.path.join(x, y + ".ebuild")
1774                 full_path = os.path.join(repodir, relative_path)
1775                 ebuild_path = y + ".ebuild"
1776                 if repolevel < 3:
1777                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1778                 if repolevel < 2:
1779                         ebuild_path = os.path.join(catdir, ebuild_path)
1780                 ebuild_path = os.path.join(".", ebuild_path)
1781                 if check_changelog and not changelog_modified \
1782                         and ebuild_path in new_ebuilds:
1783                         stats['changelog.ebuildadded'] += 1
1784                         fails['changelog.ebuildadded'].append(relative_path)
1785
1786                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1787                         #ebuild not added to vcs
1788                         stats["ebuild.notadded"] += 1
1789                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1790                 myesplit = portage.pkgsplit(y)
1791                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1792                             or pv_toolong_re.search(myesplit[1]) \
1793                             or pv_toolong_re.search(myesplit[2]):
1794                         stats["ebuild.invalidname"] += 1
1795                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1796                         continue
1797                 elif myesplit[0] != pkgdir:
1798                         print(pkgdir, myesplit[0])
1799                         stats["ebuild.namenomatch"] += 1
1800                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1801                         continue
1802
1803                 pkg = pkgs[y]
1804
1805                 if pkg.invalid:
1806                         allvalid = False
1807                         for k, msgs in pkg.invalid.items():
1808                                 for msg in msgs:
1809                                         stats[k] += 1
1810                                         fails[k].append("%s: %s" % (relative_path, msg))
1811                         continue
1812
1813                 myaux = pkg._metadata
1814                 eapi = myaux["EAPI"]
1815                 inherited = pkg.inherited
1816                 live_ebuild = live_eclasses.intersection(inherited)
1817
1818                 for k, v in myaux.items():
1819                         if not isinstance(v, basestring):
1820                                 continue
1821                         m = non_ascii_re.search(v)
1822                         if m is not None:
1823                                 stats["variable.invalidchar"] += 1
1824                                 fails["variable.invalidchar"].append(
1825                                         ("%s: %s variable contains non-ASCII " + \
1826                                         "character at position %s") % \
1827                                         (relative_path, k, m.start() + 1))
1828
1829                 if not src_uri_error:
1830                         # Check that URIs don't reference a server from thirdpartymirrors.
1831                         for uri in portage.dep.use_reduce( \
1832                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1833                                 contains_mirror = False
1834                                 for mirror, mirror_alias in thirdpartymirrors.items():
1835                                         if uri.startswith(mirror):
1836                                                 contains_mirror = True
1837                                                 break
1838                                 if not contains_mirror:
1839                                         continue
1840
1841                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1842                                 stats["SRC_URI.mirror"] += 1
1843                                 fails["SRC_URI.mirror"].append(
1844                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1845                                         (relative_path, mirror, new_uri))
1846
1847                 if myaux.get("PROVIDE"):
1848                         stats["virtual.oldstyle"] += 1
1849                         fails["virtual.oldstyle"].append(relative_path)
1850
1851                 for pos, missing_var in enumerate(missingvars):
1852                         if not myaux.get(missing_var):
1853                                 if catdir == "virtual" and \
1854                                         missing_var in ("HOMEPAGE", "LICENSE"):
1855                                         continue
1856                                 if live_ebuild and missing_var == "KEYWORDS":
1857                                         continue
1858                                 myqakey = missingvars[pos] + ".missing"
1859                                 stats[myqakey] += 1
1860                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1861
1862                 if catdir == "virtual":
1863                         for var in ("HOMEPAGE", "LICENSE"):
1864                                 if myaux.get(var):
1865                                         myqakey = var + ".virtual"
1866                                         stats[myqakey] += 1
1867                                         fails[myqakey].append(relative_path)
1868
1869                 # 14 is the length of DESCRIPTION=""
1870                 if len(myaux['DESCRIPTION']) > max_desc_len:
1871                         stats['DESCRIPTION.toolong'] += 1
1872                         fails['DESCRIPTION.toolong'].append(
1873                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1874                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1875
1876                 keywords = myaux["KEYWORDS"].split()
1877                 stable_keywords = []
1878                 for keyword in keywords:
1879                         if not keyword.startswith("~") and \
1880                                 not keyword.startswith("-"):
1881                                 stable_keywords.append(keyword)
1882                 if stable_keywords:
1883                         if ebuild_path in new_ebuilds and catdir != "virtual":
1884                                 stable_keywords.sort()
1885                                 stats["KEYWORDS.stable"] += 1
1886                                 fails["KEYWORDS.stable"].append(
1887                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1888                                                 " ".join(stable_keywords))
1889
1890                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1891                         if not kw.startswith("-"))
1892
1893                 previous_keywords = slot_keywords.get(pkg.slot)
1894                 if previous_keywords is None:
1895                         slot_keywords[pkg.slot] = set()
1896                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1897                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1898                         if dropped_keywords:
1899                                 stats["KEYWORDS.dropped"] += 1
1900                                 fails["KEYWORDS.dropped"].append(
1901                                         relative_path + ": %s" % \
1902                                         " ".join(sorted(dropped_keywords)))
1903
1904                 slot_keywords[pkg.slot].update(ebuild_archs)
1905
1906                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1907                 if "-*" in keywords:
1908                         haskeyword = False
1909                         for kw in keywords:
1910                                 if kw[0] == "~":
1911                                         kw = kw[1:]
1912                                 if kw in kwlist:
1913                                         haskeyword = True
1914                         if not haskeyword:
1915                                 stats["KEYWORDS.stupid"] += 1
1916                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1917
1918                 """
1919                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1920                 not be allowed to be marked stable
1921                 """
1922                 if live_ebuild and repo_config.name == "gentoo":
1923                         bad_stable_keywords = []
1924                         for keyword in keywords:
1925                                 if not keyword.startswith("~") and \
1926                                         not keyword.startswith("-"):
1927                                         bad_stable_keywords.append(keyword)
1928                                 del keyword
1929                         if bad_stable_keywords:
1930                                 stats["LIVEVCS.stable"] += 1
1931                                 fails["LIVEVCS.stable"].append(
1932                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1933                                                 bad_stable_keywords)
1934                         del bad_stable_keywords
1935
1936                         if keywords and not has_global_mask(pkg):
1937                                 stats["LIVEVCS.unmasked"] += 1
1938                                 fails["LIVEVCS.unmasked"].append(relative_path)
1939
1940                 if options.ignore_arches:
1941                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1942                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1943                 else:
1944                         arches = set()
1945                         for keyword in keywords:
1946                                 if keyword[0] == "-":
1947                                         continue
1948                                 elif keyword[0] == "~":
1949                                         arch = keyword[1:]
1950                                         if arch == "*":
1951                                                 for expanded_arch in profiles:
1952                                                         if expanded_arch == "**":
1953                                                                 continue
1954                                                         arches.add((keyword, expanded_arch,
1955                                                                 (expanded_arch, "~" + expanded_arch)))
1956                                         else:
1957                                                 arches.add((keyword, arch, (arch, keyword)))
1958                                 else:
1959                                         if keyword == "*":
1960                                                 for expanded_arch in profiles:
1961                                                         if expanded_arch == "**":
1962                                                                 continue
1963                                                         arches.add((keyword, expanded_arch,
1964                                                                 (expanded_arch,)))
1965                                         else:
1966                                                 arches.add((keyword, keyword, (keyword,)))
1967                         if not arches:
1968                                 # Use an empty profile for checking dependencies of
1969                                 # packages that have empty KEYWORDS.
1970                                 arches.add(('**', '**', ('**',)))
1971
1972                 unknown_pkgs = set()
1973                 baddepsyntax = False
1974                 badlicsyntax = False
1975                 badprovsyntax = False
1976                 catpkg = catdir + "/" + y
1977
1978                 inherited_java_eclass = "java-pkg-2" in inherited or \
1979                         "java-pkg-opt-2" in inherited
1980                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1981                 operator_tokens = set(["||", "(", ")"])
1982                 type_list, badsyntax = [], []
1983                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
1984                         mydepstr = myaux[mytype]
1985
1986                         buildtime = mytype in Package._buildtime_keys
1987                         runtime = mytype in Package._runtime_keys
1988                         token_class = None
1989                         if mytype.endswith("DEPEND"):
1990                                 token_class = portage.dep.Atom
1991
1992                         try:
1993                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1994                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1995                         except portage.exception.InvalidDependString as e:
1996                                 atoms = None
1997                                 badsyntax.append(str(e))
1998
1999                         if atoms and mytype.endswith("DEPEND"):
2000                                 if runtime and \
2001                                         "test?" in mydepstr.split():
2002                                         stats[mytype + '.suspect'] += 1
2003                                         fails[mytype + '.suspect'].append(relative_path + \
2004                                                 ": 'test?' USE conditional in %s" % mytype)
2005
2006                                 for atom in atoms:
2007                                         if atom == "||":
2008                                                 continue
2009
2010                                         # Skip dependency.unknown for blockers, so that we
2011                                         # don't encourage people to remove necessary blockers,
2012                                         # as discussed in bug #382407.
2013                                         if atom.blocker is None and \
2014                                                 not portdb.xmatch("match-all", atom) and \
2015                                                 not atom.cp.startswith("virtual/"):
2016                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2017
2018                                         is_blocker = atom.blocker
2019
2020                                         if catdir != "virtual":
2021                                                 if not is_blocker and \
2022                                                         atom.cp in suspect_virtual:
2023                                                         stats['virtual.suspect'] += 1
2024                                                         fails['virtual.suspect'].append(
2025                                                                 relative_path +
2026                                                                 ": %s: consider using '%s' instead of '%s'" %
2027                                                                 (mytype, suspect_virtual[atom.cp], atom))
2028
2029                                         if buildtime and \
2030                                                 not is_blocker and \
2031                                                 not inherited_java_eclass and \
2032                                                 atom.cp == "virtual/jdk":
2033                                                 stats['java.eclassesnotused'] += 1
2034                                                 fails['java.eclassesnotused'].append(relative_path)
2035                                         elif buildtime and \
2036                                                 not is_blocker and \
2037                                                 not inherited_wxwidgets_eclass and \
2038                                                 atom.cp == "x11-libs/wxGTK":
2039                                                 stats['wxwidgets.eclassnotused'] += 1
2040                                                 fails['wxwidgets.eclassnotused'].append(
2041                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2042                                                         " without inheriting wxwidgets.eclass") % mytype)
2043                                         elif runtime:
2044                                                 if not is_blocker and \
2045                                                         atom.cp in suspect_rdepend:
2046                                                         stats[mytype + '.suspect'] += 1
2047                                                         fails[mytype + '.suspect'].append(
2048                                                                 relative_path + ": '%s'" % atom)
2049
2050                                         if atom.operator == "~" and \
2051                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2052                                                 qacat = 'dependency.badtilde'
2053                                                 stats[qacat] += 1
2054                                                 fails[qacat].append(
2055                                                         (relative_path + ": %s uses the ~ operator"
2056                                                          " with a non-zero revision:" + \
2057                                                          " '%s'") % (mytype, atom))
2058
2059                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2060
2061                 for m, b in zip(type_list, badsyntax):
2062                         if m.endswith("DEPEND"):
2063                                 qacat = "dependency.syntax"
2064                         else:
2065                                 qacat = m + ".syntax"
2066                         stats[qacat] += 1
2067                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2068
2069                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2070                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2071                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2072                 badlicsyntax = badlicsyntax > 0
2073                 badprovsyntax = badprovsyntax > 0
2074
2075                 # uselist checks - global
2076                 myuse = []
2077                 default_use = []
2078                 for myflag in myaux["IUSE"].split():
2079                         flag_name = myflag.lstrip("+-")
2080                         used_useflags.add(flag_name)
2081                         if myflag != flag_name:
2082                                 default_use.append(myflag)
2083                         if flag_name not in uselist:
2084                                 myuse.append(flag_name)
2085
2086                 # uselist checks - metadata
2087                 for mypos in range(len(myuse)-1, -1, -1):
2088                         if myuse[mypos] and (myuse[mypos] in muselist):
2089                                 del myuse[mypos]
2090
2091                 if default_use and not eapi_has_iuse_defaults(eapi):
2092                         for myflag in default_use:
2093                                 stats['EAPI.incompatible'] += 1
2094                                 fails['EAPI.incompatible'].append(
2095                                         (relative_path + ": IUSE defaults" + \
2096                                         " not supported with EAPI='%s':" + \
2097                                         " '%s'") % (eapi, myflag))
2098
2099                 for mypos in range(len(myuse)):
2100                         stats["IUSE.invalid"] += 1
2101                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2102
2103                 # license checks
2104                 if not badlicsyntax:
2105                         # Parse the LICENSE variable, remove USE conditions and
2106                         # flatten it.
2107                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2108                         # Check each entry to ensure that it exists in PORTDIR's
2109                         # license directory.
2110                         for lic in licenses:
2111                                 # Need to check for "||" manually as no portage
2112                                 # function will remove it without removing values.
2113                                 if lic not in liclist and lic != "||":
2114                                         stats["LICENSE.invalid"] += 1
2115                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2116                                 elif lic in liclist_deprecated:
2117                                         stats["LICENSE.deprecated"] += 1
2118                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2119
2120                 #keyword checks
2121                 myuse = myaux["KEYWORDS"].split()
2122                 for mykey in myuse:
2123                         if mykey not in ("-*", "*", "~*"):
2124                                 myskey = mykey
2125                                 if myskey[:1] == "-":
2126                                         myskey = myskey[1:]
2127                                 if myskey[:1] == "~":
2128                                         myskey = myskey[1:]
2129                                 if myskey not in kwlist:
2130                                         stats["KEYWORDS.invalid"] += 1
2131                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2132                                 elif myskey not in profiles:
2133                                         stats["KEYWORDS.invalid"] += 1
2134                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2135
2136                 #restrict checks
2137                 myrestrict = None
2138                 try:
2139                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2140                 except portage.exception.InvalidDependString as e:
2141                         stats["RESTRICT.syntax"] += 1
2142                         fails["RESTRICT.syntax"].append(
2143                                 "%s: RESTRICT: %s" % (relative_path, e))
2144                         del e
2145                 if myrestrict:
2146                         myrestrict = set(myrestrict)
2147                         mybadrestrict = myrestrict.difference(valid_restrict)
2148                         if mybadrestrict:
2149                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2150                                 for mybad in mybadrestrict:
2151                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2152                 #REQUIRED_USE check
2153                 required_use = myaux["REQUIRED_USE"]
2154                 if required_use:
2155                         if not eapi_has_required_use(eapi):
2156                                 stats['EAPI.incompatible'] += 1
2157                                 fails['EAPI.incompatible'].append(
2158                                         relative_path + ": REQUIRED_USE" + \
2159                                         " not supported with EAPI='%s'" % (eapi,))
2160                         try:
2161                                 portage.dep.check_required_use(required_use, (),
2162                                         pkg.iuse.is_valid_flag, eapi=eapi)
2163                         except portage.exception.InvalidDependString as e:
2164                                 stats["REQUIRED_USE.syntax"] += 1
2165                                 fails["REQUIRED_USE.syntax"].append(
2166                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2167                                 del e
2168
2169                 # Syntax Checks
2170                 relative_path = os.path.join(x, y + ".ebuild")
2171                 full_path = os.path.join(repodir, relative_path)
2172                 if not vcs_preserves_mtime:
2173                         if ebuild_path not in new_ebuilds and \
2174                                 ebuild_path not in modified_ebuilds:
2175                                 pkg.mtime = None
2176                 try:
2177                         # All ebuilds should have utf_8 encoding.
2178                         f = io.open(_unicode_encode(full_path,
2179                                 encoding=_encodings['fs'], errors='strict'),
2180                                 mode='r', encoding=_encodings['repo.content'])
2181                         try:
2182                                 for check_name, e in run_checks(f, pkg):
2183                                         stats[check_name] += 1
2184                                         fails[check_name].append(relative_path + ': %s' % e)
2185                         finally:
2186                                 f.close()
2187                 except UnicodeDecodeError:
2188                         # A file.UTF8 failure will have already been recorded above.
2189                         pass
2190
2191                 if options.force:
2192                         # The dep_check() calls are the most expensive QA test. If --force
2193                         # is enabled, there's no point in wasting time on these since the
2194                         # user is intent on forcing the commit anyway.
2195                         continue
2196
2197                 relevant_profiles = []
2198                 for keyword, arch, groups in arches:
2199                         if arch not in profiles:
2200                                 # A missing profile will create an error further down
2201                                 # during the KEYWORDS verification.
2202                                 continue
2203
2204                         if include_arches is not None:
2205                                 if arch not in include_arches:
2206                                         continue
2207
2208                         relevant_profiles.extend((keyword, groups, prof)
2209                                 for prof in profiles[arch])
2210
2211                 def sort_key(item):
2212                         return item[2].sub_path
2213
2214                 relevant_profiles.sort(key=sort_key)
2215
2216                 for keyword, groups, prof in relevant_profiles:
2217
2218                                 if prof.status not in ("stable", "dev") or \
2219                                         prof.status == "dev" and not options.include_dev:
2220                                         continue
2221
2222                                 dep_settings = arch_caches.get(prof.sub_path)
2223                                 if dep_settings is None:
2224                                         dep_settings = portage.config(
2225                                                 config_profile_path=prof.abs_path,
2226                                                 config_incrementals=repoman_incrementals,
2227                                                 config_root=config_root,
2228                                                 local_config=False,
2229                                                 _unmatched_removal=options.unmatched_removal,
2230                                                 env=env)
2231                                         dep_settings.categories = repoman_settings.categories
2232                                         if options.without_mask:
2233                                                 dep_settings._mask_manager_obj = \
2234                                                         copy.deepcopy(dep_settings._mask_manager)
2235                                                 dep_settings._mask_manager._pmaskdict.clear()
2236                                         arch_caches[prof.sub_path] = dep_settings
2237
2238                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2239                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2240                                 if xcache is None:
2241                                         portdb.melt()
2242                                         portdb.freeze()
2243                                         xcache = portdb.xcache
2244                                         xcache.update(shared_xmatch_caches)
2245                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2246
2247                                 trees[root]["porttree"].settings = dep_settings
2248                                 portdb.settings = dep_settings
2249                                 portdb.xcache = xcache
2250
2251                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2252                                 # just in case, prevent config.reset() from nuking these.
2253                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2254
2255                                 # This attribute is used in dbapi._match_use() to apply
2256                                 # use.stable.{mask,force} settings based on the stable
2257                                 # status of the parent package. This is required in order
2258                                 # for USE deps of unstable packages to be resolved correctly,
2259                                 # since otherwise use.stable.{mask,force} settings of
2260                                 # dependencies may conflict (see bug #456342).
2261                                 dep_settings._parent_stable = dep_settings._isStable(pkg)
2262
2263                                 # Handle package.use*.{force,mask) calculation, for use
2264                                 # in dep_check.
2265                                 dep_settings.useforce = dep_settings._use_manager.getUseForce(
2266                                         pkg, stable=dep_settings._parent_stable)
2267                                 dep_settings.usemask = dep_settings._use_manager.getUseMask(
2268                                         pkg, stable=dep_settings._parent_stable)
2269
2270                                 if not baddepsyntax:
2271                                         ismasked = not ebuild_archs or \
2272                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2273                                         if ismasked:
2274                                                 if not have_pmasked:
2275                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2276                                                                 pkg.cpv, pkg._metadata))
2277                                                 if options.ignore_masked:
2278                                                         continue
2279                                                 #we are testing deps for a masked package; give it some lee-way
2280                                                 suffix = "masked"
2281                                                 matchmode = "minimum-all"
2282                                         else:
2283                                                 suffix = ""
2284                                                 matchmode = "minimum-visible"
2285
2286                                         if not have_dev_keywords:
2287                                                 have_dev_keywords = \
2288                                                         bool(dev_keywords.intersection(keywords))
2289
2290                                         if prof.status == "dev":
2291                                                 suffix = suffix + "indev"
2292
2293                                         for mytype in Package._dep_keys:
2294
2295                                                 mykey = "dependency.bad" + suffix
2296                                                 myvalue = myaux[mytype]
2297                                                 if not myvalue:
2298                                                         continue
2299
2300                                                 success, atoms = portage.dep_check(myvalue, portdb,
2301                                                         dep_settings, use="all", mode=matchmode,
2302                                                         trees=trees)
2303
2304                                                 if success:
2305                                                         if atoms:
2306
2307                                                                 # Don't bother with dependency.unknown for
2308                                                                 # cases in which *DEPEND.bad is triggered.
2309                                                                 for atom in atoms:
2310                                                                         # dep_check returns all blockers and they
2311                                                                         # aren't counted for *DEPEND.bad, so we
2312                                                                         # ignore them here.
2313                                                                         if not atom.blocker:
2314                                                                                 unknown_pkgs.discard(
2315                                                                                         (mytype, atom.unevaluated_atom))
2316
2317                                                                 if not prof.sub_path:
2318                                                                         # old-style virtuals currently aren't
2319                                                                         # resolvable with empty profile, since
2320                                                                         # 'virtuals' mappings are unavailable
2321                                                                         # (it would be expensive to search
2322                                                                         # for PROVIDE in all ebuilds)
2323                                                                         atoms = [atom for atom in atoms if not \
2324                                                                                 (atom.cp.startswith('virtual/') and \
2325                                                                                 not portdb.cp_list(atom.cp))]
2326
2327                                                                 #we have some unsolvable deps
2328                                                                 #remove ! deps, which always show up as unsatisfiable
2329                                                                 atoms = [str(atom.unevaluated_atom) \
2330                                                                         for atom in atoms if not atom.blocker]
2331
2332                                                                 #if we emptied out our list, continue:
2333                                                                 if not atoms:
2334                                                                         continue
2335                                                                 stats[mykey] += 1
2336                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2337                                                                         (relative_path, mytype, keyword,
2338                                                                         prof, repr(atoms)))
2339                                                 else:
2340                                                         stats[mykey] += 1
2341                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2342                                                                 (relative_path, mytype, keyword,
2343                                                                 prof, repr(atoms)))
2344
2345                 if not baddepsyntax and unknown_pkgs:
2346                         type_map = {}
2347                         for mytype, atom in unknown_pkgs:
2348                                 type_map.setdefault(mytype, set()).add(atom)
2349                         for mytype, atoms in type_map.items():
2350                                 stats["dependency.unknown"] += 1
2351                                 fails["dependency.unknown"].append("%s: %s: %s" %
2352                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2353
2354         # check if there are unused local USE-descriptions in metadata.xml
2355         # (unless there are any invalids, to avoid noise)
2356         if allvalid:
2357                 for myflag in muselist.difference(used_useflags):
2358                         stats["metadata.warning"] += 1
2359                         fails["metadata.warning"].append(
2360                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2361                                 (x, myflag))
2362
2363 if options.if_modified == "y" and len(effective_scanlist) < 1:
2364         logging.warn("--if-modified is enabled, but no modified packages were found!")
2365
2366 if options.mode == "manifest":
2367         sys.exit(dofail)
2368
2369 # dofail will be set to 1 if we have failed in at least one non-warning category
2370 dofail = 0
2371 # dowarn will be set to 1 if we tripped any warnings
2372 dowarn = 0
2373 # dofull will be set if we should print a "repoman full" informational message
2374 dofull = options.mode != 'full'
2375
2376 for x in qacats:
2377         if not stats[x]:
2378                 continue
2379         dowarn = 1
2380         if x not in qawarnings:
2381                 dofail = 1
2382
2383 if dofail or \
2384         (dowarn and not (options.quiet or options.mode == "scan")):
2385         dofull = 0
2386
2387 # Save QA output so that it can be conveniently displayed
2388 # in $EDITOR while the user creates a commit message.
2389 # Otherwise, the user would not be able to see this output
2390 # once the editor has taken over the screen.
2391 qa_output = io.StringIO()
2392 style_file = ConsoleStyleFile(sys.stdout)
2393 if options.mode == 'commit' and \
2394         (not commitmessage or not commitmessage.strip()):
2395         style_file.write_listener = qa_output
2396 console_writer = StyleWriter(file=style_file, maxcol=9999)
2397 console_writer.style_listener = style_file.new_styles
2398
2399 f = formatter.AbstractFormatter(console_writer)
2400
2401 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2402
2403 style_file.flush()
2404 del console_writer, f, style_file
2405 qa_output = qa_output.getvalue()
2406 qa_output = qa_output.splitlines(True)
2407
2408 suggest_ignore_masked = False
2409 suggest_include_dev = False
2410
2411 if have_pmasked and not (options.without_mask or options.ignore_masked):
2412         suggest_ignore_masked = True
2413 if have_dev_keywords and not options.include_dev:
2414         suggest_include_dev = True
2415
2416 if suggest_ignore_masked or suggest_include_dev:
2417         print()
2418         if suggest_ignore_masked:
2419                 print(bold("Note: use --without-mask to check " + \
2420                         "KEYWORDS on dependencies of masked packages"))
2421
2422         if suggest_include_dev:
2423                 print(bold("Note: use --include-dev (-d) to check " + \
2424                         "dependencies for 'dev' profiles"))
2425         print()
2426
2427 if options.mode != 'commit':
2428         if dofull:
2429                 print(bold("Note: type \"repoman full\" for a complete listing."))
2430         if dowarn and not dofail:
2431                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2432         elif not dofail:
2433                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2434         elif dofail:
2435                 print(bad("Please fix these important QA issues first."))
2436                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2437                 sys.exit(1)
2438 else:
2439         if dofail and can_force and options.force and not options.pretend:
2440                 print(green("RepoMan sez:") + \
2441                         " \"You want to commit even with these QA issues?\n" + \
2442                         "              I'll take it this time, but I'm not happy.\"\n")
2443         elif dofail:
2444                 if options.force and not can_force:
2445                         print(bad("The --force option has been disabled due to extraordinary issues."))
2446                 print(bad("Please fix these important QA issues first."))
2447                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2448                 sys.exit(1)
2449
2450         if options.pretend:
2451                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2452
2453         myunadded = []
2454         if vcs == "cvs":
2455                 try:
2456                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2457                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2458                 except SystemExit as e:
2459                         raise  # TODO propagate this
2460                 except:
2461                         err("Error retrieving CVS tree; exiting.")
2462         if vcs == "svn":
2463                 try:
2464                         with repoman_popen("svn status --no-ignore") as f:
2465                                 svnstatus = f.readlines()
2466                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2467                 except SystemExit as e:
2468                         raise  # TODO propagate this
2469                 except:
2470                         err("Error retrieving SVN info; exiting.")
2471         if vcs == "git":
2472                 # get list of files not under version control or missing
2473                 myf = repoman_popen("git ls-files --others")
2474                 myunadded = ["./" + elem[:-1] for elem in myf]
2475                 myf.close()
2476         if vcs == "bzr":
2477                 try:
2478                         with repoman_popen("bzr status -S .") as f:
2479                                 bzrstatus = f.readlines()
2480                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2481                 except SystemExit as e:
2482                         raise  # TODO propagate this
2483                 except:
2484                         err("Error retrieving bzr info; exiting.")
2485         if vcs == "hg":
2486                 with repoman_popen("hg status --no-status --unknown .") as f:
2487                         myunadded = f.readlines()
2488                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2489
2490                 # Mercurial doesn't handle manually deleted files as removed from
2491                 # the repository, so the user need to remove them before commit,
2492                 # using "hg remove [FILES]"
2493                 with repoman_popen("hg status --no-status --deleted .") as f:
2494                         mydeleted = f.readlines()
2495                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2496
2497
2498         myautoadd = []
2499         if myunadded:
2500                 for x in range(len(myunadded)-1, -1, -1):
2501                         xs = myunadded[x].split("/")
2502                         if xs[-1] == "files":
2503                                 print("!!! files dir is not added! Please correct this.")
2504                                 sys.exit(-1)
2505                         elif xs[-1] == "Manifest":
2506                                 # It's a manifest... auto add
2507                                 myautoadd += [myunadded[x]]
2508                                 del myunadded[x]
2509
2510         if myunadded:
2511                 print(red("!!! The following files are in your local tree but are not added to the master"))
2512                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2513                 for x in myunadded:
2514                         print("   ", x)
2515                 print()
2516                 print()
2517                 sys.exit(1)
2518
2519         if vcs == "hg" and mydeleted:
2520                 print(red("!!! The following files are removed manually from your local tree but are not"))
2521                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2522                 for x in mydeleted:
2523                         print("   ", x)
2524                 print()
2525                 print()
2526                 sys.exit(1)
2527
2528         if vcs == "cvs":
2529                 mycvstree = cvstree.getentries("./", recursive=1)
2530                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2531                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2532                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2533                 bin_blob_pattern = re.compile("^-kb$")
2534                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2535                         recursive=1, basedir="./"))
2536
2537         if vcs == "svn":
2538                 with repoman_popen("svn status") as f:
2539                         svnstatus = f.readlines()
2540                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2541                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2542                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2543
2544                 # Subversion expands keywords specified in svn:keywords properties.
2545                 with repoman_popen("svn propget -R svn:keywords") as f:
2546                         props = f.readlines()
2547                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2548                         for prop in props if " - " in prop)
2549
2550         elif vcs == "git":
2551                 with repoman_popen("git diff-index --name-only "
2552                         "--relative --diff-filter=M HEAD") as f:
2553                         mychanged = f.readlines()
2554                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2555
2556                 with repoman_popen("git diff-index --name-only "
2557                         "--relative --diff-filter=A HEAD") as f:
2558                         mynew = f.readlines()
2559                 mynew = ["./" + elem[:-1] for elem in mynew]
2560
2561                 with repoman_popen("git diff-index --name-only "
2562                         "--relative --diff-filter=D HEAD") as f:
2563                         myremoved = f.readlines()
2564                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2565
2566         if vcs == "bzr":
2567                 with repoman_popen("bzr status -S .") as f:
2568                         bzrstatus = f.readlines()
2569                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2570                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2571                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2572                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2573                 # Bazaar expands nothing.
2574
2575         if vcs == "hg":
2576                 with repoman_popen("hg status --no-status --modified .") as f:
2577                         mychanged = f.readlines()
2578                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2579
2580                 with repoman_popen("hg status --no-status --added .") as f:
2581                         mynew = f.readlines()
2582                 mynew = ["./" + elem.rstrip() for elem in mynew]
2583
2584                 with repoman_popen("hg status --no-status --removed .") as f:
2585                         myremoved = f.readlines()
2586                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2587
2588         if vcs:
2589                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2590                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2591                         print()
2592                         print("(Didn't find any changed files...)")
2593                         print()
2594                         sys.exit(1)
2595
2596         # Manifests need to be regenerated after all other commits, so don't commit
2597         # them now even if they have changed.
2598         mymanifests = set()
2599         myupdates = set()
2600         for f in mychanged + mynew:
2601                 if "Manifest" == os.path.basename(f):
2602                         mymanifests.add(f)
2603                 else:
2604                         myupdates.add(f)
2605         myupdates.difference_update(myremoved)
2606         myupdates = list(myupdates)
2607         mymanifests = list(mymanifests)
2608         myheaders = []
2609         mydirty = []
2610
2611         commitmessage = options.commitmsg
2612         if options.commitmsgfile:
2613                 try:
2614                         f = io.open(_unicode_encode(options.commitmsgfile,
2615                         encoding=_encodings['fs'], errors='strict'),
2616                         mode='r', encoding=_encodings['content'], errors='replace')
2617                         commitmessage = f.read()
2618                         f.close()
2619                         del f
2620                 except (IOError, OSError) as e:
2621                         if e.errno == errno.ENOENT:
2622                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2623                         else:
2624                                 raise
2625                 # We've read the content so the file is no longer needed.
2626                 commitmessagefile = None
2627         if not commitmessage or not commitmessage.strip():
2628                 try:
2629                         editor = os.environ.get("EDITOR")
2630                         if editor and utilities.editor_is_executable(editor):
2631                                 commitmessage = utilities.get_commit_message_with_editor(
2632                                         editor, message=qa_output)
2633                         else:
2634                                 commitmessage = utilities.get_commit_message_with_stdin()
2635                 except KeyboardInterrupt:
2636                         exithandler()
2637                 if not commitmessage or not commitmessage.strip():
2638                         print("* no commit message?  aborting commit.")
2639                         sys.exit(1)
2640         commitmessage = commitmessage.rstrip()
2641         changelog_msg = commitmessage
2642         portage_version = getattr(portage, "VERSION", None)
2643         gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
2644         dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
2645         if portage_version is None:
2646                 sys.stderr.write("Failed to insert portage version in message!\n")
2647                 sys.stderr.flush()
2648                 portage_version = "Unknown"
2649
2650         report_options = []
2651         if options.force:
2652                 report_options.append("--force")
2653         if options.ignore_arches:
2654                 report_options.append("--ignore-arches")
2655         if include_arches is not None:
2656                 report_options.append("--include-arches=\"%s\"" %
2657                         " ".join(sorted(include_arches)))
2658
2659         if vcs == "git":
2660                 # Use new footer only for git (see bug #438364).
2661                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2662                 if report_options:
2663                         commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
2664                 if sign_manifests:
2665                         commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
2666                 if dco_sob:
2667                         commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
2668         else:
2669                 unameout = platform.system() + " "
2670                 if platform.system() in ["Darwin", "SunOS"]:
2671                         unameout += platform.processor()
2672                 else:
2673                         unameout += platform.machine()
2674                 commit_footer = "\n\n"
2675                 if dco_sob:
2676                         commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
2677                 commit_footer += "(Portage version: %s/%s/%s" % \
2678                         (portage_version, vcs, unameout)
2679                 if report_options:
2680                         commit_footer += ", RepoMan options: " + " ".join(report_options)
2681                 if sign_manifests:
2682                         commit_footer += ", signed Manifest commit with key %s" % \
2683                                 (gpg_key, )
2684                 else:
2685                         commit_footer += ", unsigned Manifest commit"
2686                 commit_footer += ")"
2687
2688         commitmessage += commit_footer
2689
2690         if options.echangelog in ('y', 'force'):
2691                 logging.info("checking for unmodified ChangeLog files")
2692                 committer_name = utilities.get_committer_name(env=repoman_settings)
2693                 for x in sorted(vcs_files_to_cps(
2694                         chain(myupdates, mymanifests, myremoved))):
2695                         catdir, pkgdir = x.split("/")
2696                         checkdir = repodir + "/" + x
2697                         checkdir_relative = ""
2698                         if repolevel < 3:
2699                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2700                         if repolevel < 2:
2701                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2702                         checkdir_relative = os.path.join(".", checkdir_relative)
2703
2704                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2705                         changelog_modified = changelog_path in modified_changelogs
2706                         if changelog_modified and options.echangelog != 'force':
2707                                 continue
2708
2709                         # get changes for this package
2710                         cdrlen = len(checkdir_relative)
2711                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2712                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2713                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2714
2715                         # Skip ChangeLog generation if only the Manifest was modified,
2716                         # as discussed in bug #398009.
2717                         nontrivial_cl_files = set()
2718                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2719                         nontrivial_cl_files.difference_update(['Manifest'])
2720                         if not nontrivial_cl_files and options.echangelog != 'force':
2721                                 continue
2722
2723                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2724                                 committer_name, changelog_msg,
2725                                 os.path.join(repodir, 'skel.ChangeLog'),
2726                                 catdir, pkgdir,
2727                                 new=clnew, removed=clremoved, changed=clchanged,
2728                                 pretend=options.pretend)
2729                         if new_changelog is None:
2730                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2731                                         level=logging.ERROR, noiselevel=-1)
2732                                 sys.exit(1)
2733
2734                         # if the ChangeLog was just created, add it to vcs
2735                         if new_changelog:
2736                                 myautoadd.append(changelog_path)
2737                                 # myautoadd is appended to myupdates below
2738                         else:
2739                                 myupdates.append(changelog_path)
2740
2741                         if options.ask and not options.pretend:
2742                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2743                                 repoman_settings["O"] = checkdir
2744                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2745
2746         if myautoadd:
2747                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2748                 add_cmd = [vcs, "add"]
2749                 add_cmd += myautoadd
2750                 if options.pretend:
2751                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2752                                 noiselevel=-1)
2753                 else:
2754                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2755                                 # Python 3.1 produces the following TypeError if raw bytes are
2756                                 # passed to subprocess.call():
2757                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2758                                 #     errread, errwrite)
2759                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2760                                 #     raise child_exception
2761                                 # TypeError: expected an object with the buffer interface
2762                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2763                         retcode = subprocess.call(add_cmd)
2764                         if retcode != os.EX_OK:
2765                                 logging.error(
2766                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2767                                 sys.exit(retcode)
2768
2769                 myupdates += myautoadd
2770
2771         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2772
2773         if vcs not in ('cvs', 'svn'):
2774                 # With git, bzr and hg, there's never any keyword expansion, so
2775                 # there's no need to regenerate manifests and all files will be
2776                 # committed in one big commit at the end.
2777                 print()
2778         elif not repo_config.thin_manifest:
2779                 if vcs == 'cvs':
2780                         headerstring = "'\$(Header|Id).*\$'"
2781                 elif vcs == "svn":
2782                         svn_keywords = dict((k.lower(), k) for k in [
2783                                         "Rev",
2784                                         "Revision",
2785                                         "LastChangedRevision",
2786                                         "Date",
2787                                         "LastChangedDate",
2788                                         "Author",
2789                                         "LastChangedBy",
2790                                         "URL",
2791                                         "HeadURL",
2792                                         "Id",
2793                                         "Header",
2794                         ])
2795
2796                 for myfile in myupdates:
2797
2798                         # for CVS, no_expansion contains files that are excluded from expansion
2799                         if vcs == "cvs":
2800                                 if myfile in no_expansion:
2801                                         continue
2802
2803                         # for SVN, expansion contains files that are included in expansion
2804                         elif vcs == "svn":
2805                                 if myfile not in expansion:
2806                                         continue
2807
2808                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2809                                 enabled_keywords = []
2810                                 for k in expansion[myfile]:
2811                                         keyword = svn_keywords.get(k.lower())
2812                                         if keyword is not None:
2813                                                 enabled_keywords.append(keyword)
2814
2815                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2816
2817                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2818                                 portage._shell_quote(myfile))
2819                         if myout[0] == 0:
2820                                 myheaders.append(myfile)
2821
2822                 print("%s have headers that will change." % green(str(len(myheaders))))
2823                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2824
2825         logging.info("myupdates: %s", myupdates)
2826         logging.info("myheaders: %s", myheaders)
2827
2828         if options.ask and userquery('Commit changes?', True) != 'Yes':
2829                 print("* aborting commit.")
2830                 sys.exit(128 + signal.SIGINT)
2831
2832         # Handle the case where committed files have keywords which
2833         # will change and need a priming commit before the Manifest
2834         # can be committed.
2835         if (myupdates or myremoved) and myheaders:
2836                 myfiles = myupdates + myremoved
2837                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2838                 mymsg = os.fdopen(fd, "wb")
2839                 mymsg.write(_unicode_encode(commitmessage))
2840                 mymsg.close()
2841
2842                 print()
2843                 print(green("Using commit message:"))
2844                 print(green("------------------------------------------------------------------------------"))
2845                 print(commitmessage)
2846                 print(green("------------------------------------------------------------------------------"))
2847                 print()
2848
2849                 # Having a leading ./ prefix on file paths can trigger a bug in
2850                 # the cvs server when committing files to multiple directories,
2851                 # so strip the prefix.
2852                 myfiles = [f.lstrip("./") for f in myfiles]
2853
2854                 commit_cmd = [vcs]
2855                 commit_cmd.extend(vcs_global_opts)
2856                 commit_cmd.append("commit")
2857                 commit_cmd.extend(vcs_local_opts)
2858                 commit_cmd.extend(["-F", commitmessagefile])
2859                 commit_cmd.extend(myfiles)
2860
2861                 try:
2862                         if options.pretend:
2863                                 print("(%s)" % (" ".join(commit_cmd),))
2864                         else:
2865                                 retval = spawn(commit_cmd, env=commit_env)
2866                                 if retval != os.EX_OK:
2867                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2868                                                 "error code: %s\n") % (vcs, retval),
2869                                                 level=logging.ERROR, noiselevel=-1)
2870                                         sys.exit(retval)
2871                 finally:
2872                         try:
2873                                 os.unlink(commitmessagefile)
2874                         except OSError:
2875                                 pass
2876
2877         # Setup the GPG commands
2878         def gpgsign(filename):
2879                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2880                 if gpgcmd is None:
2881                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2882                                 " Is make.globals missing?")
2883                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2884                         "PORTAGE_GPG_KEY" not in repoman_settings:
2885                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2886                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2887                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2888                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2889                                         os.path.expanduser("~/.gnupg")
2890                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2891                                         % repoman_settings["PORTAGE_GPG_DIR"])
2892                         else:
2893                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2894                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2895                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2896                                 raise portage.exception.InvalidLocation(
2897                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2898                                         repoman_settings["PORTAGE_GPG_DIR"])
2899                 gpgvars = {"FILE": filename}
2900                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2901                         v = repoman_settings.get(k)
2902                         if v is not None:
2903                                 gpgvars[k] = v
2904                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2905                 if options.pretend:
2906                         print("(" + gpgcmd + ")")
2907                 else:
2908                         # Encode unicode manually for bug #310789.
2909                         gpgcmd = portage.util.shlex_split(gpgcmd)
2910                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2911                                 # Python 3.1 does not support bytes in Popen args.
2912                                 gpgcmd = [_unicode_encode(arg,
2913                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2914                         rValue = subprocess.call(gpgcmd)
2915                         if rValue == os.EX_OK:
2916                                 os.rename(filename + ".asc", filename)
2917                         else:
2918                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2919
2920         def need_signature(filename):
2921                 try:
2922                         with open(_unicode_encode(filename,
2923                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2924                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2925                 except IOError as e:
2926                         if e.errno in (errno.ENOENT, errno.ESTALE):
2927                                 return False
2928                         raise
2929
2930         # When files are removed and re-added, the cvs server will put /Attic/
2931         # inside the $Header path. This code detects the problem and corrects it
2932         # so that the Manifest will generate correctly. See bug #169500.
2933         # Use binary mode in order to avoid potential character encoding issues.
2934         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2935         attic_str = b'/Attic/'
2936         attic_replace = b'/'
2937         for x in myheaders:
2938                 f = open(_unicode_encode(x,
2939                         encoding=_encodings['fs'], errors='strict'),
2940                         mode='rb')
2941                 mylines = f.readlines()
2942                 f.close()
2943                 modified = False
2944                 for i, line in enumerate(mylines):
2945                         if cvs_header_re.match(line) is not None and \
2946                                 attic_str in line:
2947                                 mylines[i] = line.replace(attic_str, attic_replace)
2948                                 modified = True
2949                 if modified:
2950                         portage.util.write_atomic(x, b''.join(mylines),
2951                                 mode='wb')
2952
2953         if repolevel == 1:
2954                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2955                         "doing the entire repository.\"\n")
2956
2957         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2958
2959                 for x in sorted(vcs_files_to_cps(
2960                         chain(myupdates, myremoved, mymanifests))):
2961                         repoman_settings["O"] = os.path.join(repodir, x)
2962                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2963
2964         signed = False
2965         if sign_manifests:
2966                 signed = True
2967                 try:
2968                         for x in sorted(vcs_files_to_cps(
2969                                 chain(myupdates, myremoved, mymanifests))):
2970                                 repoman_settings["O"] = os.path.join(repodir, x)
2971                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2972                                 if not need_signature(manifest_path):
2973                                         continue
2974                                 gpgsign(manifest_path)
2975                 except portage.exception.PortageException as e:
2976                         portage.writemsg("!!! %s\n" % str(e))
2977                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2978                         signed = False
2979
2980         if vcs == 'git':
2981                 # It's not safe to use the git commit -a option since there might
2982                 # be some modified files elsewhere in the working tree that the
2983                 # user doesn't want to commit. Therefore, call git update-index
2984                 # in order to ensure that the index is updated with the latest
2985                 # versions of all new and modified files in the relevant portion
2986                 # of the working tree.
2987                 myfiles = mymanifests + myupdates
2988                 myfiles.sort()
2989                 update_index_cmd = ["git", "update-index"]
2990                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2991                 if options.pretend:
2992                         print("(%s)" % (" ".join(update_index_cmd),))
2993                 else:
2994                         retval = spawn(update_index_cmd, env=os.environ)
2995                         if retval != os.EX_OK:
2996                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2997                                         "error code: %s\n") % (vcs, retval),
2998                                         level=logging.ERROR, noiselevel=-1)
2999                                 sys.exit(retval)
3000
3001         if True:
3002                 myfiles = mymanifests[:]
3003                 # If there are no header (SVN/CVS keywords) changes in
3004                 # the files, this Manifest commit must include the
3005                 # other (yet uncommitted) files.
3006                 if not myheaders:
3007                         myfiles += myupdates
3008                         myfiles += myremoved
3009                 myfiles.sort()
3010
3011                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3012                 mymsg = os.fdopen(fd, "wb")
3013                 mymsg.write(_unicode_encode(commitmessage))
3014                 mymsg.close()
3015
3016                 commit_cmd = []
3017                 if options.pretend and vcs is None:
3018                         # substitute a bogus value for pretend output
3019                         commit_cmd.append("cvs")
3020                 else:
3021                         commit_cmd.append(vcs)
3022                 commit_cmd.extend(vcs_global_opts)
3023                 commit_cmd.append("commit")
3024                 commit_cmd.extend(vcs_local_opts)
3025                 if vcs == "hg":
3026                         commit_cmd.extend(["--logfile", commitmessagefile])
3027                         commit_cmd.extend(myfiles)
3028                 else:
3029                         commit_cmd.extend(["-F", commitmessagefile])
3030                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3031
3032                 try:
3033                         if options.pretend:
3034                                 print("(%s)" % (" ".join(commit_cmd),))
3035                         else:
3036                                 retval = spawn(commit_cmd, env=commit_env)
3037                                 if retval != os.EX_OK:
3038                                         if repo_config.sign_commit and vcs == 'git' and \
3039                                                 not git_supports_gpg_sign():
3040                                                 # Inform user that newer git is needed (bug #403323).
3041                                                 logging.error(
3042                                                         "Git >=1.7.9 is required for signed commits!")
3043
3044                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3045                                                 "error code: %s\n") % (vcs, retval),
3046                                                 level=logging.ERROR, noiselevel=-1)
3047                                         sys.exit(retval)
3048                 finally:
3049                         try:
3050                                 os.unlink(commitmessagefile)
3051                         except OSError:
3052                                 pass
3053
3054         print()
3055         if vcs:
3056                 print("Commit complete.")
3057         else:
3058                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3059         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3060 sys.exit(0)