bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import io
  17 import logging
  18 import optparse
  19 import re
  20 import signal
  21 import stat
  22 import subprocess
  23 import sys
  24 import tempfile
  25 import textwrap
  26 import time
  27 import platform
  28
  29 try:
  30         from urllib.request import urlopen as urllib_request_urlopen
  31 except ImportError:
  32         from urllib import urlopen as urllib_request_urlopen
  33
  34 from itertools import chain
  35 from stat import S_ISDIR
  36
  37 from os import path as osp
  38 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  39 sys.path.insert(0, pym_path)
  40 import portage
  41 portage._internal_caller = True
  42 portage._disable_legacy_globals()
  43
  44 try:
  45         import xml.etree.ElementTree
  46         from xml.parsers.expat import ExpatError
  47 except (SystemExit, KeyboardInterrupt):
  48         raise
  49 except (ImportError, SystemError, RuntimeError, Exception):
  50         # broken or missing xml support
  51         # http://bugs.python.org/issue14988
  52         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  53         from portage.output import EOutput
  54         out = EOutput()
  55         for line in msg:
  56                 out.eerror(line)
  57         sys.exit(1)
  58
  59 from portage import os
  60 from portage import _encodings
  61 from portage import _unicode_encode
  62 from repoman.checks import run_checks
  63 from repoman import utilities
  64 from repoman.herdbase import make_herd_base
  65 from _emerge.Package import Package
  66 from _emerge.RootConfig import RootConfig
  67 from _emerge.userquery import userquery
  68 import portage.checksum
  69 import portage.const
  70 from portage import cvstree, normalize_path
  71 from portage import util
  72 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  73         ParseError, PermissionDenied)
  74 from portage.dep import Atom
  75 from portage.process import find_binary, spawn
  76 from portage.output import bold, create_color_func, \
  77         green, nocolor, red
  78 from portage.output import ConsoleStyleFile, StyleWriter
  79 from portage.util import writemsg_level
  80 from portage.package.ebuild.digestgen import digestgen
  81 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  82
  83 if sys.hexversion >= 0x3000000:
  84         basestring = str
  85
  86 util.initialize_logger()
  87
  88 # 14 is the length of DESCRIPTION=""
  89 max_desc_len = 100
  90 allowed_filename_chars="a-zA-Z0-9._-+:"
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  93 bad = create_color_func("BAD")
  94
  95 # A sane umask is needed for files that portage creates.
  96 os.umask(0o22)
  97 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  98 # behave incrementally.
  99 repoman_incrementals = tuple(x for x in \
 100         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 101 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 102 repoman_settings = portage.config(config_root=config_root, local_config=False)
 103
 104 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 105         repoman_settings.get('TERM') == 'dumb' or \
 106         not sys.stdout.isatty():
 107         nocolor()
 108
 109 def warn(txt):
 110         print("repoman: " + txt)
 111
 112 def err(txt):
 113         warn(txt)
 114         sys.exit(1)
 115
 116 def exithandler(signum=None, frame=None):
 117         logging.fatal("Interrupted; exiting...")
 118         if signum is None:
 119                 sys.exit(1)
 120         else:
 121                 sys.exit(128 + signum)
 122
 123 signal.signal(signal.SIGINT, exithandler)
 124
 125 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 126         """Repoman needs its own HelpFormatter for now, because the default ones
 127         murder the help text."""
 128
 129         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 130                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 131
 132         def format_description(self, description):
 133                 return description
 134
 135 class RepomanOptionParser(optparse.OptionParser):
 136         """Add the on_tail function, ruby has it, optionParser should too
 137         """
 138
 139         def __init__(self, *args, **kwargs):
 140                 optparse.OptionParser.__init__(self, *args, **kwargs)
 141                 self.tail = ""
 142
 143         def on_tail(self, description):
 144                 self.tail += description
 145
 146         def format_help(self, formatter=None):
 147                 result = optparse.OptionParser.format_help(self, formatter)
 148                 result += self.tail
 149                 return result
 150
 151
 152 def ParseArgs(argv, qahelp):
 153         """This function uses a customized optionParser to parse command line arguments for repoman
 154         Args:
 155           argv - a sequence of command line arguments
 156                 qahelp - a dict of qa warning to help message
 157         Returns:
 158           (opts, args), just like a call to parser.parse_args()
 159         """
 160
 161         if argv and isinstance(argv[0], bytes):
 162                 argv = [portage._unicode_decode(x) for x in argv]
 163
 164         modes = {
 165                 'commit' : 'Run a scan then commit changes',
 166                 'ci' : 'Run a scan then commit changes',
 167                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 168                 'full' : 'Scan directory tree and print all issues (not a summary)',
 169                 'help' : 'Show this screen',
 170                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 171                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 172                 'scan' : 'Scan directory tree for QA issues'
 173         }
 174
 175         mode_keys = list(modes)
 176         mode_keys.sort()
 177
 178         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 179         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 180         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 181         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 182         parser.description += "\nmodes: " + " | ".join(map(green, mode_keys))
 183
 184         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 185                 help='Request a confirmation before commiting')
 186
 187         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 188                 help='specify a commit message on the command line')
 189
 190         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 191                 help='specify a path to a file that contains a commit message')
 192
 193         parser.add_option('--digest',
 194                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 195                 help='Automatically update Manifest digests for modified files')
 196
 197         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 198                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 199
 200         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 201                 help='do not print unnecessary messages')
 202
 203         parser.add_option(
 204                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 205                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 206                 'regardless of modification if \'force\' is specified)')
 207
 208         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 209                 help='Commit with QA violations')
 210
 211         parser.add_option('--vcs', dest='vcs',
 212                 help='Force using specific VCS instead of autodetection')
 213
 214         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 215                 help='be very verbose in output', default=0)
 216
 217         parser.add_option('-V', '--version', dest='version', action='store_true',
 218                 help='show version info')
 219
 220         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 221                 default=False, help='forces the metadata.xml parse check to be carried out')
 222
 223         parser.add_option(
 224                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 225                 metavar="<y|n>",
 226                 help='only check packages that have uncommitted modifications')
 227
 228         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 229                 default=False, help='ignore arch-specific failures (where arch != host)')
 230
 231         parser.add_option("--ignore-default-opts",
 232                 action="store_true",
 233                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 234
 235         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 236                 default=False, help='ignore masked packages (not allowed with commit mode)')
 237
 238         parser.add_option('--include-arches', dest='include_arches',
 239                 metavar='ARCHES', action='append',
 240                 help='A space separated list of arches used to '
 241                 'filter the selection of profiles for dependency checks')
 242
 243         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 244                 default=False, help='include dev profiles in dependency checks')
 245
 246         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 247                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 248
 249         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 250                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 251
 252         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 253                 help='specify which mode repoman will run in (default=full)')
 254
 255         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 256
 257         for k in mode_keys:
 258                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 259
 260         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 261
 262         sorted_qa = list(qahelp)
 263         sorted_qa.sort()
 264         for k in sorted_qa:
 265                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 266
 267         opts, args = parser.parse_args(argv[1:])
 268
 269         if not opts.ignore_default_opts:
 270                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 271                 if default_opts:
 272                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 273
 274         if opts.mode == 'help':
 275                 parser.print_help(short=False)
 276
 277         for arg in args:
 278                 if arg in modes:
 279                         if not opts.mode:
 280                                 opts.mode = arg
 281                                 break
 282                 else:
 283                         parser.error("invalid mode: %s" % arg)
 284
 285         if not opts.mode:
 286                 opts.mode = 'full'
 287
 288         if opts.mode == 'ci':
 289                 opts.mode = 'commit'  # backwards compat shortcut
 290
 291         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 292                 if opts.ignore_masked:
 293                         parser.error('Commit mode and --ignore-masked are not compatible')
 294                 if opts.without_mask:
 295                         parser.error('Commit mode and --without-mask are not compatible')
 296
 297         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 298         for val in range(opts.verbosity):
 299                 logger = logging.getLogger()
 300                 logger.setLevel(logger.getEffectiveLevel() - 10)
 301
 302         for val in range(opts.quiet):
 303                 logger = logging.getLogger()
 304                 logger.setLevel(logger.getEffectiveLevel() + 10)
 305
 306         return (opts, args)
 307
 308 qahelp = {
 309         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 310         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 311         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 312         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 313         "changelog.missing": "Missing ChangeLog files",
 314         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 315         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 316         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 317         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 318         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 319         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 320         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 321         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 322         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 323         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 324         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 325         "file.size": "Files in the files directory must be under 20 KiB",
 326         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 327         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 328         "file.UTF8": "File is not UTF8 compliant",
 329         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 330         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 331         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 332         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 333         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 334         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 335         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 336         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 337         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 338         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 339         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 340         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 341         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 342         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 343         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 344         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 345         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 346         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 347         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 348         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 349         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 350         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 351         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 352         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 353         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 354         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 355         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 356         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 357         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 358         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 359         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 360         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 361         "variable.readonly": "Assigning a readonly variable",
 362         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 363         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 364         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 365         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 366         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 367         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 368         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 369         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 370         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 371         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 372         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 373         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 374         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 375         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 376         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 377         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 378         "ebuild.badheader": "This ebuild has a malformed header",
 379         "manifest.bad": "Manifest has missing or incorrect digests",
 380         "metadata.missing": "Missing metadata.xml files",
 381         "metadata.bad": "Bad metadata.xml files",
 382         "metadata.warning": "Warnings in metadata.xml files",
 383         "portage.internal": "The ebuild uses an internal Portage function or variable",
 384         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 385         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 386         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 387         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 388 }
 389
 390 qacats = list(qahelp)
 391 qacats.sort()
 392
 393 qawarnings = set((
 394 "changelog.missing",
 395 "changelog.notadded",
 396 "dependency.unknown",
 397 "digest.assumed",
 398 "digest.unused",
 399 "ebuild.notadded",
 400 "ebuild.nesteddie",
 401 "dependency.badmasked",
 402 "dependency.badindev",
 403 "dependency.badmaskedindev",
 404 "dependency.badtilde",
 405 "DESCRIPTION.toolong",
 406 "EAPI.deprecated",
 407 "HOMEPAGE.virtual",
 408 "LICENSE.deprecated",
 409 "LICENSE.virtual",
 410 "KEYWORDS.dropped",
 411 "KEYWORDS.stupid",
 412 "KEYWORDS.missing",
 413 "PDEPEND.suspect",
 414 "RDEPEND.implicit",
 415 "RDEPEND.suspect",
 416 "virtual.suspect",
 417 "RESTRICT.invalid",
 418 "ebuild.minorsyn",
 419 "ebuild.badheader",
 420 "ebuild.patches",
 421 "file.size",
 422 "inherit.unused",
 423 "inherit.deprecated",
 424 "java.eclassesnotused",
 425 "wxwidgets.eclassnotused",
 426 "metadata.warning",
 427 "portage.internal",
 428 "usage.obsolete",
 429 "upstream.workaround",
 430 "LIVEVCS.stable",
 431 "LIVEVCS.unmasked",
 432 ))
 433
 434 if portage.const._ENABLE_INHERIT_CHECK:
 435         # This is experimental, so it's non-fatal.
 436         qawarnings.add("inherit.missing")
 437
 438 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 439
 440 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 441 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 442 allvars.update(Package.metadata_keys)
 443 allvars = sorted(allvars)
 444 commitmessage = None
 445 for x in missingvars:
 446         x += ".missing"
 447         if x not in qacats:
 448                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 449                 qacats.append(x)
 450                 qawarnings.add(x)
 451
 452 valid_restrict = frozenset(["binchecks", "bindist",
 453         "fetch", "installsources", "mirror", "preserve-libs",
 454         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 455
 456 live_eclasses = frozenset([
 457         "bzr",
 458         "cvs",
 459         "darcs",
 460         "git",
 461         "git-2",
 462         "mercurial",
 463         "subversion",
 464         "tla",
 465 ])
 466
 467 suspect_rdepend = frozenset([
 468         "app-arch/cabextract",
 469         "app-arch/rpm2targz",
 470         "app-doc/doxygen",
 471         "dev-lang/nasm",
 472         "dev-lang/swig",
 473         "dev-lang/yasm",
 474         "dev-perl/extutils-pkgconfig",
 475         "dev-util/byacc",
 476         "dev-util/cmake",
 477         "dev-util/ftjam",
 478         "dev-util/gperf",
 479         "dev-util/gtk-doc",
 480         "dev-util/gtk-doc-am",
 481         "dev-util/intltool",
 482         "dev-util/jam",
 483         "dev-util/pkg-config-lite",
 484         "dev-util/pkgconf",
 485         "dev-util/pkgconfig",
 486         "dev-util/pkgconfig-openbsd",
 487         "dev-util/scons",
 488         "dev-util/unifdef",
 489         "dev-util/yacc",
 490         "media-gfx/ebdftopcf",
 491         "sys-apps/help2man",
 492         "sys-devel/autoconf",
 493         "sys-devel/automake",
 494         "sys-devel/bin86",
 495         "sys-devel/bison",
 496         "sys-devel/dev86",
 497         "sys-devel/flex",
 498         "sys-devel/m4",
 499         "sys-devel/pmake",
 500         "virtual/linux-sources",
 501         "virtual/pkgconfig",
 502         "x11-misc/bdftopcf",
 503         "x11-misc/imake",
 504 ])
 505
 506 suspect_virtual = {
 507         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 508         "dev-util/pkgconf":"virtual/pkgconfig",
 509         "dev-util/pkgconfig":"virtual/pkgconfig",
 510         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 511         "dev-libs/libusb":"virtual/libusb",
 512         "dev-libs/libusbx":"virtual/libusb",
 513         "dev-libs/libusb-compat":"virtual/libusb",
 514 }
 515
 516 metadata_xml_encoding = 'UTF-8'
 517 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 518         (metadata_xml_encoding,)
 519 metadata_doctype_name = 'pkgmetadata'
 520 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 521 # force refetch if the local copy creation time is older than this
 522 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 523
 524 # file.executable
 525 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 526
 527 options, arguments = ParseArgs(sys.argv, qahelp)
 528
 529 if options.version:
 530         print("Portage", portage.VERSION)
 531         sys.exit(0)
 532
 533 # Set this to False when an extraordinary issue (generally
 534 # something other than a QA issue) makes it impossible to
 535 # commit (like if Manifest generation fails).
 536 can_force = True
 537
 538 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 539 if portdir is None:
 540         sys.exit(1)
 541
 542 myreporoot = os.path.basename(portdir_overlay)
 543 myreporoot += mydir[len(portdir_overlay):]
 544
 545 if options.vcs:
 546         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 547                 vcs = options.vcs
 548         else:
 549                 vcs = None
 550 else:
 551         vcses = utilities.FindVCS()
 552         if len(vcses) > 1:
 553                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 554                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 555                 sys.exit(1)
 556         elif vcses:
 557                 vcs = vcses[0]
 558         else:
 559                 vcs = None
 560
 561 if options.if_modified == "y" and vcs is None:
 562         logging.info("Not in a version controlled repository; "
 563                 "disabling --if-modified.")
 564         options.if_modified = "n"
 565
 566 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 567 vcs_preserves_mtime = vcs in ('cvs',)
 568
 569 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 570 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 571 if vcs_global_opts is None:
 572         if vcs in ('cvs', 'svn'):
 573                 vcs_global_opts = "-q"
 574         else:
 575                 vcs_global_opts = ""
 576 vcs_global_opts = vcs_global_opts.split()
 577
 578 if options.mode == 'commit' and not options.pretend and not vcs:
 579         logging.info("Not in a version controlled repository; enabling pretend mode.")
 580         options.pretend = True
 581
 582 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 583 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 584         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 585         portage._shell_quote(portdir_overlay))
 586 # We have to call the config constructor again so
 587 # that config.repositories is initialized correctly.
 588 repoman_settings = portage.config(config_root=config_root, local_config=False,
 589         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 590
 591 root = repoman_settings['EROOT']
 592 trees = {
 593         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 594 }
 595 portdb = trees[root]['porttree'].dbapi
 596
 597 # Constrain dependency resolution to the master(s)
 598 # that are specified in layout.conf.
 599 repodir = os.path.realpath(portdir_overlay)
 600 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 601 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 602 portdir = portdb.porttrees[0]
 603 commit_env = os.environ.copy()
 604
 605 if repo_config.allow_provide_virtual:
 606         qawarnings.add("virtual.oldstyle")
 607
 608 if repo_config.sign_commit:
 609         if vcs == 'git':
 610                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 611                 # the commit arguments. If key_id is unspecified, then it must be
 612                 # configured by `git config user.signingkey key_id`.
 613                 vcs_local_opts.append("--gpg-sign")
 614                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 615                         # Pass GNUPGHOME to git for bug #462362.
 616                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 617
 618 # In order to disable manifest signatures, repos may set
 619 # "sign-manifests = false" in metadata/layout.conf. This
 620 # can be used to prevent merge conflicts like those that
 621 # thin-manifests is designed to prevent.
 622 sign_manifests = "sign" in repoman_settings.features and \
 623         repo_config.sign_manifest
 624
 625 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 626         options.mode in ("commit",) and not sign_manifests:
 627         msg = ("The '%s' repository has manifest signatures enabled, "
 628         "but FEATURES=sign is currently disabled. In order to avoid this "
 629         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 630         "repositories can disable manifest signatures by setting "
 631         "'sign-manifests = false' in metadata/layout.conf.") % \
 632         (repo_config.name,)
 633         for line in textwrap.wrap(msg, 60):
 634                 logging.warn(line)
 635
 636 if sign_manifests and options.mode in ("commit",) and \
 637         repoman_settings.get("PORTAGE_GPG_KEY") and \
 638         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 639         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 640         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 641                 repoman_settings["PORTAGE_GPG_KEY"])
 642         sys.exit(1)
 643
 644 manifest_hashes = repo_config.manifest_hashes
 645 if manifest_hashes is None:
 646         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 647
 648 if options.mode in ("commit", "fix", "manifest"):
 649         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 650                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 651                 "metadata/layout.conf does not contain the '%s' hash which "
 652                 "is required by this portage version. You will have to "
 653                 "upgrade portage if you want to generate valid manifests for "
 654                 "this repository.") % \
 655                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 656                 for line in textwrap.wrap(msg, 70):
 657                         logging.error(line)
 658                 sys.exit(1)
 659
 660         unsupported_hashes = manifest_hashes.difference(
 661                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 662         if unsupported_hashes:
 663                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 664                 "metadata/layout.conf contains one or more hash types '%s' "
 665                 "which are not supported by this portage version. You will "
 666                 "have to upgrade portage if you want to generate valid "
 667                 "manifests for this repository.") % \
 668                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 669                 for line in textwrap.wrap(msg, 70):
 670                         logging.error(line)
 671                 sys.exit(1)
 672
 673 if options.echangelog is None and repo_config.update_changelog:
 674         options.echangelog = 'y'
 675
 676 if vcs is None:
 677         options.echangelog = 'n'
 678
 679 # The --echangelog option causes automatic ChangeLog generation,
 680 # which invalidates changelog.ebuildadded and changelog.missing
 681 # checks.
 682 # Note: Some don't use ChangeLogs in distributed SCMs.
 683 # It will be generated on server side from scm log,
 684 # before package moves to the rsync server.
 685 # This is needed because they try to avoid merge collisions.
 686 # Gentoo's Council decided to always use the ChangeLog file.
 687 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 688 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 689
 690 if 'digest' in repoman_settings.features and options.digest != 'n':
 691         options.digest = 'y'
 692
 693 logging.debug("vcs: %s" % (vcs,))
 694 logging.debug("repo config: %s" % (repo_config,))
 695 logging.debug("options: %s" % (options,))
 696
 697 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 698 # profile-specific config constructor calls.
 699 env = os.environ.copy()
 700 env['PORTDIR'] = portdir
 701 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 702
 703 logging.info('Setting paths:')
 704 logging.info('PORTDIR = "' + portdir + '"')
 705 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 706
 707 # It's confusing if these warnings are displayed without the user
 708 # being told which profile they come from, so disable them.
 709 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 710
 711 categories = []
 712 for path in repo_config.eclass_db.porttrees:
 713         categories.extend(portage.util.grabfile(
 714                 os.path.join(path, 'profiles', 'categories')))
 715 repoman_settings.categories = frozenset(
 716         portage.util.stack_lists([categories], incremental=1))
 717 categories = repoman_settings.categories
 718
 719 portdb.settings = repoman_settings
 720 root_config = RootConfig(repoman_settings, trees[root], None)
 721 # We really only need to cache the metadata that's necessary for visibility
 722 # filtering. Anything else can be discarded to reduce memory consumption.
 723 portdb._aux_cache_keys.clear()
 724 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 725
 726 reposplit = myreporoot.split(os.path.sep)
 727 repolevel = len(reposplit)
 728
 729 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 730 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 731 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 732 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 733         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 734         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 735         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 736         print(red("***"))
 737         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 738
 739 # Make startdir relative to the canonical repodir, so that we can pass
 740 # it to digestgen and it won't have to be canonicalized again.
 741 if repolevel == 1:
 742         startdir = repodir
 743 else:
 744         startdir = normalize_path(mydir)
 745         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 746
 747 def caterror(mycat):
 748         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 749
 750 def repoman_getstatusoutput(cmd):
 751         """
 752         Implements an interface similar to getstatusoutput(), but with
 753         customized unicode handling (see bug #310789) and without the shell.
 754         """
 755         args = portage.util.shlex_split(cmd)
 756         encoding = _encodings['fs']
 757         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 758                 # Python 3.1 does not support bytes in Popen args.
 759                 args = [_unicode_encode(x,
 760                         encoding=encoding, errors='strict') for x in args]
 761         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 762                 stderr=subprocess.STDOUT)
 763         output = portage._unicode_decode(proc.communicate()[0],
 764                 encoding=encoding, errors='strict')
 765         if output and output[-1] == "\n":
 766                 # getstatusoutput strips one newline
 767                 output = output[:-1]
 768         return (proc.wait(), output)
 769
 770 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 771         """
 772         Implements an interface similar to os.popen(), but with customized
 773         unicode handling (see bug #310789) and without the shell.
 774         """
 775
 776         __slots__ = ('_proc', '_stdout')
 777
 778         def __init__(self, cmd):
 779                 args = portage.util.shlex_split(cmd)
 780                 encoding = _encodings['fs']
 781                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 782                         # Python 3.1 does not support bytes in Popen args.
 783                         args = [_unicode_encode(x,
 784                                 encoding=encoding, errors='strict') for x in args]
 785                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 786                 object.__setattr__(self, '_proc', proc)
 787                 object.__setattr__(self, '_stdout',
 788                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 789
 790         def _get_target(self):
 791                 return object.__getattribute__(self, '_stdout')
 792
 793         __enter__ = _get_target
 794
 795         def __exit__(self, exc_type, exc_value, traceback):
 796                 proc = object.__getattribute__(self, '_proc')
 797                 proc.wait()
 798                 proc.stdout.close()
 799
 800 class ProfileDesc(object):
 801         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 802         def __init__(self, arch, status, sub_path, tree_path):
 803                 self.arch = arch
 804                 self.status = status
 805                 if sub_path:
 806                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 807                 self.sub_path = sub_path
 808                 self.tree_path = tree_path
 809                 if tree_path:
 810                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 811                 else:
 812                         self.abs_path = tree_path
 813
 814         def __str__(self):
 815                 if self.sub_path:
 816                         return self.sub_path
 817                 return 'empty profile'
 818
 819 profile_list = []
 820 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 821
 822 # get lists of valid keywords, licenses, and use
 823 kwlist = set()
 824 liclist = set()
 825 uselist = set()
 826 global_pmasklines = []
 827
 828 for path in portdb.porttrees:
 829         try:
 830                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 831         except OSError:
 832                 pass
 833         kwlist.update(portage.grabfile(os.path.join(path,
 834                 "profiles", "arch.list")))
 835
 836         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 837         for x in use_desc:
 838                 x = x.split()
 839                 if x:
 840                         uselist.add(x[0])
 841
 842         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 843         try:
 844                 expand_list = os.listdir(expand_desc_dir)
 845         except OSError:
 846                 pass
 847         else:
 848                 for fn in expand_list:
 849                         if not fn[-5:] == '.desc':
 850                                 continue
 851                         use_prefix = fn[:-5].lower() + '_'
 852                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 853                                 x = x.split()
 854                                 if x:
 855                                         uselist.add(use_prefix + x[0])
 856
 857         global_pmasklines.append(portage.util.grabfile_package(
 858                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 859
 860         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 861         try:
 862                 desc_file = io.open(_unicode_encode(desc_path,
 863                         encoding=_encodings['fs'], errors='strict'),
 864                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 865         except EnvironmentError:
 866                 pass
 867         else:
 868                 for i, x in enumerate(desc_file):
 869                         if x[0] == "#":
 870                                 continue
 871                         arch = x.split()
 872                         if len(arch) == 0:
 873                                 continue
 874                         if len(arch) != 3:
 875                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 876                                         desc_path + " line %d" % (i + 1, ))
 877                         elif arch[0] not in kwlist:
 878                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 879                                         desc_path + " line %d" % (i + 1, ))
 880                         elif arch[2] not in valid_profile_types:
 881                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 882                                         desc_path + " line %d" % (i + 1, ))
 883                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 884                         if not os.path.isdir(profile_desc.abs_path):
 885                                 logging.error(
 886                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 887                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 888                                 continue
 889                         if os.path.exists(
 890                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 891                                 continue
 892                         profile_list.append(profile_desc)
 893                 desc_file.close()
 894
 895 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 896 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 897
 898 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 899 global_pmaskdict = {}
 900 for x in global_pmasklines:
 901         global_pmaskdict.setdefault(x.cp, []).append(x)
 902 del global_pmasklines
 903
 904 def has_global_mask(pkg):
 905         mask_atoms = global_pmaskdict.get(pkg.cp)
 906         if mask_atoms:
 907                 pkg_list = [pkg]
 908                 for x in mask_atoms:
 909                         if portage.dep.match_from_list(x, pkg_list):
 910                                 return x
 911         return None
 912
 913 # Ensure that profile sub_path attributes are unique. Process in reverse order
 914 # so that profiles with duplicate sub_path from overlays will override
 915 # profiles with the same sub_path from parent repos.
 916 profiles = {}
 917 profile_list.reverse()
 918 profile_sub_paths = set()
 919 for prof in profile_list:
 920         if prof.sub_path in profile_sub_paths:
 921                 continue
 922         profile_sub_paths.add(prof.sub_path)
 923         profiles.setdefault(prof.arch, []).append(prof)
 924
 925 # Use an empty profile for checking dependencies of
 926 # packages that have empty KEYWORDS.
 927 prof = ProfileDesc('**', 'stable', '', '')
 928 profiles.setdefault(prof.arch, []).append(prof)
 929
 930 for x in repoman_settings.archlist():
 931         if x[0] == "~":
 932                 continue
 933         if x not in profiles:
 934                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 935                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 936                 print(red("up with the " + x + " team."))
 937                 print()
 938
 939 liclist_deprecated = set()
 940 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 941         liclist_deprecated.update(
 942                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 943
 944 if not liclist:
 945         logging.fatal("Couldn't find licenses?")
 946         sys.exit(1)
 947
 948 if not kwlist:
 949         logging.fatal("Couldn't read KEYWORDS from arch.list")
 950         sys.exit(1)
 951
 952 if not uselist:
 953         logging.fatal("Couldn't find use.desc?")
 954         sys.exit(1)
 955
 956 scanlist = []
 957 if repolevel == 2:
 958         # we are inside a category directory
 959         catdir = reposplit[-1]
 960         if catdir not in categories:
 961                 caterror(catdir)
 962         mydirlist = os.listdir(startdir)
 963         for x in mydirlist:
 964                 if x == "CVS" or x.startswith("."):
 965                         continue
 966                 if os.path.isdir(startdir + "/" + x):
 967                         scanlist.append(catdir + "/" + x)
 968         repo_subdir = catdir + os.sep
 969 elif repolevel == 1:
 970         for x in categories:
 971                 if not os.path.isdir(startdir + "/" + x):
 972                         continue
 973                 for y in os.listdir(startdir + "/" + x):
 974                         if y == "CVS" or y.startswith("."):
 975                                 continue
 976                         if os.path.isdir(startdir + "/" + x + "/" + y):
 977                                 scanlist.append(x + "/" + y)
 978         repo_subdir = ""
 979 elif repolevel == 3:
 980         catdir = reposplit[-2]
 981         if catdir not in categories:
 982                 caterror(catdir)
 983         scanlist.append(catdir + "/" + reposplit[-1])
 984         repo_subdir = scanlist[-1] + os.sep
 985 else:
 986         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 987                 ' from the current working directory'
 988         logging.critical(msg)
 989         sys.exit(1)
 990
 991 repo_subdir_len = len(repo_subdir)
 992 scanlist.sort()
 993
 994 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 995
 996 def vcs_files_to_cps(vcs_file_iter):
 997         """
 998         Iterate over the given modified file paths returned from the vcs,
 999         and return a frozenset containing category/pn strings for each
1000         modified package.
1001         """
1002
1003         modified_cps = []
1004
1005         if repolevel == 3:
1006                 if reposplit[-2] in categories and \
1007                         next(vcs_file_iter, None) is not None:
1008                         modified_cps.append("/".join(reposplit[-2:]))
1009
1010         elif repolevel == 2:
1011                 category = reposplit[-1]
1012                 if category in categories:
1013                         for filename in vcs_file_iter:
1014                                 f_split = filename.split(os.sep)
1015                                 # ['.', pn, ...]
1016                                 if len(f_split) > 2:
1017                                         modified_cps.append(category + "/" + f_split[1])
1018
1019         else:
1020                 # repolevel == 1
1021                 for filename in vcs_file_iter:
1022                         f_split = filename.split(os.sep)
1023                         # ['.', category, pn, ...]
1024                         if len(f_split) > 3 and f_split[1] in categories:
1025                                 modified_cps.append("/".join(f_split[1:3]))
1026
1027         return frozenset(modified_cps)
1028
1029 def git_supports_gpg_sign():
1030         status, cmd_output = \
1031                 repoman_getstatusoutput("git --version")
1032         cmd_output = cmd_output.split()
1033         if cmd_output:
1034                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1035                 if version is not None:
1036                         version = [int(x) for x in version.groups()]
1037                         if version[0] > 1 or \
1038                                 (version[0] == 1 and version[1] > 7) or \
1039                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1040                                 return True
1041         return False
1042
1043 def dev_keywords(profiles):
1044         """
1045         Create a set of KEYWORDS values that exist in 'dev'
1046         profiles. These are used
1047         to trigger a message notifying the user when they might
1048         want to add the --include-dev option.
1049         """
1050         type_arch_map = {}
1051         for arch, arch_profiles in profiles.items():
1052                 for prof in arch_profiles:
1053                         arch_set = type_arch_map.get(prof.status)
1054                         if arch_set is None:
1055                                 arch_set = set()
1056                                 type_arch_map[prof.status] = arch_set
1057                         arch_set.add(arch)
1058
1059         dev_keywords = type_arch_map.get('dev', set())
1060         dev_keywords.update(['~' + arch for arch in dev_keywords])
1061         return frozenset(dev_keywords)
1062
1063 dev_keywords = dev_keywords(profiles)
1064
1065 stats = {}
1066 fails = {}
1067
1068 for x in qacats:
1069         stats[x] = 0
1070         fails[x] = []
1071
1072 xmllint_capable = False
1073 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1074
1075 def parsedate(s):
1076         """Parse a RFC 822 date and time string.
1077         This is required for python3 compatibility, since the
1078         rfc822.parsedate() function is not available."""
1079
1080         s_split = []
1081         for x in s.upper().split():
1082                 for y in x.split(','):
1083                         if y:
1084                                 s_split.append(y)
1085
1086         if len(s_split) != 6:
1087                 return None
1088
1089         # %a, %d %b %Y %H:%M:%S %Z
1090         a, d, b, Y, H_M_S, Z = s_split
1091
1092         # Convert month to integer, since strptime %w is locale-dependent.
1093         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1094                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1095         m = month_map.get(b)
1096         if m is None:
1097                 return None
1098         m = "%02d" % m
1099
1100         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1101
1102 def fetch_metadata_dtd():
1103         """
1104         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1105         metadata_dtd_ctime_interval.
1106         @rtype: bool
1107         @return: True if successful, otherwise False
1108         """
1109
1110         must_fetch = True
1111         metadata_dtd_st = None
1112         current_time = int(time.time())
1113         try:
1114                 metadata_dtd_st = os.stat(metadata_dtd)
1115         except EnvironmentError as e:
1116                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1117                         raise
1118                 del e
1119         else:
1120                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1121                 if abs(current_time - metadata_dtd_st.st_ctime) \
1122                         < metadata_dtd_ctime_interval:
1123                         must_fetch = False
1124
1125         if must_fetch:
1126                 print()
1127                 print(green("***") + " the local copy of metadata.dtd " + \
1128                         "needs to be refetched, doing that now")
1129                 print()
1130                 try:
1131                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1132                         msg_info = url_f.info()
1133                         last_modified = msg_info.get('last-modified')
1134                         if last_modified is not None:
1135                                 last_modified = parsedate(last_modified)
1136                                 if last_modified is not None:
1137                                         last_modified = calendar.timegm(last_modified)
1138
1139                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1140                         try:
1141                                 local_f = open(metadata_dtd_tmp, mode='wb')
1142                                 local_f.write(url_f.read())
1143                                 local_f.close()
1144                                 if last_modified is not None:
1145                                         try:
1146                                                 os.utime(metadata_dtd_tmp,
1147                                                         (int(last_modified), int(last_modified)))
1148                                         except OSError:
1149                                                 # This fails on some odd non-unix-like filesystems.
1150                                                 # We don't really need the mtime to be preserved
1151                                                 # anyway here (currently we use ctime to trigger
1152                                                 # fetch), so just ignore it.
1153                                                 pass
1154                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1155                         finally:
1156                                 try:
1157                                         os.unlink(metadata_dtd_tmp)
1158                                 except OSError:
1159                                         pass
1160
1161                         url_f.close()
1162
1163                 except EnvironmentError as e:
1164                         print()
1165                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1166                         print(red("!!!")+" exception '%s' though." % (e,))
1167                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1168                         return False
1169
1170         return True
1171
1172 if options.mode == "manifest":
1173         pass
1174 elif not find_binary('xmllint'):
1175         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1176         if options.xml_parse or repolevel == 3:
1177                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1178                 sys.exit(1)
1179 else:
1180         if not fetch_metadata_dtd():
1181                 sys.exit(1)
1182         # this can be problematic if xmllint changes their output
1183         xmllint_capable = True
1184
1185 if options.mode == 'commit' and vcs:
1186         utilities.detect_vcs_conflicts(options, vcs)
1187
1188 if options.mode == "manifest":
1189         pass
1190 elif options.pretend:
1191         print(green("\nRepoMan does a once-over of the neighborhood..."))
1192 else:
1193         print(green("\nRepoMan scours the neighborhood..."))
1194
1195 new_ebuilds = set()
1196 modified_ebuilds = set()
1197 modified_changelogs = set()
1198 mychanged = []
1199 mynew = []
1200 myremoved = []
1201
1202 if vcs == "cvs":
1203         mycvstree = cvstree.getentries("./", recursive=1)
1204         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1205         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1206         if options.if_modified == "y":
1207                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1208
1209 elif vcs == "svn":
1210         with repoman_popen("svn status") as f:
1211                 svnstatus = f.readlines()
1212         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1213         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1214         if options.if_modified == "y":
1215                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1216
1217 elif vcs == "git":
1218         with repoman_popen("git diff-index --name-only "
1219                 "--relative --diff-filter=M HEAD") as f:
1220                 mychanged = f.readlines()
1221         mychanged = ["./" + elem[:-1] for elem in mychanged]
1222
1223         with repoman_popen("git diff-index --name-only "
1224                 "--relative --diff-filter=A HEAD") as f:
1225                 mynew = f.readlines()
1226         mynew = ["./" + elem[:-1] for elem in mynew]
1227         if options.if_modified == "y":
1228                 with repoman_popen("git diff-index --name-only "
1229                         "--relative --diff-filter=D HEAD") as f:
1230                         myremoved = f.readlines()
1231                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1232
1233 elif vcs == "bzr":
1234         with repoman_popen("bzr status -S .") as f:
1235                 bzrstatus = f.readlines()
1236         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1237         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1238         if options.if_modified == "y":
1239                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1240
1241 elif vcs == "hg":
1242         with repoman_popen("hg status --no-status --modified .") as f:
1243                 mychanged = f.readlines()
1244         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1245         with repoman_popen("hg status --no-status --added .") as f:
1246                 mynew = f.readlines()
1247         mynew = ["./" + elem.rstrip() for elem in mynew]
1248         if options.if_modified == "y":
1249                 with repoman_popen("hg status --no-status --removed .") as f:
1250                         myremoved = f.readlines()
1251                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1252
1253 if vcs:
1254         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1255         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1256         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1257                 if os.path.basename(x) == "ChangeLog")
1258
1259 def vcs_new_changed(relative_path):
1260         for x in chain(mychanged, mynew):
1261                 if x == relative_path:
1262                         return True
1263         return False
1264
1265 have_pmasked = False
1266 have_dev_keywords = False
1267 dofail = 0
1268
1269 # NOTE: match-all caches are not shared due to potential
1270 # differences between profiles in _get_implicit_iuse.
1271 arch_caches = {}
1272 arch_xmatch_caches = {}
1273 shared_xmatch_caches = {"cp-list":{}}
1274
1275 include_arches = None
1276 if options.include_arches:
1277         include_arches = set()
1278         include_arches.update(*[x.split() for x in options.include_arches])
1279
1280 # Disable the "ebuild.notadded" check when not in commit mode and
1281 # running `svn status` in every package dir will be too expensive.
1282
1283 check_ebuild_notadded = not \
1284         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1285
1286 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1287 thirdpartymirrors = {}
1288 for k, v in repoman_settings.thirdpartymirrors().items():
1289         for v in v:
1290                 if not v.endswith("/"):
1291                         v += "/"
1292                 thirdpartymirrors[v] = k
1293
1294 class _XMLParser(xml.etree.ElementTree.XMLParser):
1295
1296         def __init__(self, data, **kwargs):
1297                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1298                 self._portage_data = data
1299                 if hasattr(self, 'parser'):
1300                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1301                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1302                         self._base_StartDoctypeDeclHandler = \
1303                                 self.parser.StartDoctypeDeclHandler
1304                         self.parser.StartDoctypeDeclHandler = \
1305                                 self._portage_StartDoctypeDeclHandler
1306
1307         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1308                 if self._base_XmlDeclHandler is not None:
1309                         self._base_XmlDeclHandler(version, encoding, standalone)
1310                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1311
1312         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1313                 has_internal_subset):
1314                 if self._base_StartDoctypeDeclHandler is not None:
1315                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1316                                 has_internal_subset)
1317                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1318
1319 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1320         """
1321         Implements doctype() as required to avoid deprecation warnings with
1322         >=python-2.7.
1323         """
1324         def doctype(self, name, pubid, system):
1325                 pass
1326
1327 try:
1328         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1329 except (EnvironmentError, ParseError, PermissionDenied) as e:
1330         err(str(e))
1331 except FileNotFound:
1332         # TODO: Download as we do for metadata.dtd, but add a way to
1333         # disable for non-gentoo repoman users who may not have herds.
1334         herd_base = None
1335
1336 effective_scanlist = scanlist
1337 if options.if_modified == "y":
1338         effective_scanlist = sorted(vcs_files_to_cps(
1339                 chain(mychanged, mynew, myremoved)))
1340
1341 for x in effective_scanlist:
1342         #ebuilds and digests added to cvs respectively.
1343         logging.info("checking package %s" % x)
1344         # save memory by discarding xmatch caches from previous package(s)
1345         arch_xmatch_caches.clear()
1346         eadded = []
1347         catdir, pkgdir = x.split("/")
1348         checkdir = repodir + "/" + x
1349         checkdir_relative = ""
1350         if repolevel < 3:
1351                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1352         if repolevel < 2:
1353                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1354         checkdir_relative = os.path.join(".", checkdir_relative)
1355         generated_manifest = False
1356
1357         if options.mode == "manifest" or \
1358           (options.mode != 'manifest-check' and options.digest == 'y') or \
1359           options.mode in ('commit', 'fix') and not options.pretend:
1360                 auto_assumed = set()
1361                 fetchlist_dict = portage.FetchlistDict(checkdir,
1362                         repoman_settings, portdb)
1363                 if options.mode == 'manifest' and options.force:
1364                         portage._doebuild_manifest_exempt_depend += 1
1365                         try:
1366                                 distdir = repoman_settings['DISTDIR']
1367                                 mf = repoman_settings.repositories.get_repo_for_location(
1368                                         os.path.dirname(os.path.dirname(checkdir)))
1369                                 mf = mf.load_manifest(checkdir, distdir,
1370                                         fetchlist_dict=fetchlist_dict)
1371                                 mf.create(requiredDistfiles=None,
1372                                         assumeDistHashesAlways=True)
1373                                 for distfiles in fetchlist_dict.values():
1374                                         for distfile in distfiles:
1375                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1376                                                         mf.fhashdict['DIST'].pop(distfile, None)
1377                                                 else:
1378                                                         auto_assumed.add(distfile)
1379                                 mf.write()
1380                         finally:
1381                                 portage._doebuild_manifest_exempt_depend -= 1
1382
1383                 repoman_settings["O"] = checkdir
1384                 try:
1385                         generated_manifest = digestgen(
1386                                 mysettings=repoman_settings, myportdb=portdb)
1387                 except portage.exception.PermissionDenied as e:
1388                         generated_manifest = False
1389                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1390                                 level=logging.ERROR, noiselevel=-1)
1391
1392                 if not generated_manifest:
1393                         print("Unable to generate manifest.")
1394                         dofail = 1
1395
1396                 if options.mode == "manifest":
1397                         if not dofail and options.force and auto_assumed and \
1398                                 'assume-digests' in repoman_settings.features:
1399                                 # Show which digests were assumed despite the --force option
1400                                 # being given. This output will already have been shown by
1401                                 # digestgen() if assume-digests is not enabled, so only show
1402                                 # it here if assume-digests is enabled.
1403                                 pkgs = list(fetchlist_dict)
1404                                 pkgs.sort()
1405                                 portage.writemsg_stdout("  digest.assumed" + \
1406                                         portage.output.colorize("WARN",
1407                                         str(len(auto_assumed)).rjust(18)) + "\n")
1408                                 for cpv in pkgs:
1409                                         fetchmap = fetchlist_dict[cpv]
1410                                         pf = portage.catsplit(cpv)[1]
1411                                         for distfile in sorted(fetchmap):
1412                                                 if distfile in auto_assumed:
1413                                                         portage.writemsg_stdout(
1414                                                                 "   %s::%s\n" % (pf, distfile))
1415                         continue
1416                 elif dofail:
1417                         sys.exit(1)
1418
1419         if not generated_manifest:
1420                 repoman_settings['O'] = checkdir
1421                 repoman_settings['PORTAGE_QUIET'] = '1'
1422                 if not portage.digestcheck([], repoman_settings, strict=1):
1423                         stats["manifest.bad"] += 1
1424                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1425                 repoman_settings.pop('PORTAGE_QUIET', None)
1426
1427         if options.mode == 'manifest-check':
1428                 continue
1429
1430         checkdirlist = os.listdir(checkdir)
1431         ebuildlist = []
1432         pkgs = {}
1433         allvalid = True
1434         for y in checkdirlist:
1435                 if (y in no_exec or y.endswith(".ebuild")) and \
1436                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1437                                 stats["file.executable"] += 1
1438                                 fails["file.executable"].append(os.path.join(checkdir, y))
1439                 if y.endswith(".ebuild"):
1440                         pf = y[:-7]
1441                         ebuildlist.append(pf)
1442                         cpv = "%s/%s" % (catdir, pf)
1443                         try:
1444                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1445                         except KeyError:
1446                                 allvalid = False
1447                                 stats["ebuild.syntax"] += 1
1448                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1449                                 continue
1450                         except IOError:
1451                                 allvalid = False
1452                                 stats["ebuild.output"] += 1
1453                                 fails["ebuild.output"].append(os.path.join(x, y))
1454                                 continue
1455                         if not portage.eapi_is_supported(myaux["EAPI"]):
1456                                 allvalid = False
1457                                 stats["EAPI.unsupported"] += 1
1458                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1459                                 continue
1460                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1461                                 root_config=root_config, type_name="ebuild")
1462
1463         slot_keywords = {}
1464
1465         if len(pkgs) != len(ebuildlist):
1466                 # If we can't access all the metadata then it's totally unsafe to
1467                 # commit since there's no way to generate a correct Manifest.
1468                 # Do not try to do any more QA checks on this package since missing
1469                 # metadata leads to false positives for several checks, and false
1470                 # positives confuse users.
1471                 can_force = False
1472                 continue
1473
1474         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1475         ebuildlist = sorted(pkgs.values())
1476         ebuildlist = [pkg.pf for pkg in ebuildlist]
1477
1478         for y in checkdirlist:
1479                 index = repo_config.find_invalid_path_char(y)
1480                 if index != -1:
1481                         y_relative = os.path.join(checkdir_relative, y)
1482                         if vcs is not None and not vcs_new_changed(y_relative):
1483                                 # If the file isn't in the VCS new or changed set, then
1484                                 # assume that it's an irrelevant temporary file (Manifest
1485                                 # entries are not generated for file names containing
1486                                 # prohibited characters). See bug #406877.
1487                                 index = -1
1488                 if index != -1:
1489                         stats["file.name"] += 1
1490                         fails["file.name"].append("%s/%s: char '%s'" % \
1491                                 (checkdir, y, y[index]))
1492
1493                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1494                         continue
1495                 f = None
1496                 try:
1497                         line = 1
1498                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1499                                 encoding=_encodings['fs'], errors='strict'),
1500                                 mode='r', encoding=_encodings['repo.content'])
1501                         for l in f:
1502                                 line += 1
1503                 except UnicodeDecodeError as ue:
1504                         stats["file.UTF8"] += 1
1505                         s = ue.object[:ue.start]
1506                         l2 = s.count("\n")
1507                         line += l2
1508                         if l2 != 0:
1509                                 s = s[s.rfind("\n") + 1:]
1510                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1511                 finally:
1512                         if f is not None:
1513                                 f.close()
1514
1515         if vcs in ("git", "hg") and check_ebuild_notadded:
1516                 if vcs == "git":
1517                         myf = repoman_popen("git ls-files --others %s" % \
1518                                 (portage._shell_quote(checkdir_relative),))
1519                 if vcs == "hg":
1520                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1521                                 (portage._shell_quote(checkdir_relative),))
1522                 for l in myf:
1523                         if l[:-1][-7:] == ".ebuild":
1524                                 stats["ebuild.notadded"] += 1
1525                                 fails["ebuild.notadded"].append(
1526                                         os.path.join(x, os.path.basename(l[:-1])))
1527                 myf.close()
1528
1529         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1530                 try:
1531                         if vcs == "cvs":
1532                                 myf = open(checkdir + "/CVS/Entries", "r")
1533                         if vcs == "svn":
1534                                 myf = repoman_popen("svn status --depth=files --verbose " +
1535                                         portage._shell_quote(checkdir))
1536                         if vcs == "bzr":
1537                                 myf = repoman_popen("bzr ls -v --kind=file " +
1538                                         portage._shell_quote(checkdir))
1539                         myl = myf.readlines()
1540                         myf.close()
1541                         for l in myl:
1542                                 if vcs == "cvs":
1543                                         if l[0] != "/":
1544                                                 continue
1545                                         splitl = l[1:].split("/")
1546                                         if not len(splitl):
1547                                                 continue
1548                                         if splitl[0][-7:] == ".ebuild":
1549                                                 eadded.append(splitl[0][:-7])
1550                                 if vcs == "svn":
1551                                         if l[:1] == "?":
1552                                                 continue
1553                                         if l[:7] == '      >':
1554                                                 # tree conflict, new in subversion 1.6
1555                                                 continue
1556                                         l = l.split()[-1]
1557                                         if l[-7:] == ".ebuild":
1558                                                 eadded.append(os.path.basename(l[:-7]))
1559                                 if vcs == "bzr":
1560                                         if l[1:2] == "?":
1561                                                 continue
1562                                         l = l.split()[-1]
1563                                         if l[-7:] == ".ebuild":
1564                                                 eadded.append(os.path.basename(l[:-7]))
1565                         if vcs == "svn":
1566                                 myf = repoman_popen("svn status " +
1567                                         portage._shell_quote(checkdir))
1568                                 myl = myf.readlines()
1569                                 myf.close()
1570                                 for l in myl:
1571                                         if l[0] == "A":
1572                                                 l = l.rstrip().split(' ')[-1]
1573                                                 if l[-7:] == ".ebuild":
1574                                                         eadded.append(os.path.basename(l[:-7]))
1575                 except IOError:
1576                         if vcs == "cvs":
1577                                 stats["CVS/Entries.IO_error"] += 1
1578                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1579                         else:
1580                                 raise
1581                         continue
1582
1583         mf = repoman_settings.repositories.get_repo_for_location(
1584                 os.path.dirname(os.path.dirname(checkdir)))
1585         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1586         mydigests = mf.getTypeDigests("DIST")
1587
1588         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1589         myfiles_all = []
1590         src_uri_error = False
1591         for mykey in fetchlist_dict:
1592                 try:
1593                         myfiles_all.extend(fetchlist_dict[mykey])
1594                 except portage.exception.InvalidDependString as e:
1595                         src_uri_error = True
1596                         try:
1597                                 portdb.aux_get(mykey, ["SRC_URI"])
1598                         except KeyError:
1599                                 # This will be reported as an "ebuild.syntax" error.
1600                                 pass
1601                         else:
1602                                 stats["SRC_URI.syntax"] += 1
1603                                 fails["SRC_URI.syntax"].append(
1604                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1605         del fetchlist_dict
1606         if not src_uri_error:
1607                 # This test can produce false positives if SRC_URI could not
1608                 # be parsed for one or more ebuilds. There's no point in
1609                 # producing a false error here since the root cause will
1610                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1611                 # or "ebuild.sytax".
1612                 myfiles_all = set(myfiles_all)
1613                 for entry in mydigests:
1614                         if entry not in myfiles_all:
1615                                 stats["digest.unused"] += 1
1616                                 fails["digest.unused"].append(checkdir + "::" + entry)
1617                 for entry in myfiles_all:
1618                         if entry not in mydigests:
1619                                 stats["digest.missing"] += 1
1620                                 fails["digest.missing"].append(checkdir + "::" + entry)
1621         del myfiles_all
1622
1623         if os.path.exists(checkdir + "/files"):
1624                 filesdirlist = os.listdir(checkdir + "/files")
1625
1626                 # recurse through files directory
1627                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1628                 while filesdirlist:
1629                         y = filesdirlist.pop(0)
1630                         relative_path = os.path.join(x, "files", y)
1631                         full_path = os.path.join(repodir, relative_path)
1632                         try:
1633                                 mystat = os.stat(full_path)
1634                         except OSError as oe:
1635                                 if oe.errno == 2:
1636                                         # don't worry about it.  it likely was removed via fix above.
1637                                         continue
1638                                 else:
1639                                         raise oe
1640                         if S_ISDIR(mystat.st_mode):
1641                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1642                                 if y == "CVS" or y == ".svn":
1643                                         continue
1644                                 for z in os.listdir(checkdir + "/files/" + y):
1645                                         if z == "CVS" or z == ".svn":
1646                                                 continue
1647                                         filesdirlist.append(y + "/" + z)
1648                         # Current policy is no files over 20 KiB, these are the checks. File size between
1649                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1650                         elif mystat.st_size > 61440:
1651                                 stats["file.size.fatal"] += 1
1652                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1653                         elif mystat.st_size > 20480:
1654                                 stats["file.size"] += 1
1655                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1656
1657                         index = repo_config.find_invalid_path_char(y)
1658                         if index != -1:
1659                                 y_relative = os.path.join(checkdir_relative, "files", y)
1660                                 if vcs is not None and not vcs_new_changed(y_relative):
1661                                         # If the file isn't in the VCS new or changed set, then
1662                                         # assume that it's an irrelevant temporary file (Manifest
1663                                         # entries are not generated for file names containing
1664                                         # prohibited characters). See bug #406877.
1665                                         index = -1
1666                         if index != -1:
1667                                 stats["file.name"] += 1
1668                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1669                                         (checkdir, y, y[index]))
1670         del mydigests
1671
1672         if check_changelog and "ChangeLog" not in checkdirlist:
1673                 stats["changelog.missing"] += 1
1674                 fails["changelog.missing"].append(x + "/ChangeLog")
1675
1676         musedict = {}
1677         # metadata.xml file check
1678         if "metadata.xml" not in checkdirlist:
1679                 stats["metadata.missing"] += 1
1680                 fails["metadata.missing"].append(x + "/metadata.xml")
1681         # metadata.xml parse check
1682         else:
1683                 metadata_bad = False
1684                 xml_info = {}
1685                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1686
1687                 # read metadata.xml into memory
1688                 try:
1689                         _metadata_xml = xml.etree.ElementTree.parse(
1690                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1691                                 encoding=_encodings['fs'], errors='strict'),
1692                                 parser=xml_parser)
1693                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1694                         metadata_bad = True
1695                         stats["metadata.bad"] += 1
1696                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1697                         del e
1698                 else:
1699                         if not hasattr(xml_parser, 'parser') or \
1700                                 sys.hexversion < 0x2070000 or \
1701                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1702                                 # doctype is not parsed with python 2.6 or 3.1
1703                                 pass
1704                         else:
1705                                 if "XML_DECLARATION" not in xml_info:
1706                                         stats["metadata.bad"] += 1
1707                                         fails["metadata.bad"].append("%s/metadata.xml: "
1708                                                 "xml declaration is missing on first line, "
1709                                                 "should be '%s'" % (x, metadata_xml_declaration))
1710                                 else:
1711                                         xml_version, xml_encoding, xml_standalone = \
1712                                                 xml_info["XML_DECLARATION"]
1713                                         if xml_encoding is None or \
1714                                                 xml_encoding.upper() != metadata_xml_encoding:
1715                                                 stats["metadata.bad"] += 1
1716                                                 if xml_encoding is None:
1717                                                         encoding_problem = "but it is undefined"
1718                                                 else:
1719                                                         encoding_problem = "not '%s'" % xml_encoding
1720                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1721                                                         "xml declaration encoding should be '%s', %s" %
1722                                                         (x, metadata_xml_encoding, encoding_problem))
1723
1724                                 if "DOCTYPE" not in xml_info:
1725                                         metadata_bad = True
1726                                         stats["metadata.bad"] += 1
1727                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1728                                                 "DOCTYPE is missing"))
1729                                 else:
1730                                         doctype_name, doctype_system, doctype_pubid = \
1731                                                 xml_info["DOCTYPE"]
1732                                         if doctype_system != metadata_dtd_uri:
1733                                                 stats["metadata.bad"] += 1
1734                                                 if doctype_system is None:
1735                                                         system_problem = "but it is undefined"
1736                                                 else:
1737                                                         system_problem = "not '%s'" % doctype_system
1738                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1739                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1740                                                         (x, metadata_dtd_uri, system_problem))
1741
1742                                         if doctype_name != metadata_doctype_name:
1743                                                 stats["metadata.bad"] += 1
1744                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1745                                                         "DOCTYPE: name should be '%s', not '%s'" %
1746                                                         (x, metadata_doctype_name, doctype_name))
1747
1748                         # load USE flags from metadata.xml
1749                         try:
1750                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1751                         except portage.exception.ParseError as e:
1752                                 metadata_bad = True
1753                                 stats["metadata.bad"] += 1
1754                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1755                         else:
1756                                 for atom in chain(*musedict.values()):
1757                                         if atom is None:
1758                                                 continue
1759                                         try:
1760                                                 atom = Atom(atom)
1761                                         except InvalidAtom as e:
1762                                                 stats["metadata.bad"] += 1
1763                                                 fails["metadata.bad"].append(
1764                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1765                                         else:
1766                                                 if atom.cp != x:
1767                                                         stats["metadata.bad"] += 1
1768                                                         fails["metadata.bad"].append(
1769                                                                 ("%s/metadata.xml: Atom contains "
1770                                                                 "unexpected cat/pn: %s") % (x, atom))
1771
1772                         # Run other metadata.xml checkers
1773                         try:
1774                                 utilities.check_metadata(_metadata_xml, herd_base)
1775                         except (utilities.UnknownHerdsError, ) as e:
1776                                 metadata_bad = True
1777                                 stats["metadata.bad"] += 1
1778                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1779                                 del e
1780
1781                 #Only carry out if in package directory or check forced
1782                 if xmllint_capable and not metadata_bad:
1783                         # xmlint can produce garbage output even on success, so only dump
1784                         # the ouput when it fails.
1785                         st, out = repoman_getstatusoutput(
1786                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1787                                  (portage._shell_quote(metadata_dtd),
1788                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1789                         if st != os.EX_OK:
1790                                 print(red("!!!") + " metadata.xml is invalid:")
1791                                 for z in out.splitlines():
1792                                         print(red("!!! ") + z)
1793                                 stats["metadata.bad"] += 1
1794                                 fails["metadata.bad"].append(x + "/metadata.xml")
1795
1796                 del metadata_bad
1797         muselist = frozenset(musedict)
1798
1799         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1800         changelog_modified = changelog_path in modified_changelogs
1801
1802         # detect unused local USE-descriptions
1803         used_useflags = set()
1804
1805         for y in ebuildlist:
1806                 relative_path = os.path.join(x, y + ".ebuild")
1807                 full_path = os.path.join(repodir, relative_path)
1808                 ebuild_path = y + ".ebuild"
1809                 if repolevel < 3:
1810                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1811                 if repolevel < 2:
1812                         ebuild_path = os.path.join(catdir, ebuild_path)
1813                 ebuild_path = os.path.join(".", ebuild_path)
1814                 if check_changelog and not changelog_modified \
1815                         and ebuild_path in new_ebuilds:
1816                         stats['changelog.ebuildadded'] += 1
1817                         fails['changelog.ebuildadded'].append(relative_path)
1818
1819                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1820                         #ebuild not added to vcs
1821                         stats["ebuild.notadded"] += 1
1822                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1823                 myesplit = portage.pkgsplit(y)
1824                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1825                             or pv_toolong_re.search(myesplit[1]) \
1826                             or pv_toolong_re.search(myesplit[2]):
1827                         stats["ebuild.invalidname"] += 1
1828                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1829                         continue
1830                 elif myesplit[0] != pkgdir:
1831                         print(pkgdir, myesplit[0])
1832                         stats["ebuild.namenomatch"] += 1
1833                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1834                         continue
1835
1836                 pkg = pkgs[y]
1837
1838                 if pkg.invalid:
1839                         allvalid = False
1840                         for k, msgs in pkg.invalid.items():
1841                                 for msg in msgs:
1842                                         stats[k] += 1
1843                                         fails[k].append("%s: %s" % (relative_path, msg))
1844                         continue
1845
1846                 myaux = pkg._metadata
1847                 eapi = myaux["EAPI"]
1848                 inherited = pkg.inherited
1849                 live_ebuild = live_eclasses.intersection(inherited)
1850
1851                 for k, v in myaux.items():
1852                         if not isinstance(v, basestring):
1853                                 continue
1854                         m = non_ascii_re.search(v)
1855                         if m is not None:
1856                                 stats["variable.invalidchar"] += 1
1857                                 fails["variable.invalidchar"].append(
1858                                         ("%s: %s variable contains non-ASCII " + \
1859                                         "character at position %s") % \
1860                                         (relative_path, k, m.start() + 1))
1861
1862                 if not src_uri_error:
1863                         # Check that URIs don't reference a server from thirdpartymirrors.
1864                         for uri in portage.dep.use_reduce( \
1865                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1866                                 contains_mirror = False
1867                                 for mirror, mirror_alias in thirdpartymirrors.items():
1868                                         if uri.startswith(mirror):
1869                                                 contains_mirror = True
1870                                                 break
1871                                 if not contains_mirror:
1872                                         continue
1873
1874                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1875                                 stats["SRC_URI.mirror"] += 1
1876                                 fails["SRC_URI.mirror"].append(
1877                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1878                                         (relative_path, mirror, new_uri))
1879
1880                 if myaux.get("PROVIDE"):
1881                         stats["virtual.oldstyle"] += 1
1882                         fails["virtual.oldstyle"].append(relative_path)
1883
1884                 for pos, missing_var in enumerate(missingvars):
1885                         if not myaux.get(missing_var):
1886                                 if catdir == "virtual" and \
1887                                         missing_var in ("HOMEPAGE", "LICENSE"):
1888                                         continue
1889                                 if live_ebuild and missing_var == "KEYWORDS":
1890                                         continue
1891                                 myqakey = missingvars[pos] + ".missing"
1892                                 stats[myqakey] += 1
1893                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1894
1895                 if catdir == "virtual":
1896                         for var in ("HOMEPAGE", "LICENSE"):
1897                                 if myaux.get(var):
1898                                         myqakey = var + ".virtual"
1899                                         stats[myqakey] += 1
1900                                         fails[myqakey].append(relative_path)
1901
1902                 # 14 is the length of DESCRIPTION=""
1903                 if len(myaux['DESCRIPTION']) > max_desc_len:
1904                         stats['DESCRIPTION.toolong'] += 1
1905                         fails['DESCRIPTION.toolong'].append(
1906                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1907                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1908
1909                 keywords = myaux["KEYWORDS"].split()
1910                 stable_keywords = []
1911                 for keyword in keywords:
1912                         if not keyword.startswith("~") and \
1913                                 not keyword.startswith("-"):
1914                                 stable_keywords.append(keyword)
1915                 if stable_keywords:
1916                         if ebuild_path in new_ebuilds and catdir != "virtual":
1917                                 stable_keywords.sort()
1918                                 stats["KEYWORDS.stable"] += 1
1919                                 fails["KEYWORDS.stable"].append(
1920                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1921                                                 " ".join(stable_keywords))
1922
1923                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1924                         if not kw.startswith("-"))
1925
1926                 previous_keywords = slot_keywords.get(pkg.slot)
1927                 if previous_keywords is None:
1928                         slot_keywords[pkg.slot] = set()
1929                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1930                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1931                         if dropped_keywords:
1932                                 stats["KEYWORDS.dropped"] += 1
1933                                 fails["KEYWORDS.dropped"].append(
1934                                         relative_path + ": %s" % \
1935                                         " ".join(sorted(dropped_keywords)))
1936
1937                 slot_keywords[pkg.slot].update(ebuild_archs)
1938
1939                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1940                 if "-*" in keywords:
1941                         haskeyword = False
1942                         for kw in keywords:
1943                                 if kw[0] == "~":
1944                                         kw = kw[1:]
1945                                 if kw in kwlist:
1946                                         haskeyword = True
1947                         if not haskeyword:
1948                                 stats["KEYWORDS.stupid"] += 1
1949                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1950
1951                 """
1952                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1953                 not be allowed to be marked stable
1954                 """
1955                 if live_ebuild and repo_config.name == "gentoo":
1956                         bad_stable_keywords = []
1957                         for keyword in keywords:
1958                                 if not keyword.startswith("~") and \
1959                                         not keyword.startswith("-"):
1960                                         bad_stable_keywords.append(keyword)
1961                                 del keyword
1962                         if bad_stable_keywords:
1963                                 stats["LIVEVCS.stable"] += 1
1964                                 fails["LIVEVCS.stable"].append(
1965                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1966                                                 bad_stable_keywords)
1967                         del bad_stable_keywords
1968
1969                         if keywords and not has_global_mask(pkg):
1970                                 stats["LIVEVCS.unmasked"] += 1
1971                                 fails["LIVEVCS.unmasked"].append(relative_path)
1972
1973                 if options.ignore_arches:
1974                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1975                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1976                 else:
1977                         arches = set()
1978                         for keyword in keywords:
1979                                 if keyword[0] == "-":
1980                                         continue
1981                                 elif keyword[0] == "~":
1982                                         arch = keyword[1:]
1983                                         if arch == "*":
1984                                                 for expanded_arch in profiles:
1985                                                         if expanded_arch == "**":
1986                                                                 continue
1987                                                         arches.add((keyword, expanded_arch,
1988                                                                 (expanded_arch, "~" + expanded_arch)))
1989                                         else:
1990                                                 arches.add((keyword, arch, (arch, keyword)))
1991                                 else:
1992                                         if keyword == "*":
1993                                                 for expanded_arch in profiles:
1994                                                         if expanded_arch == "**":
1995                                                                 continue
1996                                                         arches.add((keyword, expanded_arch,
1997                                                                 (expanded_arch,)))
1998                                         else:
1999                                                 arches.add((keyword, keyword, (keyword,)))
2000                         if not arches:
2001                                 # Use an empty profile for checking dependencies of
2002                                 # packages that have empty KEYWORDS.
2003                                 arches.add(('**', '**', ('**',)))
2004
2005                 unknown_pkgs = set()
2006                 baddepsyntax = False
2007                 badlicsyntax = False
2008                 badprovsyntax = False
2009                 catpkg = catdir + "/" + y
2010
2011                 inherited_java_eclass = "java-pkg-2" in inherited or \
2012                         "java-pkg-opt-2" in inherited
2013                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
2014                 operator_tokens = set(["||", "(", ")"])
2015                 type_list, badsyntax = [], []
2016                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
2017                         mydepstr = myaux[mytype]
2018
2019                         buildtime = mytype in Package._buildtime_keys
2020                         runtime = mytype in Package._runtime_keys
2021                         token_class = None
2022                         if mytype.endswith("DEPEND"):
2023                                 token_class = portage.dep.Atom
2024
2025                         try:
2026                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
2027                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
2028                         except portage.exception.InvalidDependString as e:
2029                                 atoms = None
2030                                 badsyntax.append(str(e))
2031
2032                         if atoms and mytype.endswith("DEPEND"):
2033                                 if runtime and \
2034                                         "test?" in mydepstr.split():
2035                                         stats[mytype + '.suspect'] += 1
2036                                         fails[mytype + '.suspect'].append(relative_path + \
2037                                                 ": 'test?' USE conditional in %s" % mytype)
2038
2039                                 for atom in atoms:
2040                                         if atom == "||":
2041                                                 continue
2042
2043                                         # Skip dependency.unknown for blockers, so that we
2044                                         # don't encourage people to remove necessary blockers,
2045                                         # as discussed in bug #382407.
2046                                         if atom.blocker is None and \
2047                                                 not portdb.xmatch("match-all", atom) and \
2048                                                 not atom.cp.startswith("virtual/"):
2049                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2050
2051                                         is_blocker = atom.blocker
2052
2053                                         if catdir != "virtual":
2054                                                 if not is_blocker and \
2055                                                         atom.cp in suspect_virtual:
2056                                                         stats['virtual.suspect'] += 1
2057                                                         fails['virtual.suspect'].append(
2058                                                                 relative_path +
2059                                                                 ": %s: consider using '%s' instead of '%s'" %
2060                                                                 (mytype, suspect_virtual[atom.cp], atom))
2061
2062                                         if buildtime and \
2063                                                 not is_blocker and \
2064                                                 not inherited_java_eclass and \
2065                                                 atom.cp == "virtual/jdk":
2066                                                 stats['java.eclassesnotused'] += 1
2067                                                 fails['java.eclassesnotused'].append(relative_path)
2068                                         elif buildtime and \
2069                                                 not is_blocker and \
2070                                                 not inherited_wxwidgets_eclass and \
2071                                                 atom.cp == "x11-libs/wxGTK":
2072                                                 stats['wxwidgets.eclassnotused'] += 1
2073                                                 fails['wxwidgets.eclassnotused'].append(
2074                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2075                                                         " without inheriting wxwidgets.eclass") % mytype)
2076                                         elif runtime:
2077                                                 if not is_blocker and \
2078                                                         atom.cp in suspect_rdepend:
2079                                                         stats[mytype + '.suspect'] += 1
2080                                                         fails[mytype + '.suspect'].append(
2081                                                                 relative_path + ": '%s'" % atom)
2082
2083                                         if atom.operator == "~" and \
2084                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2085                                                 qacat = 'dependency.badtilde'
2086                                                 stats[qacat] += 1
2087                                                 fails[qacat].append(
2088                                                         (relative_path + ": %s uses the ~ operator"
2089                                                          " with a non-zero revision:" + \
2090                                                          " '%s'") % (mytype, atom))
2091
2092                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2093
2094                 for m, b in zip(type_list, badsyntax):
2095                         if m.endswith("DEPEND"):
2096                                 qacat = "dependency.syntax"
2097                         else:
2098                                 qacat = m + ".syntax"
2099                         stats[qacat] += 1
2100                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2101
2102                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2103                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2104                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2105                 badlicsyntax = badlicsyntax > 0
2106                 badprovsyntax = badprovsyntax > 0
2107
2108                 # uselist checks - global
2109                 myuse = []
2110                 default_use = []
2111                 for myflag in myaux["IUSE"].split():
2112                         flag_name = myflag.lstrip("+-")
2113                         used_useflags.add(flag_name)
2114                         if myflag != flag_name:
2115                                 default_use.append(myflag)
2116                         if flag_name not in uselist:
2117                                 myuse.append(flag_name)
2118
2119                 # uselist checks - metadata
2120                 for mypos in range(len(myuse)-1, -1, -1):
2121                         if myuse[mypos] and (myuse[mypos] in muselist):
2122                                 del myuse[mypos]
2123
2124                 if default_use and not eapi_has_iuse_defaults(eapi):
2125                         for myflag in default_use:
2126                                 stats['EAPI.incompatible'] += 1
2127                                 fails['EAPI.incompatible'].append(
2128                                         (relative_path + ": IUSE defaults" + \
2129                                         " not supported with EAPI='%s':" + \
2130                                         " '%s'") % (eapi, myflag))
2131
2132                 for mypos in range(len(myuse)):
2133                         stats["IUSE.invalid"] += 1
2134                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2135
2136                 # license checks
2137                 if not badlicsyntax:
2138                         # Parse the LICENSE variable, remove USE conditions and
2139                         # flatten it.
2140                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2141                         # Check each entry to ensure that it exists in PORTDIR's
2142                         # license directory.
2143                         for lic in licenses:
2144                                 # Need to check for "||" manually as no portage
2145                                 # function will remove it without removing values.
2146                                 if lic not in liclist and lic != "||":
2147                                         stats["LICENSE.invalid"] += 1
2148                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2149                                 elif lic in liclist_deprecated:
2150                                         stats["LICENSE.deprecated"] += 1
2151                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2152
2153                 #keyword checks
2154                 myuse = myaux["KEYWORDS"].split()
2155                 for mykey in myuse:
2156                         if mykey not in ("-*", "*", "~*"):
2157                                 myskey = mykey
2158                                 if myskey[:1] == "-":
2159                                         myskey = myskey[1:]
2160                                 if myskey[:1] == "~":
2161                                         myskey = myskey[1:]
2162                                 if myskey not in kwlist:
2163                                         stats["KEYWORDS.invalid"] += 1
2164                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2165                                 elif myskey not in profiles:
2166                                         stats["KEYWORDS.invalid"] += 1
2167                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2168
2169                 #restrict checks
2170                 myrestrict = None
2171                 try:
2172                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2173                 except portage.exception.InvalidDependString as e:
2174                         stats["RESTRICT.syntax"] += 1
2175                         fails["RESTRICT.syntax"].append(
2176                                 "%s: RESTRICT: %s" % (relative_path, e))
2177                         del e
2178                 if myrestrict:
2179                         myrestrict = set(myrestrict)
2180                         mybadrestrict = myrestrict.difference(valid_restrict)
2181                         if mybadrestrict:
2182                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2183                                 for mybad in mybadrestrict:
2184                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2185                 #REQUIRED_USE check
2186                 required_use = myaux["REQUIRED_USE"]
2187                 if required_use:
2188                         if not eapi_has_required_use(eapi):
2189                                 stats['EAPI.incompatible'] += 1
2190                                 fails['EAPI.incompatible'].append(
2191                                         relative_path + ": REQUIRED_USE" + \
2192                                         " not supported with EAPI='%s'" % (eapi,))
2193                         try:
2194                                 portage.dep.check_required_use(required_use, (),
2195                                         pkg.iuse.is_valid_flag, eapi=eapi)
2196                         except portage.exception.InvalidDependString as e:
2197                                 stats["REQUIRED_USE.syntax"] += 1
2198                                 fails["REQUIRED_USE.syntax"].append(
2199                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2200                                 del e
2201
2202                 # Syntax Checks
2203                 relative_path = os.path.join(x, y + ".ebuild")
2204                 full_path = os.path.join(repodir, relative_path)
2205                 if not vcs_preserves_mtime:
2206                         if ebuild_path not in new_ebuilds and \
2207                                 ebuild_path not in modified_ebuilds:
2208                                 pkg.mtime = None
2209                 try:
2210                         # All ebuilds should have utf_8 encoding.
2211                         f = io.open(_unicode_encode(full_path,
2212                                 encoding=_encodings['fs'], errors='strict'),
2213                                 mode='r', encoding=_encodings['repo.content'])
2214                         try:
2215                                 for check_name, e in run_checks(f, pkg):
2216                                         stats[check_name] += 1
2217                                         fails[check_name].append(relative_path + ': %s' % e)
2218                         finally:
2219                                 f.close()
2220                 except UnicodeDecodeError:
2221                         # A file.UTF8 failure will have already been recorded above.
2222                         pass
2223
2224                 if options.force:
2225                         # The dep_check() calls are the most expensive QA test. If --force
2226                         # is enabled, there's no point in wasting time on these since the
2227                         # user is intent on forcing the commit anyway.
2228                         continue
2229
2230                 relevant_profiles = []
2231                 for keyword, arch, groups in arches:
2232                         if arch not in profiles:
2233                                 # A missing profile will create an error further down
2234                                 # during the KEYWORDS verification.
2235                                 continue
2236
2237                         if include_arches is not None:
2238                                 if arch not in include_arches:
2239                                         continue
2240
2241                         relevant_profiles.extend((keyword, groups, prof)
2242                                 for prof in profiles[arch])
2243
2244                 def sort_key(item):
2245                         return item[2].sub_path
2246
2247                 relevant_profiles.sort(key=sort_key)
2248
2249                 for keyword, groups, prof in relevant_profiles:
2250
2251                                 if prof.status not in ("stable", "dev") or \
2252                                         prof.status == "dev" and not options.include_dev:
2253                                         continue
2254
2255                                 dep_settings = arch_caches.get(prof.sub_path)
2256                                 if dep_settings is None:
2257                                         dep_settings = portage.config(
2258                                                 config_profile_path=prof.abs_path,
2259                                                 config_incrementals=repoman_incrementals,
2260                                                 config_root=config_root,
2261                                                 local_config=False,
2262                                                 _unmatched_removal=options.unmatched_removal,
2263                                                 env=env)
2264                                         dep_settings.categories = repoman_settings.categories
2265                                         if options.without_mask:
2266                                                 dep_settings._mask_manager_obj = \
2267                                                         copy.deepcopy(dep_settings._mask_manager)
2268                                                 dep_settings._mask_manager._pmaskdict.clear()
2269                                         arch_caches[prof.sub_path] = dep_settings
2270
2271                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2272                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2273                                 if xcache is None:
2274                                         portdb.melt()
2275                                         portdb.freeze()
2276                                         xcache = portdb.xcache
2277                                         xcache.update(shared_xmatch_caches)
2278                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2279
2280                                 trees[root]["porttree"].settings = dep_settings
2281                                 portdb.settings = dep_settings
2282                                 portdb.xcache = xcache
2283
2284                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2285                                 # just in case, prevent config.reset() from nuking these.
2286                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2287
2288                                 # This attribute is used in dbapi._match_use() to apply
2289                                 # use.stable.{mask,force} settings based on the stable
2290                                 # status of the parent package. This is required in order
2291                                 # for USE deps of unstable packages to be resolved correctly,
2292                                 # since otherwise use.stable.{mask,force} settings of
2293                                 # dependencies may conflict (see bug #456342).
2294                                 dep_settings._parent_stable = dep_settings._isStable(pkg)
2295
2296                                 # Handle package.use*.{force,mask) calculation, for use
2297                                 # in dep_check.
2298                                 dep_settings.useforce = dep_settings._use_manager.getUseForce(
2299                                         pkg, stable=dep_settings._parent_stable)
2300                                 dep_settings.usemask = dep_settings._use_manager.getUseMask(
2301                                         pkg, stable=dep_settings._parent_stable)
2302
2303                                 if not baddepsyntax:
2304                                         ismasked = not ebuild_archs or \
2305                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2306                                         if ismasked:
2307                                                 if not have_pmasked:
2308                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2309                                                                 pkg.cpv, pkg._metadata))
2310                                                 if options.ignore_masked:
2311                                                         continue
2312                                                 #we are testing deps for a masked package; give it some lee-way
2313                                                 suffix = "masked"
2314                                                 matchmode = "minimum-all"
2315                                         else:
2316                                                 suffix = ""
2317                                                 matchmode = "minimum-visible"
2318
2319                                         if not have_dev_keywords:
2320                                                 have_dev_keywords = \
2321                                                         bool(dev_keywords.intersection(keywords))
2322
2323                                         if prof.status == "dev":
2324                                                 suffix = suffix + "indev"
2325
2326                                         for mytype in Package._dep_keys:
2327
2328                                                 mykey = "dependency.bad" + suffix
2329                                                 myvalue = myaux[mytype]
2330                                                 if not myvalue:
2331                                                         continue
2332
2333                                                 success, atoms = portage.dep_check(myvalue, portdb,
2334                                                         dep_settings, use="all", mode=matchmode,
2335                                                         trees=trees)
2336
2337                                                 if success:
2338                                                         if atoms:
2339
2340                                                                 # Don't bother with dependency.unknown for
2341                                                                 # cases in which *DEPEND.bad is triggered.
2342                                                                 for atom in atoms:
2343                                                                         # dep_check returns all blockers and they
2344                                                                         # aren't counted for *DEPEND.bad, so we
2345                                                                         # ignore them here.
2346                                                                         if not atom.blocker:
2347                                                                                 unknown_pkgs.discard(
2348                                                                                         (mytype, atom.unevaluated_atom))
2349
2350                                                                 if not prof.sub_path:
2351                                                                         # old-style virtuals currently aren't
2352                                                                         # resolvable with empty profile, since
2353                                                                         # 'virtuals' mappings are unavailable
2354                                                                         # (it would be expensive to search
2355                                                                         # for PROVIDE in all ebuilds)
2356                                                                         atoms = [atom for atom in atoms if not \
2357                                                                                 (atom.cp.startswith('virtual/') and \
2358                                                                                 not portdb.cp_list(atom.cp))]
2359
2360                                                                 #we have some unsolvable deps
2361                                                                 #remove ! deps, which always show up as unsatisfiable
2362                                                                 atoms = [str(atom.unevaluated_atom) \
2363                                                                         for atom in atoms if not atom.blocker]
2364
2365                                                                 #if we emptied out our list, continue:
2366                                                                 if not atoms:
2367                                                                         continue
2368                                                                 stats[mykey] += 1
2369                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2370                                                                         (relative_path, mytype, keyword,
2371                                                                         prof, repr(atoms)))
2372                                                 else:
2373                                                         stats[mykey] += 1
2374                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2375                                                                 (relative_path, mytype, keyword,
2376                                                                 prof, repr(atoms)))
2377
2378                 if not baddepsyntax and unknown_pkgs:
2379                         type_map = {}
2380                         for mytype, atom in unknown_pkgs:
2381                                 type_map.setdefault(mytype, set()).add(atom)
2382                         for mytype, atoms in type_map.items():
2383                                 stats["dependency.unknown"] += 1
2384                                 fails["dependency.unknown"].append("%s: %s: %s" %
2385                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2386
2387         # check if there are unused local USE-descriptions in metadata.xml
2388         # (unless there are any invalids, to avoid noise)
2389         if allvalid:
2390                 for myflag in muselist.difference(used_useflags):
2391                         stats["metadata.warning"] += 1
2392                         fails["metadata.warning"].append(
2393                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2394                                 (x, myflag))
2395
2396 if options.if_modified == "y" and len(effective_scanlist) < 1:
2397         logging.warn("--if-modified is enabled, but no modified packages were found!")
2398
2399 if options.mode == "manifest":
2400         sys.exit(dofail)
2401
2402 # dofail will be set to 1 if we have failed in at least one non-warning category
2403 dofail = 0
2404 # dowarn will be set to 1 if we tripped any warnings
2405 dowarn = 0
2406 # dofull will be set if we should print a "repoman full" informational message
2407 dofull = options.mode != 'full'
2408
2409 for x in qacats:
2410         if not stats[x]:
2411                 continue
2412         dowarn = 1
2413         if x not in qawarnings:
2414                 dofail = 1
2415
2416 if dofail or \
2417         (dowarn and not (options.quiet or options.mode == "scan")):
2418         dofull = 0
2419
2420 # Save QA output so that it can be conveniently displayed
2421 # in $EDITOR while the user creates a commit message.
2422 # Otherwise, the user would not be able to see this output
2423 # once the editor has taken over the screen.
2424 qa_output = io.StringIO()
2425 style_file = ConsoleStyleFile(sys.stdout)
2426 if options.mode == 'commit' and \
2427         (not commitmessage or not commitmessage.strip()):
2428         style_file.write_listener = qa_output
2429 console_writer = StyleWriter(file=style_file, maxcol=9999)
2430 console_writer.style_listener = style_file.new_styles
2431
2432 f = formatter.AbstractFormatter(console_writer)
2433
2434 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2435
2436 style_file.flush()
2437 del console_writer, f, style_file
2438 qa_output = qa_output.getvalue()
2439 qa_output = qa_output.splitlines(True)
2440
2441 suggest_ignore_masked = False
2442 suggest_include_dev = False
2443
2444 if have_pmasked and not (options.without_mask or options.ignore_masked):
2445         suggest_ignore_masked = True
2446 if have_dev_keywords and not options.include_dev:
2447         suggest_include_dev = True
2448
2449 if suggest_ignore_masked or suggest_include_dev:
2450         print()
2451         if suggest_ignore_masked:
2452                 print(bold("Note: use --without-mask to check " + \
2453                         "KEYWORDS on dependencies of masked packages"))
2454
2455         if suggest_include_dev:
2456                 print(bold("Note: use --include-dev (-d) to check " + \
2457                         "dependencies for 'dev' profiles"))
2458         print()
2459
2460 if options.mode != 'commit':
2461         if dofull:
2462                 print(bold("Note: type \"repoman full\" for a complete listing."))
2463         if dowarn and not dofail:
2464                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2465         elif not dofail:
2466                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2467         elif dofail:
2468                 print(bad("Please fix these important QA issues first."))
2469                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2470                 sys.exit(1)
2471 else:
2472         if dofail and can_force and options.force and not options.pretend:
2473                 print(green("RepoMan sez:") + \
2474                         " \"You want to commit even with these QA issues?\n" + \
2475                         "              I'll take it this time, but I'm not happy.\"\n")
2476         elif dofail:
2477                 if options.force and not can_force:
2478                         print(bad("The --force option has been disabled due to extraordinary issues."))
2479                 print(bad("Please fix these important QA issues first."))
2480                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2481                 sys.exit(1)
2482
2483         if options.pretend:
2484                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2485
2486         myunadded = []
2487         if vcs == "cvs":
2488                 try:
2489                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2490                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2491                 except SystemExit as e:
2492                         raise  # TODO propagate this
2493                 except:
2494                         err("Error retrieving CVS tree; exiting.")
2495         if vcs == "svn":
2496                 try:
2497                         with repoman_popen("svn status --no-ignore") as f:
2498                                 svnstatus = f.readlines()
2499                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2500                 except SystemExit as e:
2501                         raise  # TODO propagate this
2502                 except:
2503                         err("Error retrieving SVN info; exiting.")
2504         if vcs == "git":
2505                 # get list of files not under version control or missing
2506                 myf = repoman_popen("git ls-files --others")
2507                 myunadded = ["./" + elem[:-1] for elem in myf]
2508                 myf.close()
2509         if vcs == "bzr":
2510                 try:
2511                         with repoman_popen("bzr status -S .") as f:
2512                                 bzrstatus = f.readlines()
2513                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2514                 except SystemExit as e:
2515                         raise  # TODO propagate this
2516                 except:
2517                         err("Error retrieving bzr info; exiting.")
2518         if vcs == "hg":
2519                 with repoman_popen("hg status --no-status --unknown .") as f:
2520                         myunadded = f.readlines()
2521                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2522
2523                 # Mercurial doesn't handle manually deleted files as removed from
2524                 # the repository, so the user need to remove them before commit,
2525                 # using "hg remove [FILES]"
2526                 with repoman_popen("hg status --no-status --deleted .") as f:
2527                         mydeleted = f.readlines()
2528                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2529
2530
2531         myautoadd = []
2532         if myunadded:
2533                 for x in range(len(myunadded)-1, -1, -1):
2534                         xs = myunadded[x].split("/")
2535                         if xs[-1] == "files":
2536                                 print("!!! files dir is not added! Please correct this.")
2537                                 sys.exit(-1)
2538                         elif xs[-1] == "Manifest":
2539                                 # It's a manifest... auto add
2540                                 myautoadd += [myunadded[x]]
2541                                 del myunadded[x]
2542
2543         if myunadded:
2544                 print(red("!!! The following files are in your local tree but are not added to the master"))
2545                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2546                 for x in myunadded:
2547                         print("   ", x)
2548                 print()
2549                 print()
2550                 sys.exit(1)
2551
2552         if vcs == "hg" and mydeleted:
2553                 print(red("!!! The following files are removed manually from your local tree but are not"))
2554                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2555                 for x in mydeleted:
2556                         print("   ", x)
2557                 print()
2558                 print()
2559                 sys.exit(1)
2560
2561         if vcs == "cvs":
2562                 mycvstree = cvstree.getentries("./", recursive=1)
2563                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2564                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2565                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2566                 bin_blob_pattern = re.compile("^-kb$")
2567                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2568                         recursive=1, basedir="./"))
2569
2570         if vcs == "svn":
2571                 with repoman_popen("svn status") as f:
2572                         svnstatus = f.readlines()
2573                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2574                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2575                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2576
2577                 # Subversion expands keywords specified in svn:keywords properties.
2578                 with repoman_popen("svn propget -R svn:keywords") as f:
2579                         props = f.readlines()
2580                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2581                         for prop in props if " - " in prop)
2582
2583         elif vcs == "git":
2584                 with repoman_popen("git diff-index --name-only "
2585                         "--relative --diff-filter=M HEAD") as f:
2586                         mychanged = f.readlines()
2587                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2588
2589                 with repoman_popen("git diff-index --name-only "
2590                         "--relative --diff-filter=A HEAD") as f:
2591                         mynew = f.readlines()
2592                 mynew = ["./" + elem[:-1] for elem in mynew]
2593
2594                 with repoman_popen("git diff-index --name-only "
2595                         "--relative --diff-filter=D HEAD") as f:
2596                         myremoved = f.readlines()
2597                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2598
2599         if vcs == "bzr":
2600                 with repoman_popen("bzr status -S .") as f:
2601                         bzrstatus = f.readlines()
2602                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2603                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2604                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2605                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2606                 # Bazaar expands nothing.
2607
2608         if vcs == "hg":
2609                 with repoman_popen("hg status --no-status --modified .") as f:
2610                         mychanged = f.readlines()
2611                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2612
2613                 with repoman_popen("hg status --no-status --added .") as f:
2614                         mynew = f.readlines()
2615                 mynew = ["./" + elem.rstrip() for elem in mynew]
2616
2617                 with repoman_popen("hg status --no-status --removed .") as f:
2618                         myremoved = f.readlines()
2619                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2620
2621         if vcs:
2622                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2623                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2624                         print()
2625                         print("(Didn't find any changed files...)")
2626                         print()
2627                         sys.exit(1)
2628
2629         # Manifests need to be regenerated after all other commits, so don't commit
2630         # them now even if they have changed.
2631         mymanifests = set()
2632         myupdates = set()
2633         for f in mychanged + mynew:
2634                 if "Manifest" == os.path.basename(f):
2635                         mymanifests.add(f)
2636                 else:
2637                         myupdates.add(f)
2638         myupdates.difference_update(myremoved)
2639         myupdates = list(myupdates)
2640         mymanifests = list(mymanifests)
2641         myheaders = []
2642         mydirty = []
2643
2644         commitmessage = options.commitmsg
2645         if options.commitmsgfile:
2646                 try:
2647                         f = io.open(_unicode_encode(options.commitmsgfile,
2648                         encoding=_encodings['fs'], errors='strict'),
2649                         mode='r', encoding=_encodings['content'], errors='replace')
2650                         commitmessage = f.read()
2651                         f.close()
2652                         del f
2653                 except (IOError, OSError) as e:
2654                         if e.errno == errno.ENOENT:
2655                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2656                         else:
2657                                 raise
2658                 # We've read the content so the file is no longer needed.
2659                 commitmessagefile = None
2660         if not commitmessage or not commitmessage.strip():
2661                 try:
2662                         editor = os.environ.get("EDITOR")
2663                         if editor and utilities.editor_is_executable(editor):
2664                                 commitmessage = utilities.get_commit_message_with_editor(
2665                                         editor, message=qa_output)
2666                         else:
2667                                 commitmessage = utilities.get_commit_message_with_stdin()
2668                 except KeyboardInterrupt:
2669                         exithandler()
2670                 if not commitmessage or not commitmessage.strip():
2671                         print("* no commit message?  aborting commit.")
2672                         sys.exit(1)
2673         commitmessage = commitmessage.rstrip()
2674         changelog_msg = commitmessage
2675         portage_version = getattr(portage, "VERSION", None)
2676         gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
2677         dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
2678         if portage_version is None:
2679                 sys.stderr.write("Failed to insert portage version in message!\n")
2680                 sys.stderr.flush()
2681                 portage_version = "Unknown"
2682
2683         report_options = []
2684         if options.force:
2685                 report_options.append("--force")
2686         if options.ignore_arches:
2687                 report_options.append("--ignore-arches")
2688         if include_arches is not None:
2689                 report_options.append("--include-arches=\"%s\"" %
2690                         " ".join(sorted(include_arches)))
2691
2692         if vcs == "git":
2693                 # Use new footer only for git (see bug #438364).
2694                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2695                 if report_options:
2696                         commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
2697                 if sign_manifests:
2698                         commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
2699                 if dco_sob:
2700                         commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
2701         else:
2702                 unameout = platform.system() + " "
2703                 if platform.system() in ["Darwin", "SunOS"]:
2704                         unameout += platform.processor()
2705                 else:
2706                         unameout += platform.machine()
2707                 commit_footer = "\n\n"
2708                 if dco_sob:
2709                         commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
2710                 commit_footer += "(Portage version: %s/%s/%s" % \
2711                         (portage_version, vcs, unameout)
2712                 if report_options:
2713                         commit_footer += ", RepoMan options: " + " ".join(report_options)
2714                 if sign_manifests:
2715                         commit_footer += ", signed Manifest commit with key %s" % \
2716                                 (gpg_key, )
2717                 else:
2718                         commit_footer += ", unsigned Manifest commit"
2719                 commit_footer += ")"
2720
2721         commitmessage += commit_footer
2722
2723         if options.echangelog in ('y', 'force'):
2724                 logging.info("checking for unmodified ChangeLog files")
2725                 committer_name = utilities.get_committer_name(env=repoman_settings)
2726                 for x in sorted(vcs_files_to_cps(
2727                         chain(myupdates, mymanifests, myremoved))):
2728                         catdir, pkgdir = x.split("/")
2729                         checkdir = repodir + "/" + x
2730                         checkdir_relative = ""
2731                         if repolevel < 3:
2732                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2733                         if repolevel < 2:
2734                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2735                         checkdir_relative = os.path.join(".", checkdir_relative)
2736
2737                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2738                         changelog_modified = changelog_path in modified_changelogs
2739                         if changelog_modified and options.echangelog != 'force':
2740                                 continue
2741
2742                         # get changes for this package
2743                         cdrlen = len(checkdir_relative)
2744                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2745                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2746                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2747
2748                         # Skip ChangeLog generation if only the Manifest was modified,
2749                         # as discussed in bug #398009.
2750                         nontrivial_cl_files = set()
2751                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2752                         nontrivial_cl_files.difference_update(['Manifest'])
2753                         if not nontrivial_cl_files and options.echangelog != 'force':
2754                                 continue
2755
2756                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2757                                 committer_name, changelog_msg,
2758                                 os.path.join(repodir, 'skel.ChangeLog'),
2759                                 catdir, pkgdir,
2760                                 new=clnew, removed=clremoved, changed=clchanged,
2761                                 pretend=options.pretend)
2762                         if new_changelog is None:
2763                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2764                                         level=logging.ERROR, noiselevel=-1)
2765                                 sys.exit(1)
2766
2767                         # if the ChangeLog was just created, add it to vcs
2768                         if new_changelog:
2769                                 myautoadd.append(changelog_path)
2770                                 # myautoadd is appended to myupdates below
2771                         else:
2772                                 myupdates.append(changelog_path)
2773
2774                         if options.ask and not options.pretend:
2775                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2776                                 repoman_settings["O"] = checkdir
2777                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2778
2779         if myautoadd:
2780                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2781                 add_cmd = [vcs, "add"]
2782                 add_cmd += myautoadd
2783                 if options.pretend:
2784                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2785                                 noiselevel=-1)
2786                 else:
2787                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2788                                 # Python 3.1 produces the following TypeError if raw bytes are
2789                                 # passed to subprocess.call():
2790                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2791                                 #     errread, errwrite)
2792                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2793                                 #     raise child_exception
2794                                 # TypeError: expected an object with the buffer interface
2795                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2796                         retcode = subprocess.call(add_cmd)
2797                         if retcode != os.EX_OK:
2798                                 logging.error(
2799                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2800                                 sys.exit(retcode)
2801
2802                 myupdates += myautoadd
2803
2804         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2805
2806         if vcs not in ('cvs', 'svn'):
2807                 # With git, bzr and hg, there's never any keyword expansion, so
2808                 # there's no need to regenerate manifests and all files will be
2809                 # committed in one big commit at the end.
2810                 print()
2811         elif not repo_config.thin_manifest:
2812                 if vcs == 'cvs':
2813                         headerstring = "'\$(Header|Id).*\$'"
2814                 elif vcs == "svn":
2815                         svn_keywords = dict((k.lower(), k) for k in [
2816                                         "Rev",
2817                                         "Revision",
2818                                         "LastChangedRevision",
2819                                         "Date",
2820                                         "LastChangedDate",
2821                                         "Author",
2822                                         "LastChangedBy",
2823                                         "URL",
2824                                         "HeadURL",
2825                                         "Id",
2826                                         "Header",
2827                         ])
2828
2829                 for myfile in myupdates:
2830
2831                         # for CVS, no_expansion contains files that are excluded from expansion
2832                         if vcs == "cvs":
2833                                 if myfile in no_expansion:
2834                                         continue
2835
2836                         # for SVN, expansion contains files that are included in expansion
2837                         elif vcs == "svn":
2838                                 if myfile not in expansion:
2839                                         continue
2840
2841                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2842                                 enabled_keywords = []
2843                                 for k in expansion[myfile]:
2844                                         keyword = svn_keywords.get(k.lower())
2845                                         if keyword is not None:
2846                                                 enabled_keywords.append(keyword)
2847
2848                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2849
2850                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2851                                 portage._shell_quote(myfile))
2852                         if myout[0] == 0:
2853                                 myheaders.append(myfile)
2854
2855                 print("%s have headers that will change." % green(str(len(myheaders))))
2856                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2857
2858         logging.info("myupdates: %s", myupdates)
2859         logging.info("myheaders: %s", myheaders)
2860
2861         if options.ask and userquery('Commit changes?', True) != 'Yes':
2862                 print("* aborting commit.")
2863                 sys.exit(128 + signal.SIGINT)
2864
2865         # Handle the case where committed files have keywords which
2866         # will change and need a priming commit before the Manifest
2867         # can be committed.
2868         if (myupdates or myremoved) and myheaders:
2869                 myfiles = myupdates + myremoved
2870                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2871                 mymsg = os.fdopen(fd, "wb")
2872                 mymsg.write(_unicode_encode(commitmessage))
2873                 mymsg.close()
2874
2875                 print()
2876                 print(green("Using commit message:"))
2877                 print(green("------------------------------------------------------------------------------"))
2878                 print(commitmessage)
2879                 print(green("------------------------------------------------------------------------------"))
2880                 print()
2881
2882                 # Having a leading ./ prefix on file paths can trigger a bug in
2883                 # the cvs server when committing files to multiple directories,
2884                 # so strip the prefix.
2885                 myfiles = [f.lstrip("./") for f in myfiles]
2886
2887                 commit_cmd = [vcs]
2888                 commit_cmd.extend(vcs_global_opts)
2889                 commit_cmd.append("commit")
2890                 commit_cmd.extend(vcs_local_opts)
2891                 commit_cmd.extend(["-F", commitmessagefile])
2892                 commit_cmd.extend(myfiles)
2893
2894                 try:
2895                         if options.pretend:
2896                                 print("(%s)" % (" ".join(commit_cmd),))
2897                         else:
2898                                 retval = spawn(commit_cmd, env=commit_env)
2899                                 if retval != os.EX_OK:
2900                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2901                                                 "error code: %s\n") % (vcs, retval),
2902                                                 level=logging.ERROR, noiselevel=-1)
2903                                         sys.exit(retval)
2904                 finally:
2905                         try:
2906                                 os.unlink(commitmessagefile)
2907                         except OSError:
2908                                 pass
2909
2910         # Setup the GPG commands
2911         def gpgsign(filename):
2912                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2913                 if gpgcmd is None:
2914                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2915                                 " Is make.globals missing?")
2916                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2917                         "PORTAGE_GPG_KEY" not in repoman_settings:
2918                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2919                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2920                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2921                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2922                                         os.path.expanduser("~/.gnupg")
2923                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2924                                         % repoman_settings["PORTAGE_GPG_DIR"])
2925                         else:
2926                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2927                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2928                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2929                                 raise portage.exception.InvalidLocation(
2930                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2931                                         repoman_settings["PORTAGE_GPG_DIR"])
2932                 gpgvars = {"FILE": filename}
2933                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2934                         v = repoman_settings.get(k)
2935                         if v is not None:
2936                                 gpgvars[k] = v
2937                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2938                 if options.pretend:
2939                         print("(" + gpgcmd + ")")
2940                 else:
2941                         # Encode unicode manually for bug #310789.
2942                         gpgcmd = portage.util.shlex_split(gpgcmd)
2943                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2944                                 # Python 3.1 does not support bytes in Popen args.
2945                                 gpgcmd = [_unicode_encode(arg,
2946                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2947                         rValue = subprocess.call(gpgcmd)
2948                         if rValue == os.EX_OK:
2949                                 os.rename(filename + ".asc", filename)
2950                         else:
2951                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2952
2953         def need_signature(filename):
2954                 try:
2955                         with open(_unicode_encode(filename,
2956                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2957                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2958                 except IOError as e:
2959                         if e.errno in (errno.ENOENT, errno.ESTALE):
2960                                 return False
2961                         raise
2962
2963         # When files are removed and re-added, the cvs server will put /Attic/
2964         # inside the $Header path. This code detects the problem and corrects it
2965         # so that the Manifest will generate correctly. See bug #169500.
2966         # Use binary mode in order to avoid potential character encoding issues.
2967         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2968         attic_str = b'/Attic/'
2969         attic_replace = b'/'
2970         for x in myheaders:
2971                 f = open(_unicode_encode(x,
2972                         encoding=_encodings['fs'], errors='strict'),
2973                         mode='rb')
2974                 mylines = f.readlines()
2975                 f.close()
2976                 modified = False
2977                 for i, line in enumerate(mylines):
2978                         if cvs_header_re.match(line) is not None and \
2979                                 attic_str in line:
2980                                 mylines[i] = line.replace(attic_str, attic_replace)
2981                                 modified = True
2982                 if modified:
2983                         portage.util.write_atomic(x, b''.join(mylines),
2984                                 mode='wb')
2985
2986         if repolevel == 1:
2987                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2988                         "doing the entire repository.\"\n")
2989
2990         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2991
2992                 for x in sorted(vcs_files_to_cps(
2993                         chain(myupdates, myremoved, mymanifests))):
2994                         repoman_settings["O"] = os.path.join(repodir, x)
2995                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2996
2997         signed = False
2998         if sign_manifests:
2999                 signed = True
3000                 try:
3001                         for x in sorted(vcs_files_to_cps(
3002                                 chain(myupdates, myremoved, mymanifests))):
3003                                 repoman_settings["O"] = os.path.join(repodir, x)
3004                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
3005                                 if not need_signature(manifest_path):
3006                                         continue
3007                                 gpgsign(manifest_path)
3008                 except portage.exception.PortageException as e:
3009                         portage.writemsg("!!! %s\n" % str(e))
3010                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
3011                         signed = False
3012
3013         if vcs == 'git':
3014                 # It's not safe to use the git commit -a option since there might
3015                 # be some modified files elsewhere in the working tree that the
3016                 # user doesn't want to commit. Therefore, call git update-index
3017                 # in order to ensure that the index is updated with the latest
3018                 # versions of all new and modified files in the relevant portion
3019                 # of the working tree.
3020                 myfiles = mymanifests + myupdates
3021                 myfiles.sort()
3022                 update_index_cmd = ["git", "update-index"]
3023                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
3024                 if options.pretend:
3025                         print("(%s)" % (" ".join(update_index_cmd),))
3026                 else:
3027                         retval = spawn(update_index_cmd, env=os.environ)
3028                         if retval != os.EX_OK:
3029                                 writemsg_level(("!!! Exiting on %s (shell) " + \
3030                                         "error code: %s\n") % (vcs, retval),
3031                                         level=logging.ERROR, noiselevel=-1)
3032                                 sys.exit(retval)
3033
3034         if True:
3035                 myfiles = mymanifests[:]
3036                 # If there are no header (SVN/CVS keywords) changes in
3037                 # the files, this Manifest commit must include the
3038                 # other (yet uncommitted) files.
3039                 if not myheaders:
3040                         myfiles += myupdates
3041                         myfiles += myremoved
3042                 myfiles.sort()
3043
3044                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3045                 mymsg = os.fdopen(fd, "wb")
3046                 mymsg.write(_unicode_encode(commitmessage))
3047                 mymsg.close()
3048
3049                 commit_cmd = []
3050                 if options.pretend and vcs is None:
3051                         # substitute a bogus value for pretend output
3052                         commit_cmd.append("cvs")
3053                 else:
3054                         commit_cmd.append(vcs)
3055                 commit_cmd.extend(vcs_global_opts)
3056                 commit_cmd.append("commit")
3057                 commit_cmd.extend(vcs_local_opts)
3058                 if vcs == "hg":
3059                         commit_cmd.extend(["--logfile", commitmessagefile])
3060                         commit_cmd.extend(myfiles)
3061                 else:
3062                         commit_cmd.extend(["-F", commitmessagefile])
3063                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3064
3065                 try:
3066                         if options.pretend:
3067                                 print("(%s)" % (" ".join(commit_cmd),))
3068                         else:
3069                                 retval = spawn(commit_cmd, env=commit_env)
3070                                 if retval != os.EX_OK:
3071                                         if repo_config.sign_commit and vcs == 'git' and \
3072                                                 not git_supports_gpg_sign():
3073                                                 # Inform user that newer git is needed (bug #403323).
3074                                                 logging.error(
3075                                                         "Git >=1.7.9 is required for signed commits!")
3076
3077                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3078                                                 "error code: %s\n") % (vcs, retval),
3079                                                 level=logging.ERROR, noiselevel=-1)
3080                                         sys.exit(retval)
3081                 finally:
3082                         try:
3083                                 os.unlink(commitmessagefile)
3084                         except OSError:
3085                                 pass
3086
3087         print()
3088         if vcs:
3089                 print("Commit complete.")
3090         else:
3091                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3092         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3093 sys.exit(0)