bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import io
  17 import logging
  18 import optparse
  19 import re
  20 import signal
  21 import stat
  22 import subprocess
  23 import sys
  24 import tempfile
  25 import textwrap
  26 import time
  27 import platform
  28
  29 try:
  30         from urllib.request import urlopen as urllib_request_urlopen
  31 except ImportError:
  32         from urllib import urlopen as urllib_request_urlopen
  33
  34 from itertools import chain
  35 from stat import S_ISDIR
  36
  37 from os import path as osp
  38 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  39 sys.path.insert(0, pym_path)
  40 import portage
  41 portage._internal_caller = True
  42 portage._disable_legacy_globals()
  43
  44 try:
  45         import xml.etree.ElementTree
  46         from xml.parsers.expat import ExpatError
  47 except (SystemExit, KeyboardInterrupt):
  48         raise
  49 except (ImportError, SystemError, RuntimeError, Exception):
  50         # broken or missing xml support
  51         # http://bugs.python.org/issue14988
  52         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  53         from portage.output import EOutput
  54         out = EOutput()
  55         for line in msg:
  56                 out.eerror(line)
  57         sys.exit(1)
  58
  59 from portage import os
  60 from portage import _encodings
  61 from portage import _unicode_encode
  62 from repoman.checks import run_checks
  63 from repoman import utilities
  64 from repoman.herdbase import make_herd_base
  65 from _emerge.Package import Package
  66 from _emerge.RootConfig import RootConfig
  67 from _emerge.userquery import userquery
  68 import portage.checksum
  69 import portage.const
  70 from portage import cvstree, normalize_path
  71 from portage import util
  72 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  73         ParseError, PermissionDenied)
  74 from portage.dep import Atom
  75 from portage.process import find_binary, spawn
  76 from portage.output import bold, create_color_func, \
  77         green, nocolor, red
  78 from portage.output import ConsoleStyleFile, StyleWriter
  79 from portage.util import writemsg_level
  80 from portage.package.ebuild.digestgen import digestgen
  81 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  82
  83 if sys.hexversion >= 0x3000000:
  84         basestring = str
  85
  86 util.initialize_logger()
  87
  88 # 14 is the length of DESCRIPTION=""
  89 max_desc_len = 100
  90 allowed_filename_chars="a-zA-Z0-9._-+:"
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  93 bad = create_color_func("BAD")
  94
  95 # A sane umask is needed for files that portage creates.
  96 os.umask(0o22)
  97 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  98 # behave incrementally.
  99 repoman_incrementals = tuple(x for x in \
 100         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 101 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 102 repoman_settings = portage.config(config_root=config_root, local_config=False)
 103
 104 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 105         repoman_settings.get('TERM') == 'dumb' or \
 106         not sys.stdout.isatty():
 107         nocolor()
 108
 109 def warn(txt):
 110         print("repoman: " + txt)
 111
 112 def err(txt):
 113         warn(txt)
 114         sys.exit(1)
 115
 116 def exithandler(signum=None, frame=None):
 117         logging.fatal("Interrupted; exiting...")
 118         if signum is None:
 119                 sys.exit(1)
 120         else:
 121                 sys.exit(128 + signum)
 122
 123 signal.signal(signal.SIGINT,exithandler)
 124
 125 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 126         """Repoman needs it's own HelpFormatter for now, because the default ones
 127         murder the help text."""
 128
 129         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 130                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 131
 132         def format_description(self, description):
 133                 return description
 134
 135 class RepomanOptionParser(optparse.OptionParser):
 136         """Add the on_tail function, ruby has it, optionParser should too
 137         """
 138
 139         def __init__(self, *args, **kwargs):
 140                 optparse.OptionParser.__init__(self, *args, **kwargs)
 141                 self.tail = ""
 142
 143         def on_tail(self, description):
 144                 self.tail += description
 145
 146         def format_help(self, formatter=None):
 147                 result = optparse.OptionParser.format_help(self, formatter)
 148                 result += self.tail
 149                 return result
 150
 151
 152 def ParseArgs(argv, qahelp):
 153         """This function uses a customized optionParser to parse command line arguments for repoman
 154         Args:
 155           argv - a sequence of command line arguments
 156                 qahelp - a dict of qa warning to help message
 157         Returns:
 158           (opts, args), just like a call to parser.parse_args()
 159         """
 160
 161         if argv and isinstance(argv[0], bytes):
 162                 argv = [portage._unicode_decode(x) for x in argv]
 163
 164         modes = {
 165                 'commit' : 'Run a scan then commit changes',
 166                 'ci' : 'Run a scan then commit changes',
 167                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 168                 'full' : 'Scan directory tree and print all issues (not a summary)',
 169                 'help' : 'Show this screen',
 170                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 171                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 172                 'scan' : 'Scan directory tree for QA issues'
 173         }
 174
 175         mode_keys = list(modes)
 176         mode_keys.sort()
 177
 178         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 179         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 180         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 181         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 182         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 183
 184         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 185                 help='Request a confirmation before commiting')
 186
 187         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 188                 help='specify a commit message on the command line')
 189
 190         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 191                 help='specify a path to a file that contains a commit message')
 192
 193         parser.add_option('--digest',
 194                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 195                 help='Automatically update Manifest digests for modified files')
 196
 197         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 198                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 199
 200         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 201                 help='do not print unnecessary messages')
 202
 203         parser.add_option(
 204                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 205                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 206                 'regardless of modification if \'force\' is specified)')
 207
 208         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 209                 help='Commit with QA violations')
 210
 211         parser.add_option('--vcs', dest='vcs',
 212                 help='Force using specific VCS instead of autodetection')
 213
 214         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 215                 help='be very verbose in output', default=0)
 216
 217         parser.add_option('-V', '--version', dest='version', action='store_true',
 218                 help='show version info')
 219
 220         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 221                 default=False, help='forces the metadata.xml parse check to be carried out')
 222
 223         parser.add_option(
 224                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 225                 metavar="<y|n>",
 226                 help='only check packages that have uncommitted modifications')
 227
 228         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 229                 default=False, help='ignore arch-specific failures (where arch != host)')
 230
 231         parser.add_option("--ignore-default-opts",
 232                 action="store_true",
 233                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 234
 235         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 236                 default=False, help='ignore masked packages (not allowed with commit mode)')
 237
 238         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 239                 default=False, help='include dev profiles in dependency checks')
 240
 241         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 242                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 243
 244         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 245                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 246
 247         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 248                 help='specify which mode repoman will run in (default=full)')
 249
 250         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 251
 252         for k in mode_keys:
 253                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 254
 255         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 256
 257         sorted_qa = list(qahelp)
 258         sorted_qa.sort()
 259         for k in sorted_qa:
 260                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 261
 262         opts, args = parser.parse_args(argv[1:])
 263
 264         if not opts.ignore_default_opts:
 265                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 266                 if default_opts:
 267                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 268
 269         if opts.mode == 'help':
 270                 parser.print_help(short=False)
 271
 272         for arg in args:
 273                 if arg in modes:
 274                         if not opts.mode:
 275                                 opts.mode = arg
 276                                 break
 277                 else:
 278                         parser.error("invalid mode: %s" % arg)
 279
 280         if not opts.mode:
 281                 opts.mode = 'full'
 282
 283         if opts.mode == 'ci':
 284                 opts.mode = 'commit'  # backwards compat shortcut
 285
 286         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 287                 if opts.ignore_masked:
 288                         parser.error('Commit mode and --ignore-masked are not compatible')
 289                 if opts.without_mask:
 290                         parser.error('Commit mode and --without-mask are not compatible')
 291
 292         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 293         for val in range(opts.verbosity):
 294                 logger = logging.getLogger()
 295                 logger.setLevel(logger.getEffectiveLevel() - 10)
 296
 297         for val in range(opts.quiet):
 298                 logger = logging.getLogger()
 299                 logger.setLevel(logger.getEffectiveLevel() + 10)
 300
 301         return (opts, args)
 302
 303 qahelp={
 304         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 305         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 306         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 307         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 308         "changelog.missing":"Missing ChangeLog files",
 309         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 310         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 311         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 312         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 313         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 314         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 315         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 316         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 317         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 318         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 319         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 320         "file.size":"Files in the files directory must be under 20 KiB",
 321         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 322         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 323         "file.UTF8":"File is not UTF8 compliant",
 324         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 325         "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it",
 326         "inherit.unused":"Ebuild inherits an eclass but does not use it",
 327         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 328         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 329         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 330         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 331         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 332         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 333         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 334         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 335         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 336         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 337         "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 338         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 339         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 340         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 341         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 342         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 343         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 344         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 345         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 346         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 347         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 348         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 349         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 350         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 351         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 352         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 353         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 354         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 355         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 356         "variable.readonly":"Assigning a readonly variable",
 357         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 358         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 359         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 360         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 361         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 362         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 363         "LICENSE.deprecated":"This ebuild is listing a deprecated license.",
 364         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 365         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 366         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 367         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 368         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 369         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 370         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 371         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 372         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 373         "ebuild.badheader":"This ebuild has a malformed header",
 374         "manifest.bad":"Manifest has missing or incorrect digests",
 375         "metadata.missing":"Missing metadata.xml files",
 376         "metadata.bad":"Bad metadata.xml files",
 377         "metadata.warning":"Warnings in metadata.xml files",
 378         "portage.internal":"The ebuild uses an internal Portage function or variable",
 379         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 380         "virtual.suspect":"Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 381         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 382         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 383 }
 384
 385 qacats = list(qahelp)
 386 qacats.sort()
 387
 388 qawarnings = set((
 389 "changelog.missing",
 390 "changelog.notadded",
 391 "dependency.unknown",
 392 "digest.assumed",
 393 "digest.unused",
 394 "ebuild.notadded",
 395 "ebuild.nesteddie",
 396 "dependency.badmasked",
 397 "dependency.badindev",
 398 "dependency.badmaskedindev",
 399 "dependency.badtilde",
 400 "DESCRIPTION.toolong",
 401 "EAPI.deprecated",
 402 "HOMEPAGE.virtual",
 403 "LICENSE.deprecated",
 404 "LICENSE.virtual",
 405 "KEYWORDS.dropped",
 406 "KEYWORDS.stupid",
 407 "KEYWORDS.missing",
 408 "PDEPEND.suspect",
 409 "RDEPEND.implicit",
 410 "RDEPEND.suspect",
 411 "virtual.suspect",
 412 "RESTRICT.invalid",
 413 "ebuild.minorsyn",
 414 "ebuild.badheader",
 415 "ebuild.patches",
 416 "file.size",
 417 "inherit.unused",
 418 "inherit.deprecated",
 419 "java.eclassesnotused",
 420 "wxwidgets.eclassnotused",
 421 "metadata.warning",
 422 "portage.internal",
 423 "usage.obsolete",
 424 "upstream.workaround",
 425 "LIVEVCS.stable",
 426 "LIVEVCS.unmasked",
 427 ))
 428
 429 if portage.const._ENABLE_INHERIT_CHECK:
 430         # This is experimental, so it's non-fatal.
 431         qawarnings.add("inherit.missing")
 432
 433 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 434
 435 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 436 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 437 allvars.update(Package.metadata_keys)
 438 allvars = sorted(allvars)
 439 commitmessage=None
 440 for x in missingvars:
 441         x += ".missing"
 442         if x not in qacats:
 443                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 444                 qacats.append(x)
 445                 qawarnings.add(x)
 446
 447 valid_restrict = frozenset(["binchecks", "bindist",
 448         "fetch", "installsources", "mirror",
 449         "primaryuri", "strip", "test", "userpriv"])
 450
 451 live_eclasses = frozenset([
 452         "bzr",
 453         "cvs",
 454         "darcs",
 455         "git",
 456         "git-2",
 457         "mercurial",
 458         "subversion",
 459         "tla",
 460 ])
 461
 462 suspect_rdepend = frozenset([
 463         "app-arch/cabextract",
 464         "app-arch/rpm2targz",
 465         "app-doc/doxygen",
 466         "dev-lang/nasm",
 467         "dev-lang/swig",
 468         "dev-lang/yasm",
 469         "dev-perl/extutils-pkgconfig",
 470         "dev-util/byacc",
 471         "dev-util/cmake",
 472         "dev-util/ftjam",
 473         "dev-util/gperf",
 474         "dev-util/gtk-doc",
 475         "dev-util/gtk-doc-am",
 476         "dev-util/intltool",
 477         "dev-util/jam",
 478         "dev-util/pkg-config-lite",
 479         "dev-util/pkgconf",
 480         "dev-util/pkgconfig",
 481         "dev-util/pkgconfig-openbsd",
 482         "dev-util/scons",
 483         "dev-util/unifdef",
 484         "dev-util/yacc",
 485         "media-gfx/ebdftopcf",
 486         "sys-apps/help2man",
 487         "sys-devel/autoconf",
 488         "sys-devel/automake",
 489         "sys-devel/bin86",
 490         "sys-devel/bison",
 491         "sys-devel/dev86",
 492         "sys-devel/flex",
 493         "sys-devel/m4",
 494         "sys-devel/pmake",
 495         "virtual/linux-sources",
 496         "virtual/pkgconfig",
 497         "x11-misc/bdftopcf",
 498         "x11-misc/imake",
 499 ])
 500
 501 suspect_virtual = {
 502         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 503         "dev-util/pkgconf":"virtual/pkgconfig",
 504         "dev-util/pkgconfig":"virtual/pkgconfig",
 505         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 506         "dev-libs/libusb":"virtual/libusb",
 507         "dev-libs/libusbx":"virtual/libusb",
 508         "dev-libs/libusb-compat":"virtual/libusb",
 509 }
 510
 511 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 512 # force refetch if the local copy creation time is older than this
 513 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 514
 515 # file.executable
 516 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 517
 518 options, arguments = ParseArgs(sys.argv, qahelp)
 519
 520 if options.version:
 521         print("Portage", portage.VERSION)
 522         sys.exit(0)
 523
 524 # Set this to False when an extraordinary issue (generally
 525 # something other than a QA issue) makes it impossible to
 526 # commit (like if Manifest generation fails).
 527 can_force = True
 528
 529 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 530 if portdir is None:
 531         sys.exit(1)
 532
 533 myreporoot = os.path.basename(portdir_overlay)
 534 myreporoot += mydir[len(portdir_overlay):]
 535
 536 if options.vcs:
 537         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 538                 vcs = options.vcs
 539         else:
 540                 vcs = None
 541 else:
 542         vcses = utilities.FindVCS()
 543         if len(vcses) > 1:
 544                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 545                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 546                 sys.exit(1)
 547         elif vcses:
 548                 vcs = vcses[0]
 549         else:
 550                 vcs = None
 551
 552 if options.if_modified == "y" and vcs is None:
 553         logging.info("Not in a version controlled repository; "
 554                 "disabling --if-modified.")
 555         options.if_modified = "n"
 556
 557 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 558 vcs_preserves_mtime = vcs in ('cvs',)
 559
 560 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 561 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 562 if vcs_global_opts is None:
 563         if vcs in ('cvs', 'svn'):
 564                 vcs_global_opts = "-q"
 565         else:
 566                 vcs_global_opts = ""
 567 vcs_global_opts = vcs_global_opts.split()
 568
 569 if options.mode == 'commit' and not options.pretend and not vcs:
 570         logging.info("Not in a version controlled repository; enabling pretend mode.")
 571         options.pretend = True
 572
 573 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 574 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 575         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 576         portage._shell_quote(portdir_overlay))
 577 # We have to call the config constructor again so
 578 # that config.repositories is initialized correctly.
 579 repoman_settings = portage.config(config_root=config_root, local_config=False,
 580         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 581
 582 root = repoman_settings['EROOT']
 583 trees = {
 584         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 585 }
 586 portdb = trees[root]['porttree'].dbapi
 587
 588 # Constrain dependency resolution to the master(s)
 589 # that are specified in layout.conf.
 590 repodir = os.path.realpath(portdir_overlay)
 591 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 592 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 593 portdir = portdb.porttrees[0]
 594
 595 if repo_config.allow_provide_virtual:
 596         qawarnings.add("virtual.oldstyle")
 597
 598 if repo_config.sign_commit:
 599         if vcs == 'git':
 600                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 601                 # the commit arguments. If key_id is unspecified, then it must be
 602                 # configured by `git config user.signingkey key_id`.
 603                 vcs_local_opts.append("--gpg-sign")
 604
 605 # In order to disable manifest signatures, repos may set
 606 # "sign-manifests = false" in metadata/layout.conf. This
 607 # can be used to prevent merge conflicts like those that
 608 # thin-manifests is designed to prevent.
 609 sign_manifests = "sign" in repoman_settings.features and \
 610         repo_config.sign_manifest
 611
 612 if sign_manifests and options.mode in ("commit",) and \
 613         repoman_settings.get("PORTAGE_GPG_KEY") and \
 614         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 615         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 616         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 617                 repoman_settings["PORTAGE_GPG_KEY"])
 618         sys.exit(1)
 619
 620 manifest_hashes = repo_config.manifest_hashes
 621 if manifest_hashes is None:
 622         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 623
 624 if options.mode in ("commit", "fix", "manifest"):
 625         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 626                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 627                 "metadata/layout.conf does not contain the '%s' hash which "
 628                 "is required by this portage version. You will have to "
 629                 "upgrade portage if you want to generate valid manifests for "
 630                 "this repository.") % \
 631                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 632                 for line in textwrap.wrap(msg, 70):
 633                         logging.error(line)
 634                 sys.exit(1)
 635
 636         unsupported_hashes = manifest_hashes.difference(
 637                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 638         if unsupported_hashes:
 639                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 640                 "metadata/layout.conf contains one or more hash types '%s' "
 641                 "which are not supported by this portage version. You will "
 642                 "have to upgrade portage if you want to generate valid "
 643                 "manifests for this repository.") % \
 644                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 645                 for line in textwrap.wrap(msg, 70):
 646                         logging.error(line)
 647                 sys.exit(1)
 648
 649 if "commit" == options.mode and \
 650         repo_config.name == "gentoo" and \
 651         "RMD160" in manifest_hashes and \
 652         "RMD160" not in portage.checksum.hashorigin_map:
 653         msg = "Please install " \
 654         "pycrypto or enable python's ssl USE flag in order " \
 655         "to enable RMD160 hash support. See bug #198398 for " \
 656         "more information."
 657         prefix = bad(" * ")
 658         for line in textwrap.wrap(msg, 70):
 659                 print(prefix + line)
 660         sys.exit(1)
 661
 662 if options.echangelog is None and repo_config.update_changelog:
 663         options.echangelog = 'y'
 664
 665 if vcs is None:
 666         options.echangelog = 'n'
 667
 668 # The --echangelog option causes automatic ChangeLog generation,
 669 # which invalidates changelog.ebuildadded and changelog.missing
 670 # checks.
 671 # Note: Some don't use ChangeLogs in distributed SCMs.
 672 # It will be generated on server side from scm log,
 673 # before package moves to the rsync server.
 674 # This is needed because they try to avoid merge collisions.
 675 # Gentoo's Council decided to always use the ChangeLog file.
 676 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 677 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 678
 679 if 'digest' in repoman_settings.features and options.digest != 'n':
 680         options.digest = 'y'
 681
 682 logging.debug("vcs: %s" % (vcs,))
 683 logging.debug("repo config: %s" % (repo_config,))
 684 logging.debug("options: %s" % (options,))
 685
 686 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 687 # profile-specific config constructor calls.
 688 env = os.environ.copy()
 689 env['PORTDIR'] = portdir
 690 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 691
 692 logging.info('Setting paths:')
 693 logging.info('PORTDIR = "' + portdir + '"')
 694 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 695
 696 # It's confusing if these warnings are displayed without the user
 697 # being told which profile they come from, so disable them.
 698 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 699
 700 categories = []
 701 for path in repo_config.eclass_db.porttrees:
 702         categories.extend(portage.util.grabfile(
 703                 os.path.join(path, 'profiles', 'categories')))
 704 repoman_settings.categories = frozenset(
 705         portage.util.stack_lists([categories], incremental=1))
 706 categories = repoman_settings.categories
 707
 708 portdb.settings = repoman_settings
 709 root_config = RootConfig(repoman_settings, trees[root], None)
 710 # We really only need to cache the metadata that's necessary for visibility
 711 # filtering. Anything else can be discarded to reduce memory consumption.
 712 portdb._aux_cache_keys.clear()
 713 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 714
 715 reposplit = myreporoot.split(os.path.sep)
 716 repolevel = len(reposplit)
 717
 718 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 719 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 720 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 721 if options.mode == 'commit' and repolevel not in [1,2,3]:
 722         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 723         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 724         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 725         print(red("***"))
 726         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 727
 728 # Make startdir relative to the canonical repodir, so that we can pass
 729 # it to digestgen and it won't have to be canonicalized again.
 730 if repolevel == 1:
 731         startdir = repodir
 732 else:
 733         startdir = normalize_path(mydir)
 734         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 735
 736 def caterror(mycat):
 737         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 738
 739 def repoman_getstatusoutput(cmd):
 740         """
 741         Implements an interface similar to getstatusoutput(), but with
 742         customized unicode handling (see bug #310789) and without the shell.
 743         """
 744         args = portage.util.shlex_split(cmd)
 745         encoding = _encodings['fs']
 746         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 747                 # Python 3.1 does not support bytes in Popen args.
 748                 args = [_unicode_encode(x,
 749                         encoding=encoding, errors='strict') for x in args]
 750         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 751                 stderr=subprocess.STDOUT)
 752         output = portage._unicode_decode(proc.communicate()[0],
 753                 encoding=encoding, errors='strict')
 754         if output and output[-1] == "\n":
 755                 # getstatusoutput strips one newline
 756                 output = output[:-1]
 757         return (proc.wait(), output)
 758
 759 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 760         """
 761         Implements an interface similar to os.popen(), but with customized
 762         unicode handling (see bug #310789) and without the shell.
 763         """
 764
 765         __slots__ = ('_proc', '_stdout')
 766
 767         def __init__(self, cmd):
 768                 args = portage.util.shlex_split(cmd)
 769                 encoding = _encodings['fs']
 770                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 771                         # Python 3.1 does not support bytes in Popen args.
 772                         args = [_unicode_encode(x,
 773                                 encoding=encoding, errors='strict') for x in args]
 774                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 775                 object.__setattr__(self, '_proc', proc)
 776                 object.__setattr__(self, '_stdout',
 777                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 778
 779         def _get_target(self):
 780                 return object.__getattribute__(self, '_stdout')
 781
 782         __enter__ = _get_target
 783
 784         def __exit__(self, exc_type, exc_value, traceback):
 785                 proc = object.__getattribute__(self, '_proc')
 786                 proc.wait()
 787                 proc.stdout.close()
 788
 789 class ProfileDesc(object):
 790         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 791         def __init__(self, arch, status, sub_path, tree_path):
 792                 self.arch = arch
 793                 self.status = status
 794                 if sub_path:
 795                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 796                 self.sub_path = sub_path
 797                 self.tree_path = tree_path
 798                 if tree_path:
 799                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 800                 else:
 801                         self.abs_path = tree_path
 802
 803         def __str__(self):
 804                 if self.sub_path:
 805                         return self.sub_path
 806                 return 'empty profile'
 807
 808 profile_list = []
 809 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 810
 811 # get lists of valid keywords, licenses, and use
 812 kwlist = set()
 813 liclist = set()
 814 uselist = set()
 815 global_pmasklines = []
 816
 817 for path in portdb.porttrees:
 818         try:
 819                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 820         except OSError:
 821                 pass
 822         kwlist.update(portage.grabfile(os.path.join(path,
 823                 "profiles", "arch.list")))
 824
 825         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 826         for x in use_desc:
 827                 x = x.split()
 828                 if x:
 829                         uselist.add(x[0])
 830
 831         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 832         try:
 833                 expand_list = os.listdir(expand_desc_dir)
 834         except OSError:
 835                 pass
 836         else:
 837                 for fn in expand_list:
 838                         if not fn[-5:] == '.desc':
 839                                 continue
 840                         use_prefix = fn[:-5].lower() + '_'
 841                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 842                                 x = x.split()
 843                                 if x:
 844                                         uselist.add(use_prefix + x[0])
 845
 846         global_pmasklines.append(portage.util.grabfile_package(
 847                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 848
 849         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 850         try:
 851                 desc_file = io.open(_unicode_encode(desc_path,
 852                         encoding=_encodings['fs'], errors='strict'),
 853                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 854         except EnvironmentError:
 855                 pass
 856         else:
 857                 for i, x in enumerate(desc_file):
 858                         if x[0] == "#":
 859                                 continue
 860                         arch = x.split()
 861                         if len(arch) == 0:
 862                                 continue
 863                         if len(arch) != 3:
 864                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 865                                         desc_path + " line %d" % (i+1, ))
 866                         elif arch[0] not in kwlist:
 867                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 868                                         desc_path + " line %d" % (i+1, ))
 869                         elif arch[2] not in valid_profile_types:
 870                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 871                                         desc_path + " line %d" % (i+1, ))
 872                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 873                         if not os.path.isdir(profile_desc.abs_path):
 874                                 logging.error(
 875                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 876                                         arch[2], arch[1], arch[0], desc_path, i+1)
 877                                 continue
 878                         if os.path.exists(
 879                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 880                                 continue
 881                         profile_list.append(profile_desc)
 882                 desc_file.close()
 883
 884 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 885 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 886
 887 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 888 global_pmaskdict = {}
 889 for x in global_pmasklines:
 890         global_pmaskdict.setdefault(x.cp, []).append(x)
 891 del global_pmasklines
 892
 893 def has_global_mask(pkg):
 894         mask_atoms = global_pmaskdict.get(pkg.cp)
 895         if mask_atoms:
 896                 pkg_list = [pkg]
 897                 for x in mask_atoms:
 898                         if portage.dep.match_from_list(x, pkg_list):
 899                                 return x
 900         return None
 901
 902 # Ensure that profile sub_path attributes are unique. Process in reverse order
 903 # so that profiles with duplicate sub_path from overlays will override
 904 # profiles with the same sub_path from parent repos.
 905 profiles = {}
 906 profile_list.reverse()
 907 profile_sub_paths = set()
 908 for prof in profile_list:
 909         if prof.sub_path in profile_sub_paths:
 910                 continue
 911         profile_sub_paths.add(prof.sub_path)
 912         profiles.setdefault(prof.arch, []).append(prof)
 913
 914 # Use an empty profile for checking dependencies of
 915 # packages that have empty KEYWORDS.
 916 prof = ProfileDesc('**', 'stable', '', '')
 917 profiles.setdefault(prof.arch, []).append(prof)
 918
 919 for x in repoman_settings.archlist():
 920         if x[0] == "~":
 921                 continue
 922         if x not in profiles:
 923                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 924                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 925                 print(red("up with the "+x+" team."))
 926                 print()
 927
 928 liclist_deprecated = set()
 929 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 930         liclist_deprecated.update(
 931                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 932
 933 if not liclist:
 934         logging.fatal("Couldn't find licenses?")
 935         sys.exit(1)
 936
 937 if not kwlist:
 938         logging.fatal("Couldn't read KEYWORDS from arch.list")
 939         sys.exit(1)
 940
 941 if not uselist:
 942         logging.fatal("Couldn't find use.desc?")
 943         sys.exit(1)
 944
 945 scanlist=[]
 946 if repolevel==2:
 947         #we are inside a category directory
 948         catdir=reposplit[-1]
 949         if catdir not in categories:
 950                 caterror(catdir)
 951         mydirlist=os.listdir(startdir)
 952         for x in mydirlist:
 953                 if x == "CVS" or x.startswith("."):
 954                         continue
 955                 if os.path.isdir(startdir+"/"+x):
 956                         scanlist.append(catdir+"/"+x)
 957         repo_subdir = catdir + os.sep
 958 elif repolevel==1:
 959         for x in categories:
 960                 if not os.path.isdir(startdir+"/"+x):
 961                         continue
 962                 for y in os.listdir(startdir+"/"+x):
 963                         if y == "CVS" or y.startswith("."):
 964                                 continue
 965                         if os.path.isdir(startdir+"/"+x+"/"+y):
 966                                 scanlist.append(x+"/"+y)
 967         repo_subdir = ""
 968 elif repolevel==3:
 969         catdir = reposplit[-2]
 970         if catdir not in categories:
 971                 caterror(catdir)
 972         scanlist.append(catdir+"/"+reposplit[-1])
 973         repo_subdir = scanlist[-1] + os.sep
 974 else:
 975         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 976                 ' from the current working directory'
 977         logging.critical(msg)
 978         sys.exit(1)
 979
 980 repo_subdir_len = len(repo_subdir)
 981 scanlist.sort()
 982
 983 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 984
 985 def vcs_files_to_cps(vcs_file_iter):
 986         """
 987         Iterate over the given modified file paths returned from the vcs,
 988         and return a frozenset containing category/pn strings for each
 989         modified package.
 990         """
 991
 992         modified_cps = []
 993
 994         if repolevel == 3:
 995                 if reposplit[-2] in categories and \
 996                         next(vcs_file_iter, None) is not None:
 997                         modified_cps.append("/".join(reposplit[-2:]))
 998
 999         elif repolevel == 2:
1000                 category = reposplit[-1]
1001                 if category in categories:
1002                         for filename in vcs_file_iter:
1003                                 f_split = filename.split(os.sep)
1004                                 # ['.', pn,...]
1005                                 if len(f_split) > 2:
1006                                         modified_cps.append(category + "/" + f_split[1])
1007
1008         else:
1009                 # repolevel == 1
1010                 for filename in vcs_file_iter:
1011                         f_split = filename.split(os.sep)
1012                         # ['.', category, pn,...]
1013                         if len(f_split) > 3 and f_split[1] in categories:
1014                                 modified_cps.append("/".join(f_split[1:3]))
1015
1016         return frozenset(modified_cps)
1017
1018 def git_supports_gpg_sign():
1019         status, cmd_output = \
1020                 repoman_getstatusoutput("git --version")
1021         cmd_output = cmd_output.split()
1022         if cmd_output:
1023                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1024                 if version is not None:
1025                         version = [int(x) for x in version.groups()]
1026                         if version[0] > 1 or \
1027                                 (version[0] == 1 and version[1] > 7) or \
1028                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1029                                 return True
1030         return False
1031
1032 def dev_keywords(profiles):
1033         """
1034         Create a set of KEYWORDS values that exist in 'dev'
1035         profiles. These are used
1036         to trigger a message notifying the user when they might
1037         want to add the --include-dev option.
1038         """
1039         type_arch_map = {}
1040         for arch, arch_profiles in profiles.items():
1041                 for prof in arch_profiles:
1042                         arch_set = type_arch_map.get(prof.status)
1043                         if arch_set is None:
1044                                 arch_set = set()
1045                                 type_arch_map[prof.status] = arch_set
1046                         arch_set.add(arch)
1047
1048         dev_keywords = type_arch_map.get('dev', set())
1049         dev_keywords.update(['~' + arch for arch in dev_keywords])
1050         return frozenset(dev_keywords)
1051
1052 dev_keywords = dev_keywords(profiles)
1053
1054 stats={}
1055 fails={}
1056
1057 for x in qacats:
1058         stats[x]=0
1059         fails[x]=[]
1060
1061 xmllint_capable = False
1062 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1063
1064 def parsedate(s):
1065         """Parse a RFC 822 date and time string.
1066         This is required for python3 compatibility, since the
1067         rfc822.parsedate() function is not available."""
1068
1069         s_split = []
1070         for x in s.upper().split():
1071                 for y in x.split(','):
1072                         if y:
1073                                 s_split.append(y)
1074
1075         if len(s_split) != 6:
1076                 return None
1077
1078         # %a, %d %b %Y %H:%M:%S %Z
1079         a, d, b, Y, H_M_S, Z = s_split
1080
1081         # Convert month to integer, since strptime %w is locale-dependent.
1082         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1083                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1084         m = month_map.get(b)
1085         if m is None:
1086                 return None
1087         m = str(m).rjust(2, '0')
1088
1089         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1090
1091 def fetch_metadata_dtd():
1092         """
1093         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1094         metadata_dtd_ctime_interval.
1095         @rtype: bool
1096         @return: True if successful, otherwise False
1097         """
1098
1099         must_fetch = True
1100         metadata_dtd_st = None
1101         current_time = int(time.time())
1102         try:
1103                 metadata_dtd_st = os.stat(metadata_dtd)
1104         except EnvironmentError as e:
1105                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1106                         raise
1107                 del e
1108         else:
1109                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1110                 if abs(current_time - metadata_dtd_st.st_ctime) \
1111                         < metadata_dtd_ctime_interval:
1112                         must_fetch = False
1113
1114         if must_fetch:
1115                 print()
1116                 print(green("***") + " the local copy of metadata.dtd " + \
1117                         "needs to be refetched, doing that now")
1118                 print()
1119                 try:
1120                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1121                         msg_info = url_f.info()
1122                         last_modified = msg_info.get('last-modified')
1123                         if last_modified is not None:
1124                                 last_modified = parsedate(last_modified)
1125                                 if last_modified is not None:
1126                                         last_modified = calendar.timegm(last_modified)
1127
1128                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1129                         try:
1130                                 local_f = open(metadata_dtd_tmp, mode='wb')
1131                                 local_f.write(url_f.read())
1132                                 local_f.close()
1133                                 if last_modified is not None:
1134                                         try:
1135                                                 os.utime(metadata_dtd_tmp,
1136                                                         (int(last_modified), int(last_modified)))
1137                                         except OSError:
1138                                                 # This fails on some odd non-unix-like filesystems.
1139                                                 # We don't really need the mtime to be preserved
1140                                                 # anyway here (currently we use ctime to trigger
1141                                                 # fetch), so just ignore it.
1142                                                 pass
1143                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1144                         finally:
1145                                 try:
1146                                         os.unlink(metadata_dtd_tmp)
1147                                 except OSError:
1148                                         pass
1149
1150                         url_f.close()
1151
1152                 except EnvironmentError as e:
1153                         print()
1154                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1155                         print(red("!!!")+" exception '%s' though." % (e,))
1156                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1157                         return False
1158
1159         return True
1160
1161 if options.mode == "manifest":
1162         pass
1163 elif not find_binary('xmllint'):
1164         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1165         if options.xml_parse or repolevel==3:
1166                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1167                 sys.exit(1)
1168 else:
1169         if not fetch_metadata_dtd():
1170                 sys.exit(1)
1171         #this can be problematic if xmllint changes their output
1172         xmllint_capable=True
1173
1174 if options.mode == 'commit' and vcs:
1175         utilities.detect_vcs_conflicts(options, vcs)
1176
1177 if options.mode == "manifest":
1178         pass
1179 elif options.pretend:
1180         print(green("\nRepoMan does a once-over of the neighborhood..."))
1181 else:
1182         print(green("\nRepoMan scours the neighborhood..."))
1183
1184 new_ebuilds = set()
1185 modified_ebuilds = set()
1186 modified_changelogs = set()
1187 mychanged = []
1188 mynew = []
1189 myremoved = []
1190
1191 if vcs == "cvs":
1192         mycvstree = cvstree.getentries("./", recursive=1)
1193         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1194         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1195         if options.if_modified == "y":
1196                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1197
1198 elif vcs == "svn":
1199         with repoman_popen("svn status") as f:
1200                 svnstatus = f.readlines()
1201         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1202         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1203         if options.if_modified == "y":
1204                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1205
1206 elif vcs == "git":
1207         with repoman_popen("git diff-index --name-only "
1208                 "--relative --diff-filter=M HEAD") as f:
1209                 mychanged = f.readlines()
1210         mychanged = ["./" + elem[:-1] for elem in mychanged]
1211
1212         with repoman_popen("git diff-index --name-only "
1213                 "--relative --diff-filter=A HEAD") as f:
1214                 mynew = f.readlines()
1215         mynew = ["./" + elem[:-1] for elem in mynew]
1216         if options.if_modified == "y":
1217                 with repoman_popen("git diff-index --name-only "
1218                         "--relative --diff-filter=D HEAD") as f:
1219                         myremoved = f.readlines()
1220                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1221
1222 elif vcs == "bzr":
1223         with repoman_popen("bzr status -S .") as f:
1224                 bzrstatus = f.readlines()
1225         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1226         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1227         if options.if_modified == "y":
1228                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1229
1230 elif vcs == "hg":
1231         with repoman_popen("hg status --no-status --modified .") as f:
1232                 mychanged = f.readlines()
1233         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1234         mynew = repoman_popen("hg status --no-status --added .").readlines()
1235         mynew = ["./" + elem.rstrip() for elem in mynew]
1236         if options.if_modified == "y":
1237                 with repoman_popen("hg status --no-status --removed .") as f:
1238                         myremoved = f.readlines()
1239                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1240
1241 if vcs:
1242         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1243         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1244         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1245                 if os.path.basename(x) == "ChangeLog")
1246
1247 def vcs_new_changed(relative_path):
1248         for x in chain(mychanged, mynew):
1249                 if x == relative_path:
1250                         return True
1251         return False
1252
1253 have_pmasked = False
1254 have_dev_keywords = False
1255 dofail = 0
1256
1257 # NOTE: match-all caches are not shared due to potential
1258 # differences between profiles in _get_implicit_iuse.
1259 arch_caches={}
1260 arch_xmatch_caches = {}
1261 shared_xmatch_caches = {"cp-list":{}}
1262
1263 # Disable the "ebuild.notadded" check when not in commit mode and
1264 # running `svn status` in every package dir will be too expensive.
1265
1266 check_ebuild_notadded = not \
1267         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1268
1269 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1270 thirdpartymirrors = {}
1271 for k, v in repoman_settings.thirdpartymirrors().items():
1272         for v in v:
1273                 if not v.endswith("/"):
1274                         v += "/"
1275                 thirdpartymirrors[v] = k
1276
1277 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1278         """
1279         Implements doctype() as required to avoid deprecation warnings with
1280         >=python-2.7.
1281         """
1282         def doctype(self, name, pubid, system):
1283                 pass
1284
1285 try:
1286         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1287 except (EnvironmentError, ParseError, PermissionDenied) as e:
1288         err(str(e))
1289 except FileNotFound:
1290         # TODO: Download as we do for metadata.dtd, but add a way to
1291         # disable for non-gentoo repoman users who may not have herds.
1292         herd_base = None
1293
1294 effective_scanlist = scanlist
1295 if options.if_modified == "y":
1296         effective_scanlist = sorted(vcs_files_to_cps(
1297                 chain(mychanged, mynew, myremoved)))
1298
1299 for x in effective_scanlist:
1300         #ebuilds and digests added to cvs respectively.
1301         logging.info("checking package %s" % x)
1302         # save memory by discarding xmatch caches from previous package(s)
1303         arch_xmatch_caches.clear()
1304         eadded=[]
1305         catdir,pkgdir=x.split("/")
1306         checkdir=repodir+"/"+x
1307         checkdir_relative = ""
1308         if repolevel < 3:
1309                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1310         if repolevel < 2:
1311                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1312         checkdir_relative = os.path.join(".", checkdir_relative)
1313         generated_manifest = False
1314
1315         if options.mode == "manifest" or \
1316           (options.mode != 'manifest-check' and options.digest == 'y') or \
1317           options.mode in ('commit', 'fix') and not options.pretend:
1318                 auto_assumed = set()
1319                 fetchlist_dict = portage.FetchlistDict(checkdir,
1320                         repoman_settings, portdb)
1321                 if options.mode == 'manifest' and options.force:
1322                         portage._doebuild_manifest_exempt_depend += 1
1323                         try:
1324                                 distdir = repoman_settings['DISTDIR']
1325                                 mf = repoman_settings.repositories.get_repo_for_location(
1326                                         os.path.dirname(os.path.dirname(checkdir)))
1327                                 mf = mf.load_manifest(checkdir, distdir,
1328                                         fetchlist_dict=fetchlist_dict)
1329                                 mf.create(requiredDistfiles=None,
1330                                         assumeDistHashesAlways=True)
1331                                 for distfiles in fetchlist_dict.values():
1332                                         for distfile in distfiles:
1333                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1334                                                         mf.fhashdict['DIST'].pop(distfile, None)
1335                                                 else:
1336                                                         auto_assumed.add(distfile)
1337                                 mf.write()
1338                         finally:
1339                                 portage._doebuild_manifest_exempt_depend -= 1
1340
1341                 repoman_settings["O"] = checkdir
1342                 try:
1343                         generated_manifest = digestgen(
1344                                 mysettings=repoman_settings, myportdb=portdb)
1345                 except portage.exception.PermissionDenied as e:
1346                         generated_manifest = False
1347                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1348                                 level=logging.ERROR, noiselevel=-1)
1349
1350                 if not generated_manifest:
1351                         print("Unable to generate manifest.")
1352                         dofail = 1
1353
1354                 if options.mode == "manifest":
1355                         if not dofail and options.force and auto_assumed and \
1356                                 'assume-digests' in repoman_settings.features:
1357                                 # Show which digests were assumed despite the --force option
1358                                 # being given. This output will already have been shown by
1359                                 # digestgen() if assume-digests is not enabled, so only show
1360                                 # it here if assume-digests is enabled.
1361                                 pkgs = list(fetchlist_dict)
1362                                 pkgs.sort()
1363                                 portage.writemsg_stdout("  digest.assumed" + \
1364                                         portage.output.colorize("WARN",
1365                                         str(len(auto_assumed)).rjust(18)) + "\n")
1366                                 for cpv in pkgs:
1367                                         fetchmap = fetchlist_dict[cpv]
1368                                         pf = portage.catsplit(cpv)[1]
1369                                         for distfile in sorted(fetchmap):
1370                                                 if distfile in auto_assumed:
1371                                                         portage.writemsg_stdout(
1372                                                                 "   %s::%s\n" % (pf, distfile))
1373                         continue
1374                 elif dofail:
1375                         sys.exit(1)
1376
1377         if not generated_manifest:
1378                 repoman_settings['O'] = checkdir
1379                 repoman_settings['PORTAGE_QUIET'] = '1'
1380                 if not portage.digestcheck([], repoman_settings, strict=1):
1381                         stats["manifest.bad"] += 1
1382                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1383                 repoman_settings.pop('PORTAGE_QUIET', None)
1384
1385         if options.mode == 'manifest-check':
1386                 continue
1387
1388         checkdirlist=os.listdir(checkdir)
1389         ebuildlist=[]
1390         pkgs = {}
1391         allvalid = True
1392         for y in checkdirlist:
1393                 if (y in no_exec or y.endswith(".ebuild")) and \
1394                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1395                                 stats["file.executable"] += 1
1396                                 fails["file.executable"].append(os.path.join(checkdir, y))
1397                 if y.endswith(".ebuild"):
1398                         pf = y[:-7]
1399                         ebuildlist.append(pf)
1400                         cpv = "%s/%s" % (catdir, pf)
1401                         try:
1402                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1403                         except KeyError:
1404                                 allvalid = False
1405                                 stats["ebuild.syntax"] += 1
1406                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1407                                 continue
1408                         except IOError:
1409                                 allvalid = False
1410                                 stats["ebuild.output"] += 1
1411                                 fails["ebuild.output"].append(os.path.join(x, y))
1412                                 continue
1413                         if not portage.eapi_is_supported(myaux["EAPI"]):
1414                                 allvalid = False
1415                                 stats["EAPI.unsupported"] += 1
1416                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1417                                 continue
1418                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1419                                 root_config=root_config, type_name="ebuild")
1420
1421         slot_keywords = {}
1422
1423         if len(pkgs) != len(ebuildlist):
1424                 # If we can't access all the metadata then it's totally unsafe to
1425                 # commit since there's no way to generate a correct Manifest.
1426                 # Do not try to do any more QA checks on this package since missing
1427                 # metadata leads to false positives for several checks, and false
1428                 # positives confuse users.
1429                 can_force = False
1430                 continue
1431
1432         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1433         ebuildlist = sorted(pkgs.values())
1434         ebuildlist = [pkg.pf for pkg in ebuildlist]
1435
1436         for y in checkdirlist:
1437                 index = repo_config.find_invalid_path_char(y)
1438                 if index != -1:
1439                         y_relative = os.path.join(checkdir_relative, y)
1440                         if vcs is not None and not vcs_new_changed(y_relative):
1441                                 # If the file isn't in the VCS new or changed set, then
1442                                 # assume that it's an irrelevant temporary file (Manifest
1443                                 # entries are not generated for file names containing
1444                                 # prohibited characters). See bug #406877.
1445                                 index = -1
1446                 if index != -1:
1447                         stats["file.name"] += 1
1448                         fails["file.name"].append("%s/%s: char '%s'" % \
1449                                 (checkdir, y, y[index]))
1450
1451                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1452                         continue
1453                 f = None
1454                 try:
1455                         line = 1
1456                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1457                                 encoding=_encodings['fs'], errors='strict'),
1458                                 mode='r', encoding=_encodings['repo.content'])
1459                         for l in f:
1460                                 line +=1
1461                 except UnicodeDecodeError as ue:
1462                         stats["file.UTF8"] += 1
1463                         s = ue.object[:ue.start]
1464                         l2 = s.count("\n")
1465                         line += l2
1466                         if l2 != 0:
1467                                 s = s[s.rfind("\n") + 1:]
1468                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1469                 finally:
1470                         if f is not None:
1471                                 f.close()
1472
1473         if vcs in ("git", "hg") and check_ebuild_notadded:
1474                 if vcs == "git":
1475                         myf = repoman_popen("git ls-files --others %s" % \
1476                                 (portage._shell_quote(checkdir_relative),))
1477                 if vcs == "hg":
1478                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1479                                 (portage._shell_quote(checkdir_relative),))
1480                 for l in myf:
1481                         if l[:-1][-7:] == ".ebuild":
1482                                 stats["ebuild.notadded"] += 1
1483                                 fails["ebuild.notadded"].append(
1484                                         os.path.join(x, os.path.basename(l[:-1])))
1485                 myf.close()
1486
1487         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1488                 try:
1489                         if vcs == "cvs":
1490                                 myf=open(checkdir+"/CVS/Entries","r")
1491                         if vcs == "svn":
1492                                 myf = repoman_popen("svn status --depth=files --verbose " +
1493                                         portage._shell_quote(checkdir))
1494                         if vcs == "bzr":
1495                                 myf = repoman_popen("bzr ls -v --kind=file " +
1496                                         portage._shell_quote(checkdir))
1497                         myl = myf.readlines()
1498                         myf.close()
1499                         for l in myl:
1500                                 if vcs == "cvs":
1501                                         if l[0]!="/":
1502                                                 continue
1503                                         splitl=l[1:].split("/")
1504                                         if not len(splitl):
1505                                                 continue
1506                                         if splitl[0][-7:]==".ebuild":
1507                                                 eadded.append(splitl[0][:-7])
1508                                 if vcs == "svn":
1509                                         if l[:1] == "?":
1510                                                 continue
1511                                         if l[:7] == '      >':
1512                                                 # tree conflict, new in subversion 1.6
1513                                                 continue
1514                                         l = l.split()[-1]
1515                                         if l[-7:] == ".ebuild":
1516                                                 eadded.append(os.path.basename(l[:-7]))
1517                                 if vcs == "bzr":
1518                                         if l[1:2] == "?":
1519                                                 continue
1520                                         l = l.split()[-1]
1521                                         if l[-7:] == ".ebuild":
1522                                                 eadded.append(os.path.basename(l[:-7]))
1523                         if vcs == "svn":
1524                                 myf = repoman_popen("svn status " +
1525                                         portage._shell_quote(checkdir))
1526                                 myl=myf.readlines()
1527                                 myf.close()
1528                                 for l in myl:
1529                                         if l[0] == "A":
1530                                                 l = l.rstrip().split(' ')[-1]
1531                                                 if l[-7:] == ".ebuild":
1532                                                         eadded.append(os.path.basename(l[:-7]))
1533                 except IOError:
1534                         if vcs == "cvs":
1535                                 stats["CVS/Entries.IO_error"] += 1
1536                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1537                         else:
1538                                 raise
1539                         continue
1540
1541         mf = repoman_settings.repositories.get_repo_for_location(
1542                 os.path.dirname(os.path.dirname(checkdir)))
1543         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1544         mydigests=mf.getTypeDigests("DIST")
1545
1546         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1547         myfiles_all = []
1548         src_uri_error = False
1549         for mykey in fetchlist_dict:
1550                 try:
1551                         myfiles_all.extend(fetchlist_dict[mykey])
1552                 except portage.exception.InvalidDependString as e:
1553                         src_uri_error = True
1554                         try:
1555                                 portdb.aux_get(mykey, ["SRC_URI"])
1556                         except KeyError:
1557                                 # This will be reported as an "ebuild.syntax" error.
1558                                 pass
1559                         else:
1560                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1561                                 fails["SRC_URI.syntax"].append(
1562                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1563         del fetchlist_dict
1564         if not src_uri_error:
1565                 # This test can produce false positives if SRC_URI could not
1566                 # be parsed for one or more ebuilds. There's no point in
1567                 # producing a false error here since the root cause will
1568                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1569                 # or "ebuild.sytax".
1570                 myfiles_all = set(myfiles_all)
1571                 for entry in mydigests:
1572                         if entry not in myfiles_all:
1573                                 stats["digest.unused"] += 1
1574                                 fails["digest.unused"].append(checkdir+"::"+entry)
1575                 for entry in myfiles_all:
1576                         if entry not in mydigests:
1577                                 stats["digest.missing"] += 1
1578                                 fails["digest.missing"].append(checkdir+"::"+entry)
1579         del myfiles_all
1580
1581         if os.path.exists(checkdir+"/files"):
1582                 filesdirlist=os.listdir(checkdir+"/files")
1583
1584                 # recurse through files directory
1585                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1586                 while filesdirlist:
1587                         y = filesdirlist.pop(0)
1588                         relative_path = os.path.join(x, "files", y)
1589                         full_path = os.path.join(repodir, relative_path)
1590                         try:
1591                                 mystat = os.stat(full_path)
1592                         except OSError as oe:
1593                                 if oe.errno == 2:
1594                                         # don't worry about it.  it likely was removed via fix above.
1595                                         continue
1596                                 else:
1597                                         raise oe
1598                         if S_ISDIR(mystat.st_mode):
1599                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1600                                 if y == "CVS" or y == ".svn":
1601                                         continue
1602                                 for z in os.listdir(checkdir+"/files/"+y):
1603                                         if z == "CVS" or z == ".svn":
1604                                                 continue
1605                                         filesdirlist.append(y+"/"+z)
1606                         # Current policy is no files over 20 KiB, these are the checks. File size between
1607                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1608                         elif mystat.st_size > 61440:
1609                                 stats["file.size.fatal"] += 1
1610                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1611                         elif mystat.st_size > 20480:
1612                                 stats["file.size"] += 1
1613                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1614
1615                         index = repo_config.find_invalid_path_char(y)
1616                         if index != -1:
1617                                 y_relative = os.path.join(checkdir_relative, "files", y)
1618                                 if vcs is not None and not vcs_new_changed(y_relative):
1619                                         # If the file isn't in the VCS new or changed set, then
1620                                         # assume that it's an irrelevant temporary file (Manifest
1621                                         # entries are not generated for file names containing
1622                                         # prohibited characters). See bug #406877.
1623                                         index = -1
1624                         if index != -1:
1625                                 stats["file.name"] += 1
1626                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1627                                         (checkdir, y, y[index]))
1628         del mydigests
1629
1630         if check_changelog and "ChangeLog" not in checkdirlist:
1631                 stats["changelog.missing"]+=1
1632                 fails["changelog.missing"].append(x+"/ChangeLog")
1633
1634         musedict = {}
1635         #metadata.xml file check
1636         if "metadata.xml" not in checkdirlist:
1637                 stats["metadata.missing"]+=1
1638                 fails["metadata.missing"].append(x+"/metadata.xml")
1639         #metadata.xml parse check
1640         else:
1641                 metadata_bad = False
1642
1643                 # read metadata.xml into memory
1644                 try:
1645                         _metadata_xml = xml.etree.ElementTree.parse(
1646                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1647                                 encoding=_encodings['fs'], errors='strict'),
1648                                 parser=xml.etree.ElementTree.XMLParser(
1649                                         target=_MetadataTreeBuilder()))
1650                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1651                         metadata_bad = True
1652                         stats["metadata.bad"] += 1
1653                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1654                         del e
1655                 else:
1656                         # load USE flags from metadata.xml
1657                         try:
1658                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1659                         except portage.exception.ParseError as e:
1660                                 metadata_bad = True
1661                                 stats["metadata.bad"] += 1
1662                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1663                         else:
1664                                 for atom in chain(*musedict.values()):
1665                                         if atom is None:
1666                                                 continue
1667                                         try:
1668                                                 atom = Atom(atom)
1669                                         except InvalidAtom as e:
1670                                                 stats["metadata.bad"] += 1
1671                                                 fails["metadata.bad"].append(
1672                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1673                                         else:
1674                                                 if atom.cp != x:
1675                                                         stats["metadata.bad"] += 1
1676                                                         fails["metadata.bad"].append(
1677                                                                 ("%s/metadata.xml: Atom contains "
1678                                                                 "unexpected cat/pn: %s") % (x, atom))
1679
1680                         # Run other metadata.xml checkers
1681                         try:
1682                                 utilities.check_metadata(_metadata_xml, herd_base)
1683                         except (utilities.UnknownHerdsError, ) as e:
1684                                 metadata_bad = True
1685                                 stats["metadata.bad"] += 1
1686                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1687                                 del e
1688
1689                 #Only carry out if in package directory or check forced
1690                 if xmllint_capable and not metadata_bad:
1691                         # xmlint can produce garbage output even on success, so only dump
1692                         # the ouput when it fails.
1693                         st, out = repoman_getstatusoutput(
1694                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1695                                  (portage._shell_quote(metadata_dtd),
1696                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1697                         if st != os.EX_OK:
1698                                 print(red("!!!") + " metadata.xml is invalid:")
1699                                 for z in out.splitlines():
1700                                         print(red("!!! ")+z)
1701                                 stats["metadata.bad"]+=1
1702                                 fails["metadata.bad"].append(x+"/metadata.xml")
1703
1704                 del metadata_bad
1705         muselist = frozenset(musedict)
1706
1707         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1708         changelog_modified = changelog_path in modified_changelogs
1709
1710         # detect unused local USE-descriptions
1711         used_useflags = set()
1712
1713         for y in ebuildlist:
1714                 relative_path = os.path.join(x, y + ".ebuild")
1715                 full_path = os.path.join(repodir, relative_path)
1716                 ebuild_path = y + ".ebuild"
1717                 if repolevel < 3:
1718                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1719                 if repolevel < 2:
1720                         ebuild_path = os.path.join(catdir, ebuild_path)
1721                 ebuild_path = os.path.join(".", ebuild_path)
1722                 if check_changelog and not changelog_modified \
1723                         and ebuild_path in new_ebuilds:
1724                         stats['changelog.ebuildadded'] += 1
1725                         fails['changelog.ebuildadded'].append(relative_path)
1726
1727                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1728                         #ebuild not added to vcs
1729                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1730                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1731                 myesplit=portage.pkgsplit(y)
1732                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1733                             or pv_toolong_re.search(myesplit[1]) \
1734                             or pv_toolong_re.search(myesplit[2]):
1735                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1736                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1737                         continue
1738                 elif myesplit[0]!=pkgdir:
1739                         print(pkgdir,myesplit[0])
1740                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1741                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1742                         continue
1743
1744                 pkg = pkgs[y]
1745
1746                 if pkg.invalid:
1747                         allvalid = False
1748                         for k, msgs in pkg.invalid.items():
1749                                 for msg in msgs:
1750                                         stats[k] = stats[k] + 1
1751                                         fails[k].append("%s: %s" % (relative_path, msg))
1752                         continue
1753
1754                 myaux = pkg._metadata
1755                 eapi = myaux["EAPI"]
1756                 inherited = pkg.inherited
1757                 live_ebuild = live_eclasses.intersection(inherited)
1758
1759                 for k, v in myaux.items():
1760                         if not isinstance(v, basestring):
1761                                 continue
1762                         m = non_ascii_re.search(v)
1763                         if m is not None:
1764                                 stats["variable.invalidchar"] += 1
1765                                 fails["variable.invalidchar"].append(
1766                                         ("%s: %s variable contains non-ASCII " + \
1767                                         "character at position %s") % \
1768                                         (relative_path, k, m.start() + 1))
1769
1770                 if not src_uri_error:
1771                         # Check that URIs don't reference a server from thirdpartymirrors.
1772                         for uri in portage.dep.use_reduce( \
1773                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1774                                 contains_mirror = False
1775                                 for mirror, mirror_alias in thirdpartymirrors.items():
1776                                         if uri.startswith(mirror):
1777                                                 contains_mirror = True
1778                                                 break
1779                                 if not contains_mirror:
1780                                         continue
1781
1782                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1783                                 stats["SRC_URI.mirror"] += 1
1784                                 fails["SRC_URI.mirror"].append(
1785                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1786                                         (relative_path, mirror, new_uri))
1787
1788                 if myaux.get("PROVIDE"):
1789                         stats["virtual.oldstyle"]+=1
1790                         fails["virtual.oldstyle"].append(relative_path)
1791
1792                 for pos, missing_var in enumerate(missingvars):
1793                         if not myaux.get(missing_var):
1794                                 if catdir == "virtual" and \
1795                                         missing_var in ("HOMEPAGE", "LICENSE"):
1796                                         continue
1797                                 if live_ebuild and missing_var == "KEYWORDS":
1798                                         continue
1799                                 myqakey=missingvars[pos]+".missing"
1800                                 stats[myqakey]=stats[myqakey]+1
1801                                 fails[myqakey].append(x+"/"+y+".ebuild")
1802
1803                 if catdir == "virtual":
1804                         for var in ("HOMEPAGE", "LICENSE"):
1805                                 if myaux.get(var):
1806                                         myqakey = var + ".virtual"
1807                                         stats[myqakey] = stats[myqakey] + 1
1808                                         fails[myqakey].append(relative_path)
1809
1810                 # 14 is the length of DESCRIPTION=""
1811                 if len(myaux['DESCRIPTION']) > max_desc_len:
1812                         stats['DESCRIPTION.toolong'] += 1
1813                         fails['DESCRIPTION.toolong'].append(
1814                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1815                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1816
1817                 keywords = myaux["KEYWORDS"].split()
1818                 stable_keywords = []
1819                 for keyword in keywords:
1820                         if not keyword.startswith("~") and \
1821                                 not keyword.startswith("-"):
1822                                 stable_keywords.append(keyword)
1823                 if stable_keywords:
1824                         if ebuild_path in new_ebuilds and catdir != "virtual":
1825                                 stable_keywords.sort()
1826                                 stats["KEYWORDS.stable"] += 1
1827                                 fails["KEYWORDS.stable"].append(
1828                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1829                                                 " ".join(stable_keywords))
1830
1831                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1832                         if not kw.startswith("-"))
1833
1834                 previous_keywords = slot_keywords.get(pkg.slot)
1835                 if previous_keywords is None:
1836                         slot_keywords[pkg.slot] = set()
1837                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1838                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1839                         if dropped_keywords:
1840                                 stats["KEYWORDS.dropped"] += 1
1841                                 fails["KEYWORDS.dropped"].append(
1842                                         relative_path + ": %s" % \
1843                                         " ".join(sorted(dropped_keywords)))
1844
1845                 slot_keywords[pkg.slot].update(ebuild_archs)
1846
1847                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1848                 if "-*" in keywords:
1849                         haskeyword = False
1850                         for kw in keywords:
1851                                 if kw[0] == "~":
1852                                         kw = kw[1:]
1853                                 if kw in kwlist:
1854                                         haskeyword = True
1855                         if not haskeyword:
1856                                 stats["KEYWORDS.stupid"] += 1
1857                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1858
1859                 """
1860                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1861                 not be allowed to be marked stable
1862                 """
1863                 if live_ebuild and repo_config.name == "gentoo":
1864                         bad_stable_keywords = []
1865                         for keyword in keywords:
1866                                 if not keyword.startswith("~") and \
1867                                         not keyword.startswith("-"):
1868                                         bad_stable_keywords.append(keyword)
1869                                 del keyword
1870                         if bad_stable_keywords:
1871                                 stats["LIVEVCS.stable"] += 1
1872                                 fails["LIVEVCS.stable"].append(
1873                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1874                                                 bad_stable_keywords)
1875                         del bad_stable_keywords
1876
1877                         if keywords and not has_global_mask(pkg):
1878                                 stats["LIVEVCS.unmasked"] += 1
1879                                 fails["LIVEVCS.unmasked"].append(relative_path)
1880
1881                 if options.ignore_arches:
1882                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1883                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1884                 else:
1885                         arches = set()
1886                         for keyword in keywords:
1887                                 if (keyword[0]=="-"):
1888                                         continue
1889                                 elif (keyword[0]=="~"):
1890                                         arch = keyword[1:]
1891                                         if arch == "*":
1892                                                 for expanded_arch in profiles:
1893                                                         if expanded_arch == "**":
1894                                                                 continue
1895                                                         arches.add((keyword, expanded_arch,
1896                                                                 (expanded_arch, "~" + expanded_arch)))
1897                                         else:
1898                                                 arches.add((keyword, arch, (arch, keyword)))
1899                                 else:
1900                                         if keyword == "*":
1901                                                 for expanded_arch in profiles:
1902                                                         if expanded_arch == "**":
1903                                                                 continue
1904                                                         arches.add((keyword, expanded_arch,
1905                                                                 (expanded_arch,)))
1906                                         else:
1907                                                 arches.add((keyword, keyword, (keyword,)))
1908                         if not arches:
1909                                 # Use an empty profile for checking dependencies of
1910                                 # packages that have empty KEYWORDS.
1911                                 arches.add(('**', '**', ('**',)))
1912
1913                 unknown_pkgs = set()
1914                 baddepsyntax = False
1915                 badlicsyntax = False
1916                 badprovsyntax = False
1917                 catpkg = catdir+"/"+y
1918
1919                 inherited_java_eclass = "java-pkg-2" in inherited or \
1920                         "java-pkg-opt-2" in inherited
1921                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1922                 operator_tokens = set(["||", "(", ")"])
1923                 type_list, badsyntax = [], []
1924                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
1925                         mydepstr = myaux[mytype]
1926
1927                         buildtime = mytype in Package._buildtime_keys
1928                         runtime = mytype in Package._runtime_keys
1929                         token_class = None
1930                         if mytype.endswith("DEPEND"):
1931                                 token_class=portage.dep.Atom
1932
1933                         try:
1934                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1935                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1936                         except portage.exception.InvalidDependString as e:
1937                                 atoms = None
1938                                 badsyntax.append(str(e))
1939
1940                         if atoms and mytype.endswith("DEPEND"):
1941                                 if runtime and \
1942                                         "test?" in mydepstr.split():
1943                                         stats[mytype + '.suspect'] += 1
1944                                         fails[mytype + '.suspect'].append(relative_path + \
1945                                                 ": 'test?' USE conditional in %s" % mytype)
1946
1947                                 for atom in atoms:
1948                                         if atom == "||":
1949                                                 continue
1950
1951                                         # Skip dependency.unknown for blockers, so that we
1952                                         # don't encourage people to remove necessary blockers,
1953                                         # as discussed in bug #382407.
1954                                         if atom.blocker is None and \
1955                                                 not portdb.xmatch("match-all", atom) and \
1956                                                 not atom.cp.startswith("virtual/"):
1957                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1958
1959                                         is_blocker = atom.blocker
1960
1961                                         if catdir != "virtual":
1962                                                 if not is_blocker and \
1963                                                         atom.cp in suspect_virtual:
1964                                                         stats['virtual.suspect'] += 1
1965                                                         fails['virtual.suspect'].append(
1966                                                                 relative_path +
1967                                                                 ": %s: consider using '%s' instead of '%s'" %
1968                                                                 (mytype, suspect_virtual[atom.cp], atom))
1969
1970                                         if buildtime and \
1971                                                 not is_blocker and \
1972                                                 not inherited_java_eclass and \
1973                                                 atom.cp == "virtual/jdk":
1974                                                 stats['java.eclassesnotused'] += 1
1975                                                 fails['java.eclassesnotused'].append(relative_path)
1976                                         elif buildtime and \
1977                                                 not is_blocker and \
1978                                                 not inherited_wxwidgets_eclass and \
1979                                                 atom.cp == "x11-libs/wxGTK":
1980                                                 stats['wxwidgets.eclassnotused'] += 1
1981                                                 fails['wxwidgets.eclassnotused'].append(
1982                                                         (relative_path + ": %ss on x11-libs/wxGTK"
1983                                                         " without inheriting wxwidgets.eclass") % mytype)
1984                                         elif runtime:
1985                                                 if not is_blocker and \
1986                                                         atom.cp in suspect_rdepend:
1987                                                         stats[mytype + '.suspect'] += 1
1988                                                         fails[mytype + '.suspect'].append(
1989                                                                 relative_path + ": '%s'" % atom)
1990
1991                                         if atom.operator == "~" and \
1992                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1993                                                 qacat = 'dependency.badtilde'
1994                                                 stats[qacat] += 1
1995                                                 fails[qacat].append(
1996                                                         (relative_path + ": %s uses the ~ operator"
1997                                                          " with a non-zero revision:" + \
1998                                                          " '%s'") % (mytype, atom))
1999
2000                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2001
2002                 for m,b in zip(type_list, badsyntax):
2003                         if m.endswith("DEPEND"):
2004                                 qacat = "dependency.syntax"
2005                         else:
2006                                 qacat = m + ".syntax"
2007                         stats[qacat] += 1
2008                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2009
2010                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2011                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2012                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2013                 badlicsyntax = badlicsyntax > 0
2014                 badprovsyntax = badprovsyntax > 0
2015
2016                 # uselist checks - global
2017                 myuse = []
2018                 default_use = []
2019                 for myflag in myaux["IUSE"].split():
2020                         flag_name = myflag.lstrip("+-")
2021                         used_useflags.add(flag_name)
2022                         if myflag != flag_name:
2023                                 default_use.append(myflag)
2024                         if flag_name not in uselist:
2025                                 myuse.append(flag_name)
2026
2027                 # uselist checks - metadata
2028                 for mypos in range(len(myuse)-1,-1,-1):
2029                         if myuse[mypos] and (myuse[mypos] in muselist):
2030                                 del myuse[mypos]
2031
2032                 if default_use and not eapi_has_iuse_defaults(eapi):
2033                         for myflag in default_use:
2034                                 stats['EAPI.incompatible'] += 1
2035                                 fails['EAPI.incompatible'].append(
2036                                         (relative_path + ": IUSE defaults" + \
2037                                         " not supported with EAPI='%s':" + \
2038                                         " '%s'") % (eapi, myflag))
2039
2040                 for mypos in range(len(myuse)):
2041                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
2042                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
2043
2044                 # license checks
2045                 if not badlicsyntax:
2046                         # Parse the LICENSE variable, remove USE conditions and
2047                         # flatten it.
2048                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2049                         # Check each entry to ensure that it exists in PORTDIR's
2050                         # license directory.
2051                         for lic in licenses:
2052                                 # Need to check for "||" manually as no portage
2053                                 # function will remove it without removing values.
2054                                 if lic not in liclist and lic != "||":
2055                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
2056                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
2057                                 elif lic in liclist_deprecated:
2058                                         stats["LICENSE.deprecated"] = stats["LICENSE.deprecated"] + 1
2059                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2060
2061                 #keyword checks
2062                 myuse = myaux["KEYWORDS"].split()
2063                 for mykey in myuse:
2064                         if mykey not in ("-*", "*", "~*"):
2065                                 myskey = mykey
2066                                 if myskey[:1] == "-":
2067                                         myskey = myskey[1:]
2068                                 if myskey[:1] == "~":
2069                                         myskey = myskey[1:]
2070                                 if myskey not in kwlist:
2071                                         stats["KEYWORDS.invalid"] += 1
2072                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
2073                                 elif myskey not in profiles:
2074                                         stats["KEYWORDS.invalid"] += 1
2075                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
2076
2077                 #restrict checks
2078                 myrestrict = None
2079                 try:
2080                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2081                 except portage.exception.InvalidDependString as e:
2082                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
2083                         fails["RESTRICT.syntax"].append(
2084                                 "%s: RESTRICT: %s" % (relative_path, e))
2085                         del e
2086                 if myrestrict:
2087                         myrestrict = set(myrestrict)
2088                         mybadrestrict = myrestrict.difference(valid_restrict)
2089                         if mybadrestrict:
2090                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2091                                 for mybad in mybadrestrict:
2092                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
2093                 #REQUIRED_USE check
2094                 required_use = myaux["REQUIRED_USE"]
2095                 if required_use:
2096                         if not eapi_has_required_use(eapi):
2097                                 stats['EAPI.incompatible'] += 1
2098                                 fails['EAPI.incompatible'].append(
2099                                         relative_path + ": REQUIRED_USE" + \
2100                                         " not supported with EAPI='%s'" % (eapi,))
2101                         try:
2102                                 portage.dep.check_required_use(required_use, (),
2103                                         pkg.iuse.is_valid_flag, eapi=eapi)
2104                         except portage.exception.InvalidDependString as e:
2105                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2106                                 fails["REQUIRED_USE.syntax"].append(
2107                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2108                                 del e
2109
2110                 # Syntax Checks
2111                 relative_path = os.path.join(x, y + ".ebuild")
2112                 full_path = os.path.join(repodir, relative_path)
2113                 if not vcs_preserves_mtime:
2114                         if ebuild_path not in new_ebuilds and \
2115                                 ebuild_path not in modified_ebuilds:
2116                                 pkg.mtime = None
2117                 try:
2118                         # All ebuilds should have utf_8 encoding.
2119                         f = io.open(_unicode_encode(full_path,
2120                                 encoding=_encodings['fs'], errors='strict'),
2121                                 mode='r', encoding=_encodings['repo.content'])
2122                         try:
2123                                 for check_name, e in run_checks(f, pkg):
2124                                         stats[check_name] += 1
2125                                         fails[check_name].append(relative_path + ': %s' % e)
2126                         finally:
2127                                 f.close()
2128                 except UnicodeDecodeError:
2129                         # A file.UTF8 failure will have already been recorded above.
2130                         pass
2131
2132                 if options.force:
2133                         # The dep_check() calls are the most expensive QA test. If --force
2134                         # is enabled, there's no point in wasting time on these since the
2135                         # user is intent on forcing the commit anyway.
2136                         continue
2137
2138                 relevant_profiles = []
2139                 for keyword,arch,groups in arches:
2140
2141                         if arch not in profiles:
2142                                 # A missing profile will create an error further down
2143                                 # during the KEYWORDS verification.
2144                                 continue
2145                         relevant_profiles.extend((keyword, groups, prof)
2146                                 for prof in profiles[arch])
2147
2148                 def sort_key(item):
2149                         return item[2].sub_path
2150
2151                 relevant_profiles.sort(key=sort_key)
2152
2153                 for keyword, groups, prof in relevant_profiles:
2154
2155                                 if prof.status not in ("stable", "dev") or \
2156                                         prof.status == "dev" and not options.include_dev:
2157                                         continue
2158
2159                                 dep_settings = arch_caches.get(prof.sub_path)
2160                                 if dep_settings is None:
2161                                         dep_settings = portage.config(
2162                                                 config_profile_path=prof.abs_path,
2163                                                 config_incrementals=repoman_incrementals,
2164                                                 config_root=config_root,
2165                                                 local_config=False,
2166                                                 _unmatched_removal=options.unmatched_removal,
2167                                                 env=env)
2168                                         dep_settings.categories = repoman_settings.categories
2169                                         if options.without_mask:
2170                                                 dep_settings._mask_manager_obj = \
2171                                                         copy.deepcopy(dep_settings._mask_manager)
2172                                                 dep_settings._mask_manager._pmaskdict.clear()
2173                                         arch_caches[prof.sub_path] = dep_settings
2174
2175                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2176                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2177                                 if xcache is None:
2178                                         portdb.melt()
2179                                         portdb.freeze()
2180                                         xcache = portdb.xcache
2181                                         xcache.update(shared_xmatch_caches)
2182                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2183
2184                                 trees[root]["porttree"].settings = dep_settings
2185                                 portdb.settings = dep_settings
2186                                 portdb.xcache = xcache
2187                                 # for package.use.mask support inside dep_check
2188                                 dep_settings.setcpv(pkg)
2189                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2190                                 # just in case, prevent config.reset() from nuking these.
2191                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2192
2193                                 if not baddepsyntax:
2194                                         ismasked = not ebuild_archs or \
2195                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2196                                         if ismasked:
2197                                                 if not have_pmasked:
2198                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2199                                                                 pkg.cpv, pkg._metadata))
2200                                                 if options.ignore_masked:
2201                                                         continue
2202                                                 #we are testing deps for a masked package; give it some lee-way
2203                                                 suffix="masked"
2204                                                 matchmode = "minimum-all"
2205                                         else:
2206                                                 suffix=""
2207                                                 matchmode = "minimum-visible"
2208
2209                                         if not have_dev_keywords:
2210                                                 have_dev_keywords = \
2211                                                         bool(dev_keywords.intersection(keywords))
2212
2213                                         if prof.status == "dev":
2214                                                 suffix=suffix+"indev"
2215
2216                                         for mytype in Package._dep_keys:
2217
2218                                                 mykey = "dependency.bad" + suffix
2219                                                 myvalue = myaux[mytype]
2220                                                 if not myvalue:
2221                                                         continue
2222
2223                                                 success, atoms = portage.dep_check(myvalue, portdb,
2224                                                         dep_settings, use="all", mode=matchmode,
2225                                                         trees=trees)
2226
2227                                                 if success:
2228                                                         if atoms:
2229
2230                                                                 # Don't bother with dependency.unknown for
2231                                                                 # cases in which *DEPEND.bad is triggered.
2232                                                                 for atom in atoms:
2233                                                                         # dep_check returns all blockers and they
2234                                                                         # aren't counted for *DEPEND.bad, so we
2235                                                                         # ignore them here.
2236                                                                         if not atom.blocker:
2237                                                                                 unknown_pkgs.discard(
2238                                                                                         (mytype, atom.unevaluated_atom))
2239
2240                                                                 if not prof.sub_path:
2241                                                                         # old-style virtuals currently aren't
2242                                                                         # resolvable with empty profile, since
2243                                                                         # 'virtuals' mappings are unavailable
2244                                                                         # (it would be expensive to search
2245                                                                         # for PROVIDE in all ebuilds)
2246                                                                         atoms = [atom for atom in atoms if not \
2247                                                                                 (atom.cp.startswith('virtual/') and \
2248                                                                                 not portdb.cp_list(atom.cp))]
2249
2250                                                                 #we have some unsolvable deps
2251                                                                 #remove ! deps, which always show up as unsatisfiable
2252                                                                 atoms = [str(atom.unevaluated_atom) \
2253                                                                         for atom in atoms if not atom.blocker]
2254
2255                                                                 #if we emptied out our list, continue:
2256                                                                 if not atoms:
2257                                                                         continue
2258                                                                 stats[mykey]=stats[mykey]+1
2259                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2260                                                                         (relative_path, mytype, keyword,
2261                                                                         prof, repr(atoms)))
2262                                                 else:
2263                                                         stats[mykey]=stats[mykey]+1
2264                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2265                                                                 (relative_path, mytype, keyword,
2266                                                                 prof, repr(atoms)))
2267
2268                 if not baddepsyntax and unknown_pkgs:
2269                         type_map = {}
2270                         for mytype, atom in unknown_pkgs:
2271                                 type_map.setdefault(mytype, set()).add(atom)
2272                         for mytype, atoms in type_map.items():
2273                                 stats["dependency.unknown"] += 1
2274                                 fails["dependency.unknown"].append("%s: %s: %s" %
2275                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2276
2277         # check if there are unused local USE-descriptions in metadata.xml
2278         # (unless there are any invalids, to avoid noise)
2279         if allvalid:
2280                 for myflag in muselist.difference(used_useflags):
2281                         stats["metadata.warning"] += 1
2282                         fails["metadata.warning"].append(
2283                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2284                                 (x, myflag))
2285
2286 if options.if_modified == "y" and len(effective_scanlist) < 1:
2287         logging.warn("--if-modified is enabled, but no modified packages were found!")
2288
2289 if options.mode == "manifest":
2290         sys.exit(dofail)
2291
2292 #dofail will be set to 1 if we have failed in at least one non-warning category
2293 dofail=0
2294 #dowarn will be set to 1 if we tripped any warnings
2295 dowarn=0
2296 #dofull will be set if we should print a "repoman full" informational message
2297 dofull = options.mode != 'full'
2298
2299 for x in qacats:
2300         if not stats[x]:
2301                 continue
2302         dowarn = 1
2303         if x not in qawarnings:
2304                 dofail = 1
2305
2306 if dofail or \
2307         (dowarn and not (options.quiet or options.mode == "scan")):
2308         dofull = 0
2309
2310 # Save QA output so that it can be conveniently displayed
2311 # in $EDITOR while the user creates a commit message.
2312 # Otherwise, the user would not be able to see this output
2313 # once the editor has taken over the screen.
2314 qa_output = io.StringIO()
2315 style_file = ConsoleStyleFile(sys.stdout)
2316 if options.mode == 'commit' and \
2317         (not commitmessage or not commitmessage.strip()):
2318         style_file.write_listener = qa_output
2319 console_writer = StyleWriter(file=style_file, maxcol=9999)
2320 console_writer.style_listener = style_file.new_styles
2321
2322 f = formatter.AbstractFormatter(console_writer)
2323
2324 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2325
2326 style_file.flush()
2327 del console_writer, f, style_file
2328 qa_output = qa_output.getvalue()
2329 qa_output = qa_output.splitlines(True)
2330
2331 def grouplist(mylist,seperator="/"):
2332         """(list,seperator="/") -- Takes a list of elements; groups them into
2333         same initial element categories. Returns a dict of {base:[sublist]}
2334         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2335         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2336         mygroups={}
2337         for x in mylist:
2338                 xs=x.split(seperator)
2339                 if xs[0]==".":
2340                         xs=xs[1:]
2341                 if xs[0] not in mygroups:
2342                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2343                 else:
2344                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2345         return mygroups
2346
2347 suggest_ignore_masked = False
2348 suggest_include_dev = False
2349
2350 if have_pmasked and not (options.without_mask or options.ignore_masked):
2351         suggest_ignore_masked = True
2352 if have_dev_keywords and not options.include_dev:
2353         suggest_include_dev = True
2354
2355 if suggest_ignore_masked or suggest_include_dev:
2356         print()
2357         if suggest_ignore_masked:
2358                 print(bold("Note: use --without-mask to check " + \
2359                         "KEYWORDS on dependencies of masked packages"))
2360
2361         if suggest_include_dev:
2362                 print(bold("Note: use --include-dev (-d) to check " + \
2363                         "dependencies for 'dev' profiles"))
2364         print()
2365
2366 if options.mode != 'commit':
2367         if dofull:
2368                 print(bold("Note: type \"repoman full\" for a complete listing."))
2369         if dowarn and not dofail:
2370                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2371         elif not dofail:
2372                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2373         elif dofail:
2374                 print(bad("Please fix these important QA issues first."))
2375                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2376                 sys.exit(1)
2377 else:
2378         if dofail and can_force and options.force and not options.pretend:
2379                 print(green("RepoMan sez:") + \
2380                         " \"You want to commit even with these QA issues?\n" + \
2381                         "              I'll take it this time, but I'm not happy.\"\n")
2382         elif dofail:
2383                 if options.force and not can_force:
2384                         print(bad("The --force option has been disabled due to extraordinary issues."))
2385                 print(bad("Please fix these important QA issues first."))
2386                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2387                 sys.exit(1)
2388
2389         if options.pretend:
2390                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2391
2392         myunadded = []
2393         if vcs == "cvs":
2394                 try:
2395                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2396                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2397                 except SystemExit as e:
2398                         raise  # TODO propagate this
2399                 except:
2400                         err("Error retrieving CVS tree; exiting.")
2401         if vcs == "svn":
2402                 try:
2403                         with repoman_popen("svn status --no-ignore") as f:
2404                                 svnstatus = f.readlines()
2405                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2406                 except SystemExit as e:
2407                         raise  # TODO propagate this
2408                 except:
2409                         err("Error retrieving SVN info; exiting.")
2410         if vcs == "git":
2411                 # get list of files not under version control or missing
2412                 myf = repoman_popen("git ls-files --others")
2413                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2414                 myf.close()
2415         if vcs == "bzr":
2416                 try:
2417                         with repoman_popen("bzr status -S .") as f:
2418                                 bzrstatus = f.readlines()
2419                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2420                 except SystemExit as e:
2421                         raise  # TODO propagate this
2422                 except:
2423                         err("Error retrieving bzr info; exiting.")
2424         if vcs == "hg":
2425                 with repoman_popen("hg status --no-status --unknown .") as f:
2426                         myunadded = f.readlines()
2427                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2428
2429                 # Mercurial doesn't handle manually deleted files as removed from
2430                 # the repository, so the user need to remove them before commit,
2431                 # using "hg remove [FILES]"
2432                 with repoman_popen("hg status --no-status --deleted .") as f:
2433                         mydeleted = f.readlines()
2434                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2435
2436
2437         myautoadd=[]
2438         if myunadded:
2439                 for x in range(len(myunadded)-1,-1,-1):
2440                         xs=myunadded[x].split("/")
2441                         if xs[-1]=="files":
2442                                 print("!!! files dir is not added! Please correct this.")
2443                                 sys.exit(-1)
2444                         elif xs[-1]=="Manifest":
2445                                 # It's a manifest... auto add
2446                                 myautoadd+=[myunadded[x]]
2447                                 del myunadded[x]
2448
2449         if myunadded:
2450                 print(red("!!! The following files are in your local tree but are not added to the master"))
2451                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2452                 for x in myunadded:
2453                         print("   ",x)
2454                 print()
2455                 print()
2456                 sys.exit(1)
2457
2458         if vcs == "hg" and mydeleted:
2459                 print(red("!!! The following files are removed manually from your local tree but are not"))
2460                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2461                 for x in mydeleted:
2462                         print("   ",x)
2463                 print()
2464                 print()
2465                 sys.exit(1)
2466
2467         if vcs == "cvs":
2468                 mycvstree = cvstree.getentries("./", recursive=1)
2469                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2470                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2471                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2472                 bin_blob_pattern = re.compile("^-kb$")
2473                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2474                         recursive=1, basedir="./"))
2475
2476
2477         if vcs == "svn":
2478                 with repoman_popen("svn status") as f:
2479                         svnstatus = f.readlines()
2480                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2481                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2482                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2483
2484                 # Subversion expands keywords specified in svn:keywords properties.
2485                 with repoman_popen("svn propget -R svn:keywords") as f:
2486                         props = f.readlines()
2487                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2488                         for prop in props if " - " in prop)
2489
2490         elif vcs == "git":
2491                 with repoman_popen("git diff-index --name-only "
2492                         "--relative --diff-filter=M HEAD") as f:
2493                         mychanged = f.readlines()
2494                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2495
2496                 with repoman_popen("git diff-index --name-only "
2497                         "--relative --diff-filter=A HEAD") as f:
2498                         mynew = f.readlines()
2499                 mynew = ["./" + elem[:-1] for elem in mynew]
2500
2501                 with repoman_popen("git diff-index --name-only "
2502                         "--relative --diff-filter=D HEAD") as f:
2503                         myremoved = f.readlines()
2504                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2505
2506         if vcs == "bzr":
2507                 with repoman_popen("bzr status -S .") as f:
2508                         bzrstatus = f.readlines()
2509                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2510                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2511                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2512                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2513                 # Bazaar expands nothing.
2514
2515         if vcs == "hg":
2516                 with repoman_popen("hg status --no-status --modified .") as f:
2517                         mychanged = f.readlines()
2518                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2519
2520                 with repoman_popen("hg status --no-status --added .") as f:
2521                         mynew = f.readlines()
2522                 mynew = ["./" + elem.rstrip() for elem in mynew]
2523
2524                 with repoman_popen("hg status --no-status --removed .") as f:
2525                         myremoved = f.readlines()
2526                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2527
2528         if vcs:
2529                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2530                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2531                         print()
2532                         print("(Didn't find any changed files...)")
2533                         print()
2534                         sys.exit(1)
2535
2536         # Manifests need to be regenerated after all other commits, so don't commit
2537         # them now even if they have changed.
2538         mymanifests = set()
2539         myupdates = set()
2540         for f in mychanged + mynew:
2541                 if "Manifest" == os.path.basename(f):
2542                         mymanifests.add(f)
2543                 else:
2544                         myupdates.add(f)
2545         myupdates.difference_update(myremoved)
2546         myupdates = list(myupdates)
2547         mymanifests = list(mymanifests)
2548         myheaders = []
2549         mydirty = []
2550
2551         commitmessage = options.commitmsg
2552         if options.commitmsgfile:
2553                 try:
2554                         f = io.open(_unicode_encode(options.commitmsgfile,
2555                         encoding=_encodings['fs'], errors='strict'),
2556                         mode='r', encoding=_encodings['content'], errors='replace')
2557                         commitmessage = f.read()
2558                         f.close()
2559                         del f
2560                 except (IOError, OSError) as e:
2561                         if e.errno == errno.ENOENT:
2562                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2563                         else:
2564                                 raise
2565                 # We've read the content so the file is no longer needed.
2566                 commitmessagefile = None
2567         if not commitmessage or not commitmessage.strip():
2568                 try:
2569                         editor = os.environ.get("EDITOR")
2570                         if editor and utilities.editor_is_executable(editor):
2571                                 commitmessage = utilities.get_commit_message_with_editor(
2572                                         editor, message=qa_output)
2573                         else:
2574                                 commitmessage = utilities.get_commit_message_with_stdin()
2575                 except KeyboardInterrupt:
2576                         exithandler()
2577                 if not commitmessage or not commitmessage.strip():
2578                         print("* no commit message?  aborting commit.")
2579                         sys.exit(1)
2580         commitmessage = commitmessage.rstrip()
2581         changelog_msg = commitmessage
2582         portage_version = getattr(portage, "VERSION", None)
2583         if portage_version is None:
2584                 sys.stderr.write("Failed to insert portage version in message!\n")
2585                 sys.stderr.flush()
2586                 portage_version = "Unknown"
2587
2588         if vcs == "git":
2589                 # Use new footer only for git (see bug #438364).
2590                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2591                 if options.force:
2592                         commit_footer += "\nRepoMan-Options: --force"
2593                 if sign_manifests:
2594                         commit_footer += "\nManifest-Sign-Key: %s" % \
2595                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2596         else:
2597                 unameout = platform.system() + " "
2598                 if platform.system() in ["Darwin", "SunOS"]:
2599                         unameout += platform.processor()
2600                 else:
2601                         unameout += platform.machine()
2602                 commit_footer = "\n\n(Portage version: %s/%s/%s" % \
2603                         (portage_version, vcs, unameout)
2604                 if options.force:
2605                         commit_footer += ", RepoMan options: --force"
2606                 if sign_manifests:
2607                         commit_footer += ", signed Manifest commit with key %s" % \
2608                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2609                 else:
2610                         commit_footer += ", unsigned Manifest commit"
2611                 commit_footer += ")"
2612
2613         commitmessage += commit_footer
2614
2615         if options.echangelog in ('y', 'force'):
2616                 logging.info("checking for unmodified ChangeLog files")
2617                 committer_name = utilities.get_committer_name(env=repoman_settings)
2618                 for x in sorted(vcs_files_to_cps(
2619                         chain(myupdates, mymanifests, myremoved))):
2620                         catdir, pkgdir = x.split("/")
2621                         checkdir = repodir + "/" + x
2622                         checkdir_relative = ""
2623                         if repolevel < 3:
2624                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2625                         if repolevel < 2:
2626                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2627                         checkdir_relative = os.path.join(".", checkdir_relative)
2628
2629                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2630                         changelog_modified = changelog_path in modified_changelogs
2631                         if changelog_modified and options.echangelog != 'force':
2632                                 continue
2633
2634                         # get changes for this package
2635                         cdrlen = len(checkdir_relative)
2636                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2637                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2638                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2639
2640                         # Skip ChangeLog generation if only the Manifest was modified,
2641                         # as discussed in bug #398009.
2642                         nontrivial_cl_files = set()
2643                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2644                         nontrivial_cl_files.difference_update(['Manifest'])
2645                         if not nontrivial_cl_files and options.echangelog != 'force':
2646                                 continue
2647
2648                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2649                                 committer_name, changelog_msg,
2650                                 os.path.join(repodir, 'skel.ChangeLog'),
2651                                 catdir, pkgdir,
2652                                 new=clnew, removed=clremoved, changed=clchanged,
2653                                 pretend=options.pretend)
2654                         if new_changelog is None:
2655                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2656                                         level=logging.ERROR, noiselevel=-1)
2657                                 sys.exit(1)
2658
2659                         # if the ChangeLog was just created, add it to vcs
2660                         if new_changelog:
2661                                 myautoadd.append(changelog_path)
2662                                 # myautoadd is appended to myupdates below
2663                         else:
2664                                 myupdates.append(changelog_path)
2665
2666                         if options.ask and not options.pretend:
2667                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2668                                 repoman_settings["O"] = checkdir
2669                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2670
2671         if myautoadd:
2672                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2673                 add_cmd = [vcs, "add"]
2674                 add_cmd += myautoadd
2675                 if options.pretend:
2676                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2677                                 noiselevel=-1)
2678                 else:
2679                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2680                                 # Python 3.1 produces the following TypeError if raw bytes are
2681                                 # passed to subprocess.call():
2682                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2683                                 #     errread, errwrite)
2684                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2685                                 #     raise child_exception
2686                                 # TypeError: expected an object with the buffer interface
2687                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2688                         retcode = subprocess.call(add_cmd)
2689                         if retcode != os.EX_OK:
2690                                 logging.error(
2691                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2692                                 sys.exit(retcode)
2693
2694                 myupdates += myautoadd
2695
2696         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2697
2698         if vcs not in ('cvs', 'svn'):
2699                 # With git, bzr and hg, there's never any keyword expansion, so
2700                 # there's no need to regenerate manifests and all files will be
2701                 # committed in one big commit at the end.
2702                 print()
2703         elif not repo_config.thin_manifest:
2704                 if vcs == 'cvs':
2705                         headerstring = "'\$(Header|Id).*\$'"
2706                 elif vcs == "svn":
2707                         svn_keywords = dict((k.lower(), k) for k in [
2708                                         "Rev",
2709                                         "Revision",
2710                                         "LastChangedRevision",
2711                                         "Date",
2712                                         "LastChangedDate",
2713                                         "Author",
2714                                         "LastChangedBy",
2715                                         "URL",
2716                                         "HeadURL",
2717                                         "Id",
2718                                         "Header",
2719                         ])
2720
2721                 for myfile in myupdates:
2722
2723                         # for CVS, no_expansion contains files that are excluded from expansion
2724                         if vcs == "cvs":
2725                                 if myfile in no_expansion:
2726                                         continue
2727
2728                         # for SVN, expansion contains files that are included in expansion
2729                         elif vcs == "svn":
2730                                 if myfile not in expansion:
2731                                         continue
2732
2733                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2734                                 enabled_keywords = []
2735                                 for k in expansion[myfile]:
2736                                         keyword = svn_keywords.get(k.lower())
2737                                         if keyword is not None:
2738                                                 enabled_keywords.append(keyword)
2739
2740                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2741
2742                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2743                                 portage._shell_quote(myfile))
2744                         if myout[0] == 0:
2745                                 myheaders.append(myfile)
2746
2747                 print("%s have headers that will change." % green(str(len(myheaders))))
2748                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2749
2750         logging.info("myupdates: %s", myupdates)
2751         logging.info("myheaders: %s", myheaders)
2752
2753         if options.ask and userquery('Commit changes?', True) != 'Yes':
2754                 print("* aborting commit.")
2755                 sys.exit(128 + signal.SIGINT)
2756
2757         # Handle the case where committed files have keywords which
2758         # will change and need a priming commit before the Manifest
2759         # can be committed.
2760         if (myupdates or myremoved) and myheaders:
2761                 myfiles = myupdates + myremoved
2762                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2763                 mymsg = os.fdopen(fd, "wb")
2764                 mymsg.write(_unicode_encode(commitmessage))
2765                 mymsg.close()
2766
2767                 print()
2768                 print(green("Using commit message:"))
2769                 print(green("------------------------------------------------------------------------------"))
2770                 print(commitmessage)
2771                 print(green("------------------------------------------------------------------------------"))
2772                 print()
2773
2774                 # Having a leading ./ prefix on file paths can trigger a bug in
2775                 # the cvs server when committing files to multiple directories,
2776                 # so strip the prefix.
2777                 myfiles = [f.lstrip("./") for f in myfiles]
2778
2779                 commit_cmd = [vcs]
2780                 commit_cmd.extend(vcs_global_opts)
2781                 commit_cmd.append("commit")
2782                 commit_cmd.extend(vcs_local_opts)
2783                 commit_cmd.extend(["-F", commitmessagefile])
2784                 commit_cmd.extend(myfiles)
2785
2786                 try:
2787                         if options.pretend:
2788                                 print("(%s)" % (" ".join(commit_cmd),))
2789                         else:
2790                                 retval = spawn(commit_cmd, env=os.environ)
2791                                 if retval != os.EX_OK:
2792                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2793                                                 "error code: %s\n") % (vcs, retval),
2794                                                 level=logging.ERROR, noiselevel=-1)
2795                                         sys.exit(retval)
2796                 finally:
2797                         try:
2798                                 os.unlink(commitmessagefile)
2799                         except OSError:
2800                                 pass
2801
2802         # Setup the GPG commands
2803         def gpgsign(filename):
2804                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2805                 if gpgcmd is None:
2806                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2807                                 " Is make.globals missing?")
2808                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2809                         "PORTAGE_GPG_KEY" not in repoman_settings:
2810                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2811                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2812                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2813                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2814                                         os.path.expanduser("~/.gnupg")
2815                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2816                                         % repoman_settings["PORTAGE_GPG_DIR"])
2817                         else:
2818                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2819                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2820                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2821                                 raise portage.exception.InvalidLocation(
2822                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2823                                         repoman_settings["PORTAGE_GPG_DIR"])
2824                 gpgvars = {"FILE": filename}
2825                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2826                         v = repoman_settings.get(k)
2827                         if v is not None:
2828                                 gpgvars[k] = v
2829                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2830                 if options.pretend:
2831                         print("("+gpgcmd+")")
2832                 else:
2833                         # Encode unicode manually for bug #310789.
2834                         gpgcmd = portage.util.shlex_split(gpgcmd)
2835                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2836                                 # Python 3.1 does not support bytes in Popen args.
2837                                 gpgcmd = [_unicode_encode(arg,
2838                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2839                         rValue = subprocess.call(gpgcmd)
2840                         if rValue == os.EX_OK:
2841                                 os.rename(filename+".asc", filename)
2842                         else:
2843                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2844
2845         def need_signature(filename):
2846                 try:
2847                         with open(_unicode_encode(filename,
2848                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2849                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2850                 except IOError as e:
2851                         if e.errno in (errno.ENOENT, errno.ESTALE):
2852                                 return False
2853                         raise
2854
2855         # When files are removed and re-added, the cvs server will put /Attic/
2856         # inside the $Header path. This code detects the problem and corrects it
2857         # so that the Manifest will generate correctly. See bug #169500.
2858         # Use binary mode in order to avoid potential character encoding issues.
2859         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2860         attic_str = b'/Attic/'
2861         attic_replace = b'/'
2862         for x in myheaders:
2863                 f = open(_unicode_encode(x,
2864                         encoding=_encodings['fs'], errors='strict'),
2865                         mode='rb')
2866                 mylines = f.readlines()
2867                 f.close()
2868                 modified = False
2869                 for i, line in enumerate(mylines):
2870                         if cvs_header_re.match(line) is not None and \
2871                                 attic_str in line:
2872                                 mylines[i] = line.replace(attic_str, attic_replace)
2873                                 modified = True
2874                 if modified:
2875                         portage.util.write_atomic(x, b''.join(mylines),
2876                                 mode='wb')
2877
2878         if repolevel == 1:
2879                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2880                         "doing the entire repository.\"\n")
2881
2882         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2883
2884                 for x in sorted(vcs_files_to_cps(
2885                         chain(myupdates, myremoved, mymanifests))):
2886                         repoman_settings["O"] = os.path.join(repodir, x)
2887                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2888
2889         signed = False
2890         if sign_manifests:
2891                 signed = True
2892                 try:
2893                         for x in sorted(vcs_files_to_cps(
2894                                 chain(myupdates, myremoved, mymanifests))):
2895                                 repoman_settings["O"] = os.path.join(repodir, x)
2896                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2897                                 if not need_signature(manifest_path):
2898                                         continue
2899                                 gpgsign(manifest_path)
2900                 except portage.exception.PortageException as e:
2901                         portage.writemsg("!!! %s\n" % str(e))
2902                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2903                         signed = False
2904
2905         if vcs == 'git':
2906                 # It's not safe to use the git commit -a option since there might
2907                 # be some modified files elsewhere in the working tree that the
2908                 # user doesn't want to commit. Therefore, call git update-index
2909                 # in order to ensure that the index is updated with the latest
2910                 # versions of all new and modified files in the relevant portion
2911                 # of the working tree.
2912                 myfiles = mymanifests + myupdates
2913                 myfiles.sort()
2914                 update_index_cmd = ["git", "update-index"]
2915                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2916                 if options.pretend:
2917                         print("(%s)" % (" ".join(update_index_cmd),))
2918                 else:
2919                         retval = spawn(update_index_cmd, env=os.environ)
2920                         if retval != os.EX_OK:
2921                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2922                                         "error code: %s\n") % (vcs, retval),
2923                                         level=logging.ERROR, noiselevel=-1)
2924                                 sys.exit(retval)
2925
2926         if True:
2927
2928                 myfiles = mymanifests[:]
2929                 # If there are no header (SVN/CVS keywords) changes in
2930                 # the files, this Manifest commit must include the
2931                 # other (yet uncommitted) files.
2932                 if not myheaders:
2933                         myfiles += myupdates
2934                         myfiles += myremoved
2935                 myfiles.sort()
2936
2937                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2938                 mymsg = os.fdopen(fd, "wb")
2939                 mymsg.write(_unicode_encode(commitmessage))
2940                 mymsg.close()
2941
2942                 commit_cmd = []
2943                 if options.pretend and vcs is None:
2944                         # substitute a bogus value for pretend output
2945                         commit_cmd.append("cvs")
2946                 else:
2947                         commit_cmd.append(vcs)
2948                 commit_cmd.extend(vcs_global_opts)
2949                 commit_cmd.append("commit")
2950                 commit_cmd.extend(vcs_local_opts)
2951                 if vcs == "hg":
2952                         commit_cmd.extend(["--logfile", commitmessagefile])
2953                         commit_cmd.extend(myfiles)
2954                 else:
2955                         commit_cmd.extend(["-F", commitmessagefile])
2956                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2957
2958                 try:
2959                         if options.pretend:
2960                                 print("(%s)" % (" ".join(commit_cmd),))
2961                         else:
2962                                 retval = spawn(commit_cmd, env=os.environ)
2963                                 if retval != os.EX_OK:
2964
2965                                         if repo_config.sign_commit and vcs == 'git' and \
2966                                                 not git_supports_gpg_sign():
2967                                                 # Inform user that newer git is needed (bug #403323).
2968                                                 logging.error(
2969                                                         "Git >=1.7.9 is required for signed commits!")
2970
2971                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2972                                                 "error code: %s\n") % (vcs, retval),
2973                                                 level=logging.ERROR, noiselevel=-1)
2974                                         sys.exit(retval)
2975                 finally:
2976                         try:
2977                                 os.unlink(commitmessagefile)
2978                         except OSError:
2979                                 pass
2980
2981         print()
2982         if vcs:
2983                 print("Commit complete.")
2984         else:
2985                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2986         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2987 sys.exit(0)
2988