bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import codecs
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27 from itertools import chain
  28 from stat import S_ISDIR
  29
  30 try:
  31         from urllib.parse import urlparse
  32 except ImportError:
  33         from urlparse import urlparse
  34
  35 from os import path as osp
  36 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  37 sys.path.insert(0, pym_path)
  38 import portage
  39 portage._internal_caller = True
  40 portage._disable_legacy_globals()
  41
  42 try:
  43         import xml.etree.ElementTree
  44         from xml.parsers.expat import ExpatError
  45 except (SystemExit, KeyboardInterrupt):
  46         raise
  47 except (ImportError, SystemError, RuntimeError, Exception):
  48         # broken or missing xml support
  49         # http://bugs.python.org/issue14988
  50         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  51         from portage.output import EOutput
  52         out = EOutput()
  53         for line in msg:
  54                 out.eerror(line)
  55         sys.exit(1)
  56
  57 from portage import os
  58 from portage import _encodings
  59 from portage import _unicode_encode
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 import portage.repository.config
  69 from portage import cvstree, normalize_path
  70 from portage import util
  71 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  72         ParseError, PermissionDenied)
  73 from portage.dep import Atom
  74 from portage.process import find_binary, spawn
  75 from portage.output import bold, create_color_func, \
  76         green, nocolor, red
  77 from portage.output import ConsoleStyleFile, StyleWriter
  78 from portage.util import writemsg_level
  79 from portage.package.ebuild.digestgen import digestgen
  80 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  81
  82 if sys.hexversion >= 0x3000000:
  83         basestring = str
  84
  85 util.initialize_logger()
  86
  87 # 14 is the length of DESCRIPTION=""
  88 max_desc_len = 100
  89 allowed_filename_chars="a-zA-Z0-9._-+:"
  90 pv_toolong_re = re.compile(r'[0-9]{19,}')
  91 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  92 bad = create_color_func("BAD")
  93
  94 # A sane umask is needed for files that portage creates.
  95 os.umask(0o22)
  96 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  97 # behave incrementally.
  98 repoman_incrementals = tuple(x for x in \
  99         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 100 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 101 repoman_settings = portage.config(config_root=config_root, local_config=False)
 102
 103 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 104         repoman_settings.get('TERM') == 'dumb' or \
 105         not sys.stdout.isatty():
 106         nocolor()
 107
 108 def warn(txt):
 109         print("repoman: " + txt)
 110
 111 def err(txt):
 112         warn(txt)
 113         sys.exit(1)
 114
 115 def exithandler(signum=None, frame=None):
 116         logging.fatal("Interrupted; exiting...")
 117         if signum is None:
 118                 sys.exit(1)
 119         else:
 120                 sys.exit(128 + signum)
 121
 122 signal.signal(signal.SIGINT, exithandler)
 123
 124 def ParseArgs(argv, qahelp):
 125         """This function uses a customized optionParser to parse command line arguments for repoman
 126         Args:
 127           argv - a sequence of command line arguments
 128                 qahelp - a dict of qa warning to help message
 129         Returns:
 130           (opts, args), just like a call to parser.parse_args()
 131         """
 132
 133         argv = portage._decode_argv(argv)
 134
 135         modes = {
 136                 'commit' : 'Run a scan then commit changes',
 137                 'ci' : 'Run a scan then commit changes',
 138                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 139                 'full' : 'Scan directory tree and print all issues (not a summary)',
 140                 'help' : 'Show this screen',
 141                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 142                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 143                 'scan' : 'Scan directory tree for QA issues'
 144         }
 145
 146         mode_keys = list(modes)
 147         mode_keys.sort()
 148
 149         parser = optparse.OptionParser(usage="%prog [options] [mode]",
 150                 description="Modes: %s" % " | ".join(mode_keys),
 151                 epilog="For more help consult the man page.")
 152
 153         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 154                 help='Request a confirmation before commiting')
 155
 156         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 157                 help='specify a commit message on the command line')
 158
 159         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 160                 help='specify a path to a file that contains a commit message')
 161
 162         parser.add_option('--digest',
 163                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 164                 help='Automatically update Manifest digests for modified files')
 165
 166         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 167                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 168
 169         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 170                 help='do not print unnecessary messages')
 171
 172         parser.add_option(
 173                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 174                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 175                 'regardless of modification if \'force\' is specified)')
 176
 177         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 178                 help='Commit with QA violations')
 179
 180         parser.add_option('--vcs', dest='vcs',
 181                 help='Force using specific VCS instead of autodetection')
 182
 183         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 184                 help='be very verbose in output', default=0)
 185
 186         parser.add_option('-V', '--version', dest='version', action='store_true',
 187                 help='show version info')
 188
 189         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 190                 default=False, help='forces the metadata.xml parse check to be carried out')
 191
 192         parser.add_option(
 193                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 194                 metavar="<y|n>",
 195                 help='only check packages that have uncommitted modifications')
 196
 197         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 198                 default=False, help='ignore arch-specific failures (where arch != host)')
 199
 200         parser.add_option("--ignore-default-opts",
 201                 action="store_true",
 202                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 203
 204         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 205                 default=False, help='ignore masked packages (not allowed with commit mode)')
 206
 207         parser.add_option('--include-arches', dest='include_arches',
 208                 metavar='ARCHES', action='append',
 209                 help='A space separated list of arches used to '
 210                 'filter the selection of profiles for dependency checks')
 211
 212         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 213                 default=False, help='include dev profiles in dependency checks')
 214
 215         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 216                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 217
 218         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 219                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 220
 221         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 222                 help='specify which mode repoman will run in (default=full)')
 223
 224         opts, args = parser.parse_args(argv[1:])
 225
 226         if not opts.ignore_default_opts:
 227                 default_opts = portage.util.shlex_split(
 228                         repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
 229                 if default_opts:
 230                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 231
 232         if opts.mode == 'help':
 233                 parser.print_help(short=False)
 234
 235         for arg in args:
 236                 if arg in modes:
 237                         if not opts.mode:
 238                                 opts.mode = arg
 239                                 break
 240                 else:
 241                         parser.error("invalid mode: %s" % arg)
 242
 243         if not opts.mode:
 244                 opts.mode = 'full'
 245
 246         if opts.mode == 'ci':
 247                 opts.mode = 'commit'  # backwards compat shortcut
 248
 249         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 250         for val in range(opts.verbosity):
 251                 logger = logging.getLogger()
 252                 logger.setLevel(logger.getEffectiveLevel() - 10)
 253
 254         for val in range(opts.quiet):
 255                 logger = logging.getLogger()
 256                 logger.setLevel(logger.getEffectiveLevel() + 10)
 257
 258         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 259                 if opts.ignore_masked:
 260                         opts.ignore_masked = False
 261                         logging.warn('Commit mode automatically disables --ignore-masked')
 262                 if opts.without_mask:
 263                         opts.without_mask = False
 264                         logging.warn('Commit mode automatically disables --without-mask')
 265
 266         return (opts, args)
 267
 268 qahelp = {
 269         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 270         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 271         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 272         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 273         "changelog.missing": "Missing ChangeLog files",
 274         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 275         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 276         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 277         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 278         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 279         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 280         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 281         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 282         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 283         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 284         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 285         "file.size": "Files in the files directory must be under 20 KiB",
 286         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 287         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 288         "file.UTF8": "File is not UTF8 compliant",
 289         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 290         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 291         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 292         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 293         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 294         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 295         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 296         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 297         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 298         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 299         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 300         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 301         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 302         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 303         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 304         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 305         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 306         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 307         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 308         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 309         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 310         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 311         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 312         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 313         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 314         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 315         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 316         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 317         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 318         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 319         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 320         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 321         "variable.readonly": "Assigning a readonly variable",
 322         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 323         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 324         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 325         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 326         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 327         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 328         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 329         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 330         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 331         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 332         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 333         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 334         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 335         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 336         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 337         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 338         "ebuild.badheader": "This ebuild has a malformed header",
 339         "manifest.bad": "Manifest has missing or incorrect digests",
 340         "metadata.missing": "Missing metadata.xml files",
 341         "metadata.bad": "Bad metadata.xml files",
 342         "metadata.warning": "Warnings in metadata.xml files",
 343         "portage.internal": "The ebuild uses an internal Portage function or variable",
 344         "repo.eapi.banned": "The ebuild uses an EAPI which is banned by the repository's metadata/layout.conf settings",
 345         "repo.eapi.deprecated": "The ebuild uses an EAPI which is deprecated by the repository's metadata/layout.conf settings",
 346         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 347         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 348         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 349         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 350 }
 351
 352 qacats = list(qahelp)
 353 qacats.sort()
 354
 355 qawarnings = set((
 356 "changelog.missing",
 357 "changelog.notadded",
 358 "dependency.unknown",
 359 "digest.assumed",
 360 "digest.unused",
 361 "ebuild.notadded",
 362 "ebuild.nesteddie",
 363 "dependency.badmasked",
 364 "dependency.badindev",
 365 "dependency.badmaskedindev",
 366 "dependency.badtilde",
 367 "DESCRIPTION.toolong",
 368 "EAPI.deprecated",
 369 "HOMEPAGE.virtual",
 370 "LICENSE.deprecated",
 371 "LICENSE.virtual",
 372 "KEYWORDS.dropped",
 373 "KEYWORDS.stupid",
 374 "KEYWORDS.missing",
 375 "PDEPEND.suspect",
 376 "RDEPEND.implicit",
 377 "RDEPEND.suspect",
 378 "virtual.suspect",
 379 "RESTRICT.invalid",
 380 "ebuild.minorsyn",
 381 "ebuild.badheader",
 382 "ebuild.patches",
 383 "file.size",
 384 "inherit.unused",
 385 "inherit.deprecated",
 386 "java.eclassesnotused",
 387 "wxwidgets.eclassnotused",
 388 "metadata.warning",
 389 "portage.internal",
 390 "repo.eapi.deprecated",
 391 "usage.obsolete",
 392 "upstream.workaround",
 393 "LIVEVCS.stable",
 394 "LIVEVCS.unmasked",
 395 ))
 396
 397 if portage.const._ENABLE_INHERIT_CHECK:
 398         # This is experimental, so it's non-fatal.
 399         qawarnings.add("inherit.missing")
 400
 401 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 402
 403 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 404 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 405 allvars.update(Package.metadata_keys)
 406 allvars = sorted(allvars)
 407 commitmessage = None
 408 for x in missingvars:
 409         x += ".missing"
 410         if x not in qacats:
 411                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 412                 qacats.append(x)
 413                 qawarnings.add(x)
 414
 415 valid_restrict = frozenset(["binchecks", "bindist",
 416         "fetch", "installsources", "mirror", "preserve-libs",
 417         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 418
 419 live_eclasses = frozenset([
 420         "bzr",
 421         "cvs",
 422         "darcs",
 423         "git",
 424         "git-2",
 425         "mercurial",
 426         "subversion",
 427         "tla",
 428 ])
 429
 430 suspect_rdepend = frozenset([
 431         "app-arch/cabextract",
 432         "app-arch/rpm2targz",
 433         "app-doc/doxygen",
 434         "dev-lang/nasm",
 435         "dev-lang/swig",
 436         "dev-lang/yasm",
 437         "dev-perl/extutils-pkgconfig",
 438         "dev-util/byacc",
 439         "dev-util/cmake",
 440         "dev-util/ftjam",
 441         "dev-util/gperf",
 442         "dev-util/gtk-doc",
 443         "dev-util/gtk-doc-am",
 444         "dev-util/intltool",
 445         "dev-util/jam",
 446         "dev-util/pkg-config-lite",
 447         "dev-util/pkgconf",
 448         "dev-util/pkgconfig",
 449         "dev-util/pkgconfig-openbsd",
 450         "dev-util/scons",
 451         "dev-util/unifdef",
 452         "dev-util/yacc",
 453         "media-gfx/ebdftopcf",
 454         "sys-apps/help2man",
 455         "sys-devel/autoconf",
 456         "sys-devel/automake",
 457         "sys-devel/bin86",
 458         "sys-devel/bison",
 459         "sys-devel/dev86",
 460         "sys-devel/flex",
 461         "sys-devel/m4",
 462         "sys-devel/pmake",
 463         "virtual/linux-sources",
 464         "virtual/pkgconfig",
 465         "x11-misc/bdftopcf",
 466         "x11-misc/imake",
 467 ])
 468
 469 suspect_virtual = {
 470         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 471         "dev-util/pkgconf":"virtual/pkgconfig",
 472         "dev-util/pkgconfig":"virtual/pkgconfig",
 473         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 474         "dev-libs/libusb":"virtual/libusb",
 475         "dev-libs/libusbx":"virtual/libusb",
 476         "dev-libs/libusb-compat":"virtual/libusb",
 477 }
 478
 479 metadata_xml_encoding = 'UTF-8'
 480 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 481         (metadata_xml_encoding,)
 482 metadata_doctype_name = 'pkgmetadata'
 483 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 484 # force refetch if the local copy creation time is older than this
 485 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 486
 487 # file.executable
 488 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 489
 490 options, arguments = ParseArgs(sys.argv, qahelp)
 491
 492 if options.version:
 493         print("Portage", portage.VERSION)
 494         sys.exit(0)
 495
 496 # Set this to False when an extraordinary issue (generally
 497 # something other than a QA issue) makes it impossible to
 498 # commit (like if Manifest generation fails).
 499 can_force = True
 500
 501 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 502 if portdir is None:
 503         sys.exit(1)
 504
 505 myreporoot = os.path.basename(portdir_overlay)
 506 myreporoot += mydir[len(portdir_overlay):]
 507
 508 if options.vcs:
 509         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 510                 vcs = options.vcs
 511         else:
 512                 vcs = None
 513 else:
 514         vcses = utilities.FindVCS()
 515         if len(vcses) > 1:
 516                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 517                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 518                 sys.exit(1)
 519         elif vcses:
 520                 vcs = vcses[0]
 521         else:
 522                 vcs = None
 523
 524 if options.if_modified == "y" and vcs is None:
 525         logging.info("Not in a version controlled repository; "
 526                 "disabling --if-modified.")
 527         options.if_modified = "n"
 528
 529 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 530 vcs_preserves_mtime = vcs in ('cvs',)
 531
 532 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 533 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 534 if vcs_global_opts is None:
 535         if vcs in ('cvs', 'svn'):
 536                 vcs_global_opts = "-q"
 537         else:
 538                 vcs_global_opts = ""
 539 vcs_global_opts = vcs_global_opts.split()
 540
 541 if options.mode == 'commit' and not options.pretend and not vcs:
 542         logging.info("Not in a version controlled repository; enabling pretend mode.")
 543         options.pretend = True
 544
 545 # Ensure that current repository is in the list of enabled repositories.
 546 repodir = os.path.realpath(portdir_overlay)
 547 try:
 548         repoman_settings.repositories.get_repo_for_location(repodir)
 549 except KeyError:
 550         repo_name = portage.repository.config.RepoConfig._read_valid_repo_name(portdir_overlay)[0]
 551         layout_conf_data = portage.repository.config.parse_layout_conf(portdir_overlay)[0]
 552         if layout_conf_data['repo-name']:
 553                 repo_name = layout_conf_data['repo-name']
 554         tmp_conf_file = io.StringIO(textwrap.dedent("""
 555                 [%s]
 556                 location = %s
 557                 """) % (repo_name, portdir_overlay))
 558         # Ensure that the repository corresponding to $PWD overrides a
 559         # repository of the same name referenced by the existing PORTDIR
 560         # or PORTDIR_OVERLAY settings.
 561         repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 562                 (repoman_settings.get('PORTDIR_OVERLAY', ''),
 563                 portage._shell_quote(portdir_overlay))
 564         repositories = portage.repository.config.load_repository_config(repoman_settings, extra_files=[tmp_conf_file])
 565         # We have to call the config constructor again so that attributes
 566         # dependent on config.repositories are initialized correctly.
 567         repoman_settings = portage.config(config_root=config_root, local_config=False, repositories=repositories)
 568
 569 root = repoman_settings['EROOT']
 570 trees = {
 571         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 572 }
 573 portdb = trees[root]['porttree'].dbapi
 574
 575 # Constrain dependency resolution to the master(s)
 576 # that are specified in layout.conf.
 577 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 578 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 579 portdir = portdb.porttrees[0]
 580 commit_env = os.environ.copy()
 581 # list() is for iteration on a copy.
 582 for repo in list(repoman_settings.repositories):
 583         # all paths are canonical
 584         if repo.location not in repo_config.eclass_db.porttrees:
 585                 del repoman_settings.repositories[repo.name]
 586
 587 if repo_config.allow_provide_virtual:
 588         qawarnings.add("virtual.oldstyle")
 589
 590 if repo_config.sign_commit:
 591         if vcs == 'git':
 592                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 593                 # the commit arguments. If key_id is unspecified, then it must be
 594                 # configured by `git config user.signingkey key_id`.
 595                 vcs_local_opts.append("--gpg-sign")
 596                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 597                         # Pass GNUPGHOME to git for bug #462362.
 598                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 599
 600                 # Pass GPG_TTY to git for bug #477728.
 601                 try:
 602                         commit_env["GPG_TTY"] = os.ttyname(sys.stdin.fileno())
 603                 except OSError:
 604                         pass
 605
 606 # In order to disable manifest signatures, repos may set
 607 # "sign-manifests = false" in metadata/layout.conf. This
 608 # can be used to prevent merge conflicts like those that
 609 # thin-manifests is designed to prevent.
 610 sign_manifests = "sign" in repoman_settings.features and \
 611         repo_config.sign_manifest
 612
 613 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 614         options.mode in ("commit",) and not sign_manifests:
 615         msg = ("The '%s' repository has manifest signatures enabled, "
 616         "but FEATURES=sign is currently disabled. In order to avoid this "
 617         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 618         "repositories can disable manifest signatures by setting "
 619         "'sign-manifests = false' in metadata/layout.conf.") % \
 620         (repo_config.name,)
 621         for line in textwrap.wrap(msg, 60):
 622                 logging.warn(line)
 623
 624 if sign_manifests and options.mode in ("commit",) and \
 625         repoman_settings.get("PORTAGE_GPG_KEY") and \
 626         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 627         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 628         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 629                 repoman_settings["PORTAGE_GPG_KEY"])
 630         sys.exit(1)
 631
 632 manifest_hashes = repo_config.manifest_hashes
 633 if manifest_hashes is None:
 634         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 635
 636 if options.mode in ("commit", "fix", "manifest"):
 637         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 638                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 639                 "metadata/layout.conf does not contain the '%s' hash which "
 640                 "is required by this portage version. You will have to "
 641                 "upgrade portage if you want to generate valid manifests for "
 642                 "this repository.") % \
 643                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 644                 for line in textwrap.wrap(msg, 70):
 645                         logging.error(line)
 646                 sys.exit(1)
 647
 648         unsupported_hashes = manifest_hashes.difference(
 649                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 650         if unsupported_hashes:
 651                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 652                 "metadata/layout.conf contains one or more hash types '%s' "
 653                 "which are not supported by this portage version. You will "
 654                 "have to upgrade portage if you want to generate valid "
 655                 "manifests for this repository.") % \
 656                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 657                 for line in textwrap.wrap(msg, 70):
 658                         logging.error(line)
 659                 sys.exit(1)
 660
 661 if options.echangelog is None and repo_config.update_changelog:
 662         options.echangelog = 'y'
 663
 664 if vcs is None:
 665         options.echangelog = 'n'
 666
 667 # The --echangelog option causes automatic ChangeLog generation,
 668 # which invalidates changelog.ebuildadded and changelog.missing
 669 # checks.
 670 # Note: Some don't use ChangeLogs in distributed SCMs.
 671 # It will be generated on server side from scm log,
 672 # before package moves to the rsync server.
 673 # This is needed because they try to avoid merge collisions.
 674 # Gentoo's Council decided to always use the ChangeLog file.
 675 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 676 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 677
 678 if 'digest' in repoman_settings.features and options.digest != 'n':
 679         options.digest = 'y'
 680
 681 logging.debug("vcs: %s" % (vcs,))
 682 logging.debug("repo config: %s" % (repo_config,))
 683 logging.debug("options: %s" % (options,))
 684
 685 # It's confusing if these warnings are displayed without the user
 686 # being told which profile they come from, so disable them.
 687 env = os.environ.copy()
 688 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 689
 690 categories = []
 691 for path in repo_config.eclass_db.porttrees:
 692         categories.extend(portage.util.grabfile(
 693                 os.path.join(path, 'profiles', 'categories')))
 694 repoman_settings.categories = frozenset(
 695         portage.util.stack_lists([categories], incremental=1))
 696 categories = repoman_settings.categories
 697
 698 portdb.settings = repoman_settings
 699 root_config = RootConfig(repoman_settings, trees[root], None)
 700 # We really only need to cache the metadata that's necessary for visibility
 701 # filtering. Anything else can be discarded to reduce memory consumption.
 702 portdb._aux_cache_keys.clear()
 703 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 704
 705 reposplit = myreporoot.split(os.path.sep)
 706 repolevel = len(reposplit)
 707
 708 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 709 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 710 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 711 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 712         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 713         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 714         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 715         print(red("***"))
 716         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 717
 718 # Make startdir relative to the canonical repodir, so that we can pass
 719 # it to digestgen and it won't have to be canonicalized again.
 720 if repolevel == 1:
 721         startdir = repodir
 722 else:
 723         startdir = normalize_path(mydir)
 724         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 725
 726 def caterror(mycat):
 727         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 728
 729 def repoman_getstatusoutput(cmd):
 730         """
 731         Implements an interface similar to getstatusoutput(), but with
 732         customized unicode handling (see bug #310789) and without the shell.
 733         """
 734         args = portage.util.shlex_split(cmd)
 735
 736         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 737                 not os.path.isabs(args[0]):
 738                 # Python 3.1 _execvp throws TypeError for non-absolute executable
 739                 # path passed as bytes (see http://bugs.python.org/issue8513).
 740                 fullname = find_binary(args[0])
 741                 if fullname is None:
 742                         raise portage.exception.CommandNotFound(args[0])
 743                 args[0] = fullname
 744
 745         encoding = _encodings['fs']
 746         args = [_unicode_encode(x,
 747                 encoding=encoding, errors='strict') for x in args]
 748         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 749                 stderr=subprocess.STDOUT)
 750         output = portage._unicode_decode(proc.communicate()[0],
 751                 encoding=encoding, errors='strict')
 752         if output and output[-1] == "\n":
 753                 # getstatusoutput strips one newline
 754                 output = output[:-1]
 755         return (proc.wait(), output)
 756
 757 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 758         """
 759         Implements an interface similar to os.popen(), but with customized
 760         unicode handling (see bug #310789) and without the shell.
 761         """
 762
 763         __slots__ = ('_proc', '_stdout')
 764
 765         def __init__(self, cmd):
 766                 args = portage.util.shlex_split(cmd)
 767
 768                 if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 769                         not os.path.isabs(args[0]):
 770                         # Python 3.1 _execvp throws TypeError for non-absolute executable
 771                         # path passed as bytes (see http://bugs.python.org/issue8513).
 772                         fullname = find_binary(args[0])
 773                         if fullname is None:
 774                                 raise portage.exception.CommandNotFound(args[0])
 775                         args[0] = fullname
 776
 777                 encoding = _encodings['fs']
 778                 args = [_unicode_encode(x,
 779                         encoding=encoding, errors='strict') for x in args]
 780                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 781                 object.__setattr__(self, '_proc', proc)
 782                 object.__setattr__(self, '_stdout',
 783                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 784
 785         def _get_target(self):
 786                 return object.__getattribute__(self, '_stdout')
 787
 788         __enter__ = _get_target
 789
 790         def __exit__(self, exc_type, exc_value, traceback):
 791                 proc = object.__getattribute__(self, '_proc')
 792                 proc.wait()
 793                 proc.stdout.close()
 794
 795 class ProfileDesc(object):
 796         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 797         def __init__(self, arch, status, sub_path, tree_path):
 798                 self.arch = arch
 799                 self.status = status
 800                 if sub_path:
 801                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 802                 self.sub_path = sub_path
 803                 self.tree_path = tree_path
 804                 if tree_path:
 805                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 806                 else:
 807                         self.abs_path = tree_path
 808
 809         def __str__(self):
 810                 if self.sub_path:
 811                         return self.sub_path
 812                 return 'empty profile'
 813
 814 profile_list = []
 815 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 816
 817 # get lists of valid keywords, licenses, and use
 818 kwlist = set()
 819 liclist = set()
 820 uselist = set()
 821 global_pmasklines = []
 822
 823 for path in portdb.porttrees:
 824         try:
 825                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 826         except OSError:
 827                 pass
 828         kwlist.update(portage.grabfile(os.path.join(path,
 829                 "profiles", "arch.list")))
 830
 831         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 832         for x in use_desc:
 833                 x = x.split()
 834                 if x:
 835                         uselist.add(x[0])
 836
 837         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 838         try:
 839                 expand_list = os.listdir(expand_desc_dir)
 840         except OSError:
 841                 pass
 842         else:
 843                 for fn in expand_list:
 844                         if not fn[-5:] == '.desc':
 845                                 continue
 846                         use_prefix = fn[:-5].lower() + '_'
 847                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 848                                 x = x.split()
 849                                 if x:
 850                                         uselist.add(use_prefix + x[0])
 851
 852         global_pmasklines.append(portage.util.grabfile_package(
 853                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 854
 855         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 856         try:
 857                 desc_file = io.open(_unicode_encode(desc_path,
 858                         encoding=_encodings['fs'], errors='strict'),
 859                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 860         except EnvironmentError:
 861                 pass
 862         else:
 863                 for i, x in enumerate(desc_file):
 864                         if x[0] == "#":
 865                                 continue
 866                         arch = x.split()
 867                         if len(arch) == 0:
 868                                 continue
 869                         if len(arch) != 3:
 870                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 871                                         desc_path + " line %d" % (i + 1, ))
 872                         elif arch[0] not in kwlist:
 873                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 874                                         desc_path + " line %d" % (i + 1, ))
 875                         elif arch[2] not in valid_profile_types:
 876                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 877                                         desc_path + " line %d" % (i + 1, ))
 878                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 879                         if not os.path.isdir(profile_desc.abs_path):
 880                                 logging.error(
 881                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 882                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 883                                 continue
 884                         if os.path.exists(
 885                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 886                                 continue
 887                         profile_list.append(profile_desc)
 888                 desc_file.close()
 889
 890 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 891 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 892
 893 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 894 global_pmaskdict = {}
 895 for x in global_pmasklines:
 896         global_pmaskdict.setdefault(x.cp, []).append(x)
 897 del global_pmasklines
 898
 899 def has_global_mask(pkg):
 900         mask_atoms = global_pmaskdict.get(pkg.cp)
 901         if mask_atoms:
 902                 pkg_list = [pkg]
 903                 for x in mask_atoms:
 904                         if portage.dep.match_from_list(x, pkg_list):
 905                                 return x
 906         return None
 907
 908 # Ensure that profile sub_path attributes are unique. Process in reverse order
 909 # so that profiles with duplicate sub_path from overlays will override
 910 # profiles with the same sub_path from parent repos.
 911 profiles = {}
 912 profile_list.reverse()
 913 profile_sub_paths = set()
 914 for prof in profile_list:
 915         if prof.sub_path in profile_sub_paths:
 916                 continue
 917         profile_sub_paths.add(prof.sub_path)
 918         profiles.setdefault(prof.arch, []).append(prof)
 919
 920 # Use an empty profile for checking dependencies of
 921 # packages that have empty KEYWORDS.
 922 prof = ProfileDesc('**', 'stable', '', '')
 923 profiles.setdefault(prof.arch, []).append(prof)
 924
 925 for x in repoman_settings.archlist():
 926         if x[0] == "~":
 927                 continue
 928         if x not in profiles:
 929                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 930                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 931                 print(red("up with the " + x + " team."))
 932                 print()
 933
 934 liclist_deprecated = set()
 935 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 936         liclist_deprecated.update(
 937                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 938
 939 if not liclist:
 940         logging.fatal("Couldn't find licenses?")
 941         sys.exit(1)
 942
 943 if not kwlist:
 944         logging.fatal("Couldn't read KEYWORDS from arch.list")
 945         sys.exit(1)
 946
 947 if not uselist:
 948         logging.fatal("Couldn't find use.desc?")
 949         sys.exit(1)
 950
 951 scanlist = []
 952 if repolevel == 2:
 953         # we are inside a category directory
 954         catdir = reposplit[-1]
 955         if catdir not in categories:
 956                 caterror(catdir)
 957         mydirlist = os.listdir(startdir)
 958         for x in mydirlist:
 959                 if x == "CVS" or x.startswith("."):
 960                         continue
 961                 if os.path.isdir(startdir + "/" + x):
 962                         scanlist.append(catdir + "/" + x)
 963         repo_subdir = catdir + os.sep
 964 elif repolevel == 1:
 965         for x in categories:
 966                 if not os.path.isdir(startdir + "/" + x):
 967                         continue
 968                 for y in os.listdir(startdir + "/" + x):
 969                         if y == "CVS" or y.startswith("."):
 970                                 continue
 971                         if os.path.isdir(startdir + "/" + x + "/" + y):
 972                                 scanlist.append(x + "/" + y)
 973         repo_subdir = ""
 974 elif repolevel == 3:
 975         catdir = reposplit[-2]
 976         if catdir not in categories:
 977                 caterror(catdir)
 978         scanlist.append(catdir + "/" + reposplit[-1])
 979         repo_subdir = scanlist[-1] + os.sep
 980 else:
 981         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 982                 ' from the current working directory'
 983         logging.critical(msg)
 984         sys.exit(1)
 985
 986 repo_subdir_len = len(repo_subdir)
 987 scanlist.sort()
 988
 989 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 990
 991 def vcs_files_to_cps(vcs_file_iter):
 992         """
 993         Iterate over the given modified file paths returned from the vcs,
 994         and return a frozenset containing category/pn strings for each
 995         modified package.
 996         """
 997
 998         modified_cps = []
 999
1000         if repolevel == 3:
1001                 if reposplit[-2] in categories and \
1002                         next(vcs_file_iter, None) is not None:
1003                         modified_cps.append("/".join(reposplit[-2:]))
1004
1005         elif repolevel == 2:
1006                 category = reposplit[-1]
1007                 if category in categories:
1008                         for filename in vcs_file_iter:
1009                                 f_split = filename.split(os.sep)
1010                                 # ['.', pn, ...]
1011                                 if len(f_split) > 2:
1012                                         modified_cps.append(category + "/" + f_split[1])
1013
1014         else:
1015                 # repolevel == 1
1016                 for filename in vcs_file_iter:
1017                         f_split = filename.split(os.sep)
1018                         # ['.', category, pn, ...]
1019                         if len(f_split) > 3 and f_split[1] in categories:
1020                                 modified_cps.append("/".join(f_split[1:3]))
1021
1022         return frozenset(modified_cps)
1023
1024 def git_supports_gpg_sign():
1025         status, cmd_output = \
1026                 repoman_getstatusoutput("git --version")
1027         cmd_output = cmd_output.split()
1028         if cmd_output:
1029                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1030                 if version is not None:
1031                         version = [int(x) for x in version.groups()]
1032                         if version[0] > 1 or \
1033                                 (version[0] == 1 and version[1] > 7) or \
1034                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1035                                 return True
1036         return False
1037
1038 def dev_keywords(profiles):
1039         """
1040         Create a set of KEYWORDS values that exist in 'dev'
1041         profiles. These are used
1042         to trigger a message notifying the user when they might
1043         want to add the --include-dev option.
1044         """
1045         type_arch_map = {}
1046         for arch, arch_profiles in profiles.items():
1047                 for prof in arch_profiles:
1048                         arch_set = type_arch_map.get(prof.status)
1049                         if arch_set is None:
1050                                 arch_set = set()
1051                                 type_arch_map[prof.status] = arch_set
1052                         arch_set.add(arch)
1053
1054         dev_keywords = type_arch_map.get('dev', set())
1055         dev_keywords.update(['~' + arch for arch in dev_keywords])
1056         return frozenset(dev_keywords)
1057
1058 dev_keywords = dev_keywords(profiles)
1059
1060 stats = {}
1061 fails = {}
1062
1063 for x in qacats:
1064         stats[x] = 0
1065         fails[x] = []
1066
1067 xmllint_capable = False
1068 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1069
1070 def fetch_metadata_dtd():
1071         """
1072         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1073         metadata_dtd_ctime_interval.
1074         @rtype: bool
1075         @return: True if successful, otherwise False
1076         """
1077
1078         must_fetch = True
1079         metadata_dtd_st = None
1080         current_time = int(time.time())
1081         try:
1082                 metadata_dtd_st = os.stat(metadata_dtd)
1083         except EnvironmentError as e:
1084                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1085                         raise
1086                 del e
1087         else:
1088                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1089                 if abs(current_time - metadata_dtd_st.st_ctime) \
1090                         < metadata_dtd_ctime_interval:
1091                         must_fetch = False
1092
1093         if must_fetch:
1094                 print()
1095                 print(green("***") + " the local copy of metadata.dtd " + \
1096                         "needs to be refetched, doing that now")
1097                 print()
1098                 parsed_url = urlparse(metadata_dtd_uri)
1099                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
1100                 fcmd = repoman_settings.get(setting)
1101                 if not fcmd:
1102                         fcmd = repoman_settings.get('FETCHCOMMAND')
1103                         if not fcmd:
1104                                 logging.error("FETCHCOMMAND is unset")
1105                                 return False
1106
1107                 destdir = repoman_settings["DISTDIR"]
1108                 fd, metadata_dtd_tmp = tempfile.mkstemp(
1109                         prefix='metadata.dtd.', dir=destdir)
1110                 os.close(fd)
1111
1112                 try:
1113                         if not portage.getbinpkg.file_get(metadata_dtd_uri,
1114                                 destdir, fcmd=fcmd,
1115                                 filename=os.path.basename(metadata_dtd_tmp)):
1116                                 logging.error("failed to fetch metadata.dtd from '%s'" %
1117                                         metadata_dtd_uri)
1118                                 return False
1119
1120                         try:
1121                                 portage.util.apply_secpass_permissions(metadata_dtd_tmp,
1122                                         gid=portage.data.portage_gid, mode=0o664, mask=0o2)
1123                         except portage.exception.PortageException:
1124                                 pass
1125
1126                         os.rename(metadata_dtd_tmp, metadata_dtd)
1127                 finally:
1128                         try:
1129                                 os.unlink(metadata_dtd_tmp)
1130                         except OSError:
1131                                 pass
1132
1133         return True
1134
1135 if options.mode == "manifest":
1136         pass
1137 elif not find_binary('xmllint'):
1138         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1139         if options.xml_parse or repolevel == 3:
1140                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1141                 sys.exit(1)
1142 else:
1143         if not fetch_metadata_dtd():
1144                 sys.exit(1)
1145         # this can be problematic if xmllint changes their output
1146         xmllint_capable = True
1147
1148 if options.mode == 'commit' and vcs:
1149         utilities.detect_vcs_conflicts(options, vcs)
1150
1151 if options.mode == "manifest":
1152         pass
1153 elif options.pretend:
1154         print(green("\nRepoMan does a once-over of the neighborhood..."))
1155 else:
1156         print(green("\nRepoMan scours the neighborhood..."))
1157
1158 new_ebuilds = set()
1159 modified_ebuilds = set()
1160 modified_changelogs = set()
1161 mychanged = []
1162 mynew = []
1163 myremoved = []
1164
1165 if vcs == "cvs":
1166         mycvstree = cvstree.getentries("./", recursive=1)
1167         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1168         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1169         if options.if_modified == "y":
1170                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1171
1172 elif vcs == "svn":
1173         with repoman_popen("svn status") as f:
1174                 svnstatus = f.readlines()
1175         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1176         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1177         if options.if_modified == "y":
1178                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1179
1180 elif vcs == "git":
1181         with repoman_popen("git diff-index --name-only "
1182                 "--relative --diff-filter=M HEAD") as f:
1183                 mychanged = f.readlines()
1184         mychanged = ["./" + elem[:-1] for elem in mychanged]
1185
1186         with repoman_popen("git diff-index --name-only "
1187                 "--relative --diff-filter=A HEAD") as f:
1188                 mynew = f.readlines()
1189         mynew = ["./" + elem[:-1] for elem in mynew]
1190         if options.if_modified == "y":
1191                 with repoman_popen("git diff-index --name-only "
1192                         "--relative --diff-filter=D HEAD") as f:
1193                         myremoved = f.readlines()
1194                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1195
1196 elif vcs == "bzr":
1197         with repoman_popen("bzr status -S .") as f:
1198                 bzrstatus = f.readlines()
1199         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1200         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1201         if options.if_modified == "y":
1202                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1203
1204 elif vcs == "hg":
1205         with repoman_popen("hg status --no-status --modified .") as f:
1206                 mychanged = f.readlines()
1207         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1208         with repoman_popen("hg status --no-status --added .") as f:
1209                 mynew = f.readlines()
1210         mynew = ["./" + elem.rstrip() for elem in mynew]
1211         if options.if_modified == "y":
1212                 with repoman_popen("hg status --no-status --removed .") as f:
1213                         myremoved = f.readlines()
1214                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1215
1216 if vcs:
1217         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1218         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1219         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1220                 if os.path.basename(x) == "ChangeLog")
1221
1222 def vcs_new_changed(relative_path):
1223         for x in chain(mychanged, mynew):
1224                 if x == relative_path:
1225                         return True
1226         return False
1227
1228 have_pmasked = False
1229 have_dev_keywords = False
1230 dofail = 0
1231
1232 # NOTE: match-all caches are not shared due to potential
1233 # differences between profiles in _get_implicit_iuse.
1234 arch_caches = {}
1235 arch_xmatch_caches = {}
1236 shared_xmatch_caches = {"cp-list":{}}
1237
1238 include_arches = None
1239 if options.include_arches:
1240         include_arches = set()
1241         include_arches.update(*[x.split() for x in options.include_arches])
1242
1243 # Disable the "ebuild.notadded" check when not in commit mode and
1244 # running `svn status` in every package dir will be too expensive.
1245
1246 check_ebuild_notadded = not \
1247         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1248
1249 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1250 thirdpartymirrors = {}
1251 for k, v in repoman_settings.thirdpartymirrors().items():
1252         for v in v:
1253                 if not v.endswith("/"):
1254                         v += "/"
1255                 thirdpartymirrors[v] = k
1256
1257 class _XMLParser(xml.etree.ElementTree.XMLParser):
1258
1259         def __init__(self, data, **kwargs):
1260                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1261                 self._portage_data = data
1262                 if hasattr(self, 'parser'):
1263                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1264                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1265                         self._base_StartDoctypeDeclHandler = \
1266                                 self.parser.StartDoctypeDeclHandler
1267                         self.parser.StartDoctypeDeclHandler = \
1268                                 self._portage_StartDoctypeDeclHandler
1269
1270         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1271                 if self._base_XmlDeclHandler is not None:
1272                         self._base_XmlDeclHandler(version, encoding, standalone)
1273                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1274
1275         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1276                 has_internal_subset):
1277                 if self._base_StartDoctypeDeclHandler is not None:
1278                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1279                                 has_internal_subset)
1280                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1281
1282 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1283         """
1284         Implements doctype() as required to avoid deprecation warnings with
1285         >=python-2.7.
1286         """
1287         def doctype(self, name, pubid, system):
1288                 pass
1289
1290 try:
1291         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1292 except (EnvironmentError, ParseError, PermissionDenied) as e:
1293         err(str(e))
1294 except FileNotFound:
1295         # TODO: Download as we do for metadata.dtd, but add a way to
1296         # disable for non-gentoo repoman users who may not have herds.
1297         herd_base = None
1298
1299 effective_scanlist = scanlist
1300 if options.if_modified == "y":
1301         effective_scanlist = sorted(vcs_files_to_cps(
1302                 chain(mychanged, mynew, myremoved)))
1303
1304 for x in effective_scanlist:
1305         #ebuilds and digests added to cvs respectively.
1306         logging.info("checking package %s" % x)
1307         # save memory by discarding xmatch caches from previous package(s)
1308         arch_xmatch_caches.clear()
1309         eadded = []
1310         catdir, pkgdir = x.split("/")
1311         checkdir = repodir + "/" + x
1312         checkdir_relative = ""
1313         if repolevel < 3:
1314                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1315         if repolevel < 2:
1316                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1317         checkdir_relative = os.path.join(".", checkdir_relative)
1318         generated_manifest = False
1319
1320         if options.mode == "manifest" or \
1321           (options.mode != 'manifest-check' and options.digest == 'y') or \
1322           options.mode in ('commit', 'fix') and not options.pretend:
1323                 auto_assumed = set()
1324                 fetchlist_dict = portage.FetchlistDict(checkdir,
1325                         repoman_settings, portdb)
1326                 if options.mode == 'manifest' and options.force:
1327                         portage._doebuild_manifest_exempt_depend += 1
1328                         try:
1329                                 distdir = repoman_settings['DISTDIR']
1330                                 mf = repoman_settings.repositories.get_repo_for_location(
1331                                         os.path.dirname(os.path.dirname(checkdir)))
1332                                 mf = mf.load_manifest(checkdir, distdir,
1333                                         fetchlist_dict=fetchlist_dict)
1334                                 mf.create(requiredDistfiles=None,
1335                                         assumeDistHashesAlways=True)
1336                                 for distfiles in fetchlist_dict.values():
1337                                         for distfile in distfiles:
1338                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1339                                                         mf.fhashdict['DIST'].pop(distfile, None)
1340                                                 else:
1341                                                         auto_assumed.add(distfile)
1342                                 mf.write()
1343                         finally:
1344                                 portage._doebuild_manifest_exempt_depend -= 1
1345
1346                 repoman_settings["O"] = checkdir
1347                 try:
1348                         generated_manifest = digestgen(
1349                                 mysettings=repoman_settings, myportdb=portdb)
1350                 except portage.exception.PermissionDenied as e:
1351                         generated_manifest = False
1352                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1353                                 level=logging.ERROR, noiselevel=-1)
1354
1355                 if not generated_manifest:
1356                         print("Unable to generate manifest.")
1357                         dofail = 1
1358
1359                 if options.mode == "manifest":
1360                         if not dofail and options.force and auto_assumed and \
1361                                 'assume-digests' in repoman_settings.features:
1362                                 # Show which digests were assumed despite the --force option
1363                                 # being given. This output will already have been shown by
1364                                 # digestgen() if assume-digests is not enabled, so only show
1365                                 # it here if assume-digests is enabled.
1366                                 pkgs = list(fetchlist_dict)
1367                                 pkgs.sort()
1368                                 portage.writemsg_stdout("  digest.assumed" + \
1369                                         portage.output.colorize("WARN",
1370                                         str(len(auto_assumed)).rjust(18)) + "\n")
1371                                 for cpv in pkgs:
1372                                         fetchmap = fetchlist_dict[cpv]
1373                                         pf = portage.catsplit(cpv)[1]
1374                                         for distfile in sorted(fetchmap):
1375                                                 if distfile in auto_assumed:
1376                                                         portage.writemsg_stdout(
1377                                                                 "   %s::%s\n" % (pf, distfile))
1378                         continue
1379                 elif dofail:
1380                         sys.exit(1)
1381
1382         if not generated_manifest:
1383                 repoman_settings['O'] = checkdir
1384                 repoman_settings['PORTAGE_QUIET'] = '1'
1385                 if not portage.digestcheck([], repoman_settings, strict=1):
1386                         stats["manifest.bad"] += 1
1387                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1388                 repoman_settings.pop('PORTAGE_QUIET', None)
1389
1390         if options.mode == 'manifest-check':
1391                 continue
1392
1393         checkdirlist = os.listdir(checkdir)
1394         ebuildlist = []
1395         pkgs = {}
1396         allvalid = True
1397         for y in checkdirlist:
1398                 if (y in no_exec or y.endswith(".ebuild")) and \
1399                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1400                                 stats["file.executable"] += 1
1401                                 fails["file.executable"].append(os.path.join(checkdir, y))
1402                 if y.endswith(".ebuild"):
1403                         pf = y[:-7]
1404                         ebuildlist.append(pf)
1405                         cpv = "%s/%s" % (catdir, pf)
1406                         try:
1407                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1408                         except KeyError:
1409                                 allvalid = False
1410                                 stats["ebuild.syntax"] += 1
1411                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1412                                 continue
1413                         except IOError:
1414                                 allvalid = False
1415                                 stats["ebuild.output"] += 1
1416                                 fails["ebuild.output"].append(os.path.join(x, y))
1417                                 continue
1418                         if not portage.eapi_is_supported(myaux["EAPI"]):
1419                                 allvalid = False
1420                                 stats["EAPI.unsupported"] += 1
1421                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1422                                 continue
1423                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1424                                 root_config=root_config, type_name="ebuild")
1425
1426         slot_keywords = {}
1427
1428         if len(pkgs) != len(ebuildlist):
1429                 # If we can't access all the metadata then it's totally unsafe to
1430                 # commit since there's no way to generate a correct Manifest.
1431                 # Do not try to do any more QA checks on this package since missing
1432                 # metadata leads to false positives for several checks, and false
1433                 # positives confuse users.
1434                 can_force = False
1435                 continue
1436
1437         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1438         ebuildlist = sorted(pkgs.values())
1439         ebuildlist = [pkg.pf for pkg in ebuildlist]
1440
1441         for y in checkdirlist:
1442                 index = repo_config.find_invalid_path_char(y)
1443                 if index != -1:
1444                         y_relative = os.path.join(checkdir_relative, y)
1445                         if vcs is not None and not vcs_new_changed(y_relative):
1446                                 # If the file isn't in the VCS new or changed set, then
1447                                 # assume that it's an irrelevant temporary file (Manifest
1448                                 # entries are not generated for file names containing
1449                                 # prohibited characters). See bug #406877.
1450                                 index = -1
1451                 if index != -1:
1452                         stats["file.name"] += 1
1453                         fails["file.name"].append("%s/%s: char '%s'" % \
1454                                 (checkdir, y, y[index]))
1455
1456                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1457                         continue
1458                 f = None
1459                 try:
1460                         line = 1
1461                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1462                                 encoding=_encodings['fs'], errors='strict'),
1463                                 mode='r', encoding=_encodings['repo.content'])
1464                         for l in f:
1465                                 line += 1
1466                 except UnicodeDecodeError as ue:
1467                         stats["file.UTF8"] += 1
1468                         s = ue.object[:ue.start]
1469                         l2 = s.count("\n")
1470                         line += l2
1471                         if l2 != 0:
1472                                 s = s[s.rfind("\n") + 1:]
1473                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1474                 finally:
1475                         if f is not None:
1476                                 f.close()
1477
1478         if vcs in ("git", "hg") and check_ebuild_notadded:
1479                 if vcs == "git":
1480                         myf = repoman_popen("git ls-files --others %s" % \
1481                                 (portage._shell_quote(checkdir_relative),))
1482                 if vcs == "hg":
1483                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1484                                 (portage._shell_quote(checkdir_relative),))
1485                 for l in myf:
1486                         if l[:-1][-7:] == ".ebuild":
1487                                 stats["ebuild.notadded"] += 1
1488                                 fails["ebuild.notadded"].append(
1489                                         os.path.join(x, os.path.basename(l[:-1])))
1490                 myf.close()
1491
1492         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1493                 try:
1494                         if vcs == "cvs":
1495                                 myf = open(checkdir + "/CVS/Entries", "r")
1496                         if vcs == "svn":
1497                                 myf = repoman_popen("svn status --depth=files --verbose " +
1498                                         portage._shell_quote(checkdir))
1499                         if vcs == "bzr":
1500                                 myf = repoman_popen("bzr ls -v --kind=file " +
1501                                         portage._shell_quote(checkdir))
1502                         myl = myf.readlines()
1503                         myf.close()
1504                         for l in myl:
1505                                 if vcs == "cvs":
1506                                         if l[0] != "/":
1507                                                 continue
1508                                         splitl = l[1:].split("/")
1509                                         if not len(splitl):
1510                                                 continue
1511                                         if splitl[0][-7:] == ".ebuild":
1512                                                 eadded.append(splitl[0][:-7])
1513                                 if vcs == "svn":
1514                                         if l[:1] == "?":
1515                                                 continue
1516                                         if l[:7] == '      >':
1517                                                 # tree conflict, new in subversion 1.6
1518                                                 continue
1519                                         l = l.split()[-1]
1520                                         if l[-7:] == ".ebuild":
1521                                                 eadded.append(os.path.basename(l[:-7]))
1522                                 if vcs == "bzr":
1523                                         if l[1:2] == "?":
1524                                                 continue
1525                                         l = l.split()[-1]
1526                                         if l[-7:] == ".ebuild":
1527                                                 eadded.append(os.path.basename(l[:-7]))
1528                         if vcs == "svn":
1529                                 myf = repoman_popen("svn status " +
1530                                         portage._shell_quote(checkdir))
1531                                 myl = myf.readlines()
1532                                 myf.close()
1533                                 for l in myl:
1534                                         if l[0] == "A":
1535                                                 l = l.rstrip().split(' ')[-1]
1536                                                 if l[-7:] == ".ebuild":
1537                                                         eadded.append(os.path.basename(l[:-7]))
1538                 except IOError:
1539                         if vcs == "cvs":
1540                                 stats["CVS/Entries.IO_error"] += 1
1541                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1542                         else:
1543                                 raise
1544                         continue
1545
1546         mf = repoman_settings.repositories.get_repo_for_location(
1547                 os.path.dirname(os.path.dirname(checkdir)))
1548         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1549         mydigests = mf.getTypeDigests("DIST")
1550
1551         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1552         myfiles_all = []
1553         src_uri_error = False
1554         for mykey in fetchlist_dict:
1555                 try:
1556                         myfiles_all.extend(fetchlist_dict[mykey])
1557                 except portage.exception.InvalidDependString as e:
1558                         src_uri_error = True
1559                         try:
1560                                 portdb.aux_get(mykey, ["SRC_URI"])
1561                         except KeyError:
1562                                 # This will be reported as an "ebuild.syntax" error.
1563                                 pass
1564                         else:
1565                                 stats["SRC_URI.syntax"] += 1
1566                                 fails["SRC_URI.syntax"].append(
1567                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1568         del fetchlist_dict
1569         if not src_uri_error:
1570                 # This test can produce false positives if SRC_URI could not
1571                 # be parsed for one or more ebuilds. There's no point in
1572                 # producing a false error here since the root cause will
1573                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1574                 # or "ebuild.sytax".
1575                 myfiles_all = set(myfiles_all)
1576                 for entry in mydigests:
1577                         if entry not in myfiles_all:
1578                                 stats["digest.unused"] += 1
1579                                 fails["digest.unused"].append(checkdir + "::" + entry)
1580                 for entry in myfiles_all:
1581                         if entry not in mydigests:
1582                                 stats["digest.missing"] += 1
1583                                 fails["digest.missing"].append(checkdir + "::" + entry)
1584         del myfiles_all
1585
1586         if os.path.exists(checkdir + "/files"):
1587                 filesdirlist = os.listdir(checkdir + "/files")
1588
1589                 # recurse through files directory
1590                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1591                 while filesdirlist:
1592                         y = filesdirlist.pop(0)
1593                         relative_path = os.path.join(x, "files", y)
1594                         full_path = os.path.join(repodir, relative_path)
1595                         try:
1596                                 mystat = os.stat(full_path)
1597                         except OSError as oe:
1598                                 if oe.errno == 2:
1599                                         # don't worry about it.  it likely was removed via fix above.
1600                                         continue
1601                                 else:
1602                                         raise oe
1603                         if S_ISDIR(mystat.st_mode):
1604                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1605                                 if y == "CVS" or y == ".svn":
1606                                         continue
1607                                 for z in os.listdir(checkdir + "/files/" + y):
1608                                         if z == "CVS" or z == ".svn":
1609                                                 continue
1610                                         filesdirlist.append(y + "/" + z)
1611                         # Current policy is no files over 20 KiB, these are the checks. File size between
1612                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1613                         elif mystat.st_size > 61440:
1614                                 stats["file.size.fatal"] += 1
1615                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1616                         elif mystat.st_size > 20480:
1617                                 stats["file.size"] += 1
1618                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1619
1620                         index = repo_config.find_invalid_path_char(y)
1621                         if index != -1:
1622                                 y_relative = os.path.join(checkdir_relative, "files", y)
1623                                 if vcs is not None and not vcs_new_changed(y_relative):
1624                                         # If the file isn't in the VCS new or changed set, then
1625                                         # assume that it's an irrelevant temporary file (Manifest
1626                                         # entries are not generated for file names containing
1627                                         # prohibited characters). See bug #406877.
1628                                         index = -1
1629                         if index != -1:
1630                                 stats["file.name"] += 1
1631                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1632                                         (checkdir, y, y[index]))
1633         del mydigests
1634
1635         if check_changelog and "ChangeLog" not in checkdirlist:
1636                 stats["changelog.missing"] += 1
1637                 fails["changelog.missing"].append(x + "/ChangeLog")
1638
1639         musedict = {}
1640         # metadata.xml file check
1641         if "metadata.xml" not in checkdirlist:
1642                 stats["metadata.missing"] += 1
1643                 fails["metadata.missing"].append(x + "/metadata.xml")
1644         # metadata.xml parse check
1645         else:
1646                 metadata_bad = False
1647                 xml_info = {}
1648                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1649
1650                 # read metadata.xml into memory
1651                 try:
1652                         _metadata_xml = xml.etree.ElementTree.parse(
1653                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1654                                 encoding=_encodings['fs'], errors='strict'),
1655                                 parser=xml_parser)
1656                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1657                         metadata_bad = True
1658                         stats["metadata.bad"] += 1
1659                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1660                         del e
1661                 else:
1662                         if not hasattr(xml_parser, 'parser') or \
1663                                 sys.hexversion < 0x2070000 or \
1664                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1665                                 # doctype is not parsed with python 2.6 or 3.1
1666                                 pass
1667                         else:
1668                                 if "XML_DECLARATION" not in xml_info:
1669                                         stats["metadata.bad"] += 1
1670                                         fails["metadata.bad"].append("%s/metadata.xml: "
1671                                                 "xml declaration is missing on first line, "
1672                                                 "should be '%s'" % (x, metadata_xml_declaration))
1673                                 else:
1674                                         xml_version, xml_encoding, xml_standalone = \
1675                                                 xml_info["XML_DECLARATION"]
1676                                         if xml_encoding is None or \
1677                                                 xml_encoding.upper() != metadata_xml_encoding:
1678                                                 stats["metadata.bad"] += 1
1679                                                 if xml_encoding is None:
1680                                                         encoding_problem = "but it is undefined"
1681                                                 else:
1682                                                         encoding_problem = "not '%s'" % xml_encoding
1683                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1684                                                         "xml declaration encoding should be '%s', %s" %
1685                                                         (x, metadata_xml_encoding, encoding_problem))
1686
1687                                 if "DOCTYPE" not in xml_info:
1688                                         metadata_bad = True
1689                                         stats["metadata.bad"] += 1
1690                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1691                                                 "DOCTYPE is missing"))
1692                                 else:
1693                                         doctype_name, doctype_system, doctype_pubid = \
1694                                                 xml_info["DOCTYPE"]
1695                                         if doctype_system != metadata_dtd_uri:
1696                                                 stats["metadata.bad"] += 1
1697                                                 if doctype_system is None:
1698                                                         system_problem = "but it is undefined"
1699                                                 else:
1700                                                         system_problem = "not '%s'" % doctype_system
1701                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1702                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1703                                                         (x, metadata_dtd_uri, system_problem))
1704
1705                                         if doctype_name != metadata_doctype_name:
1706                                                 stats["metadata.bad"] += 1
1707                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1708                                                         "DOCTYPE: name should be '%s', not '%s'" %
1709                                                         (x, metadata_doctype_name, doctype_name))
1710
1711                         # load USE flags from metadata.xml
1712                         try:
1713                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1714                         except portage.exception.ParseError as e:
1715                                 metadata_bad = True
1716                                 stats["metadata.bad"] += 1
1717                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1718                         else:
1719                                 for atom in chain(*musedict.values()):
1720                                         if atom is None:
1721                                                 continue
1722                                         try:
1723                                                 atom = Atom(atom)
1724                                         except InvalidAtom as e:
1725                                                 stats["metadata.bad"] += 1
1726                                                 fails["metadata.bad"].append(
1727                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1728                                         else:
1729                                                 if atom.cp != x:
1730                                                         stats["metadata.bad"] += 1
1731                                                         fails["metadata.bad"].append(
1732                                                                 ("%s/metadata.xml: Atom contains "
1733                                                                 "unexpected cat/pn: %s") % (x, atom))
1734
1735                         # Run other metadata.xml checkers
1736                         try:
1737                                 utilities.check_metadata(_metadata_xml, herd_base)
1738                         except (utilities.UnknownHerdsError, ) as e:
1739                                 metadata_bad = True
1740                                 stats["metadata.bad"] += 1
1741                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1742                                 del e
1743
1744                 #Only carry out if in package directory or check forced
1745                 if xmllint_capable and not metadata_bad:
1746                         # xmlint can produce garbage output even on success, so only dump
1747                         # the ouput when it fails.
1748                         st, out = repoman_getstatusoutput(
1749                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1750                                  (portage._shell_quote(metadata_dtd),
1751                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1752                         if st != os.EX_OK:
1753                                 print(red("!!!") + " metadata.xml is invalid:")
1754                                 for z in out.splitlines():
1755                                         print(red("!!! ") + z)
1756                                 stats["metadata.bad"] += 1
1757                                 fails["metadata.bad"].append(x + "/metadata.xml")
1758
1759                 del metadata_bad
1760         muselist = frozenset(musedict)
1761
1762         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1763         changelog_modified = changelog_path in modified_changelogs
1764
1765         # detect unused local USE-descriptions
1766         used_useflags = set()
1767
1768         for y in ebuildlist:
1769                 relative_path = os.path.join(x, y + ".ebuild")
1770                 full_path = os.path.join(repodir, relative_path)
1771                 ebuild_path = y + ".ebuild"
1772                 if repolevel < 3:
1773                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1774                 if repolevel < 2:
1775                         ebuild_path = os.path.join(catdir, ebuild_path)
1776                 ebuild_path = os.path.join(".", ebuild_path)
1777                 if check_changelog and not changelog_modified \
1778                         and ebuild_path in new_ebuilds:
1779                         stats['changelog.ebuildadded'] += 1
1780                         fails['changelog.ebuildadded'].append(relative_path)
1781
1782                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1783                         #ebuild not added to vcs
1784                         stats["ebuild.notadded"] += 1
1785                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1786                 myesplit = portage.pkgsplit(y)
1787                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1788                             or pv_toolong_re.search(myesplit[1]) \
1789                             or pv_toolong_re.search(myesplit[2]):
1790                         stats["ebuild.invalidname"] += 1
1791                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1792                         continue
1793                 elif myesplit[0] != pkgdir:
1794                         print(pkgdir, myesplit[0])
1795                         stats["ebuild.namenomatch"] += 1
1796                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1797                         continue
1798
1799                 pkg = pkgs[y]
1800
1801                 if pkg.invalid:
1802                         allvalid = False
1803                         for k, msgs in pkg.invalid.items():
1804                                 for msg in msgs:
1805                                         stats[k] += 1
1806                                         fails[k].append("%s: %s" % (relative_path, msg))
1807                         continue
1808
1809                 myaux = pkg._metadata
1810                 eapi = myaux["EAPI"]
1811                 inherited = pkg.inherited
1812                 live_ebuild = live_eclasses.intersection(inherited)
1813
1814                 if repo_config.eapi_is_banned(eapi):
1815                         stats["repo.eapi.banned"] += 1
1816                         fails["repo.eapi.banned"].append(
1817                                 "%s: %s" % (relative_path, eapi))
1818
1819                 elif repo_config.eapi_is_deprecated(eapi):
1820                         stats["repo.eapi.deprecated"] += 1
1821                         fails["repo.eapi.deprecated"].append(
1822                                 "%s: %s" % (relative_path, eapi))
1823
1824                 for k, v in myaux.items():
1825                         if not isinstance(v, basestring):
1826                                 continue
1827                         m = non_ascii_re.search(v)
1828                         if m is not None:
1829                                 stats["variable.invalidchar"] += 1
1830                                 fails["variable.invalidchar"].append(
1831                                         ("%s: %s variable contains non-ASCII " + \
1832                                         "character at position %s") % \
1833                                         (relative_path, k, m.start() + 1))
1834
1835                 if not src_uri_error:
1836                         # Check that URIs don't reference a server from thirdpartymirrors.
1837                         for uri in portage.dep.use_reduce( \
1838                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1839                                 contains_mirror = False
1840                                 for mirror, mirror_alias in thirdpartymirrors.items():
1841                                         if uri.startswith(mirror):
1842                                                 contains_mirror = True
1843                                                 break
1844                                 if not contains_mirror:
1845                                         continue
1846
1847                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1848                                 stats["SRC_URI.mirror"] += 1
1849                                 fails["SRC_URI.mirror"].append(
1850                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1851                                         (relative_path, mirror, new_uri))
1852
1853                 if myaux.get("PROVIDE"):
1854                         stats["virtual.oldstyle"] += 1
1855                         fails["virtual.oldstyle"].append(relative_path)
1856
1857                 for pos, missing_var in enumerate(missingvars):
1858                         if not myaux.get(missing_var):
1859                                 if catdir == "virtual" and \
1860                                         missing_var in ("HOMEPAGE", "LICENSE"):
1861                                         continue
1862                                 if live_ebuild and missing_var == "KEYWORDS":
1863                                         continue
1864                                 myqakey = missingvars[pos] + ".missing"
1865                                 stats[myqakey] += 1
1866                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1867
1868                 if catdir == "virtual":
1869                         for var in ("HOMEPAGE", "LICENSE"):
1870                                 if myaux.get(var):
1871                                         myqakey = var + ".virtual"
1872                                         stats[myqakey] += 1
1873                                         fails[myqakey].append(relative_path)
1874
1875                 # 14 is the length of DESCRIPTION=""
1876                 if len(myaux['DESCRIPTION']) > max_desc_len:
1877                         stats['DESCRIPTION.toolong'] += 1
1878                         fails['DESCRIPTION.toolong'].append(
1879                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1880                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1881
1882                 keywords = myaux["KEYWORDS"].split()
1883                 stable_keywords = []
1884                 for keyword in keywords:
1885                         if not keyword.startswith("~") and \
1886                                 not keyword.startswith("-"):
1887                                 stable_keywords.append(keyword)
1888                 if stable_keywords:
1889                         if ebuild_path in new_ebuilds and catdir != "virtual":
1890                                 stable_keywords.sort()
1891                                 stats["KEYWORDS.stable"] += 1
1892                                 fails["KEYWORDS.stable"].append(
1893                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1894                                                 " ".join(stable_keywords))
1895
1896                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1897                         if not kw.startswith("-"))
1898
1899                 previous_keywords = slot_keywords.get(pkg.slot)
1900                 if previous_keywords is None:
1901                         slot_keywords[pkg.slot] = set()
1902                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1903                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1904                         if dropped_keywords:
1905                                 stats["KEYWORDS.dropped"] += 1
1906                                 fails["KEYWORDS.dropped"].append(
1907                                         relative_path + ": %s" % \
1908                                         " ".join(sorted(dropped_keywords)))
1909
1910                 slot_keywords[pkg.slot].update(ebuild_archs)
1911
1912                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1913                 if "-*" in keywords:
1914                         haskeyword = False
1915                         for kw in keywords:
1916                                 if kw[0] == "~":
1917                                         kw = kw[1:]
1918                                 if kw in kwlist:
1919                                         haskeyword = True
1920                         if not haskeyword:
1921                                 stats["KEYWORDS.stupid"] += 1
1922                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1923
1924                 """
1925                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1926                 not be allowed to be marked stable
1927                 """
1928                 if live_ebuild and repo_config.name == "gentoo":
1929                         bad_stable_keywords = []
1930                         for keyword in keywords:
1931                                 if not keyword.startswith("~") and \
1932                                         not keyword.startswith("-"):
1933                                         bad_stable_keywords.append(keyword)
1934                                 del keyword
1935                         if bad_stable_keywords:
1936                                 stats["LIVEVCS.stable"] += 1
1937                                 fails["LIVEVCS.stable"].append(
1938                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1939                                                 bad_stable_keywords)
1940                         del bad_stable_keywords
1941
1942                         if keywords and not has_global_mask(pkg):
1943                                 stats["LIVEVCS.unmasked"] += 1
1944                                 fails["LIVEVCS.unmasked"].append(relative_path)
1945
1946                 if options.ignore_arches:
1947                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1948                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1949                 else:
1950                         arches = set()
1951                         for keyword in keywords:
1952                                 if keyword[0] == "-":
1953                                         continue
1954                                 elif keyword[0] == "~":
1955                                         arch = keyword[1:]
1956                                         if arch == "*":
1957                                                 for expanded_arch in profiles:
1958                                                         if expanded_arch == "**":
1959                                                                 continue
1960                                                         arches.add((keyword, expanded_arch,
1961                                                                 (expanded_arch, "~" + expanded_arch)))
1962                                         else:
1963                                                 arches.add((keyword, arch, (arch, keyword)))
1964                                 else:
1965                                         if keyword == "*":
1966                                                 for expanded_arch in profiles:
1967                                                         if expanded_arch == "**":
1968                                                                 continue
1969                                                         arches.add((keyword, expanded_arch,
1970                                                                 (expanded_arch,)))
1971                                         else:
1972                                                 arches.add((keyword, keyword, (keyword,)))
1973                         if not arches:
1974                                 # Use an empty profile for checking dependencies of
1975                                 # packages that have empty KEYWORDS.
1976                                 arches.add(('**', '**', ('**',)))
1977
1978                 unknown_pkgs = set()
1979                 baddepsyntax = False
1980                 badlicsyntax = False
1981                 badprovsyntax = False
1982                 catpkg = catdir + "/" + y
1983
1984                 inherited_java_eclass = "java-pkg-2" in inherited or \
1985                         "java-pkg-opt-2" in inherited
1986                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1987                 operator_tokens = set(["||", "(", ")"])
1988                 type_list, badsyntax = [], []
1989                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
1990                         mydepstr = myaux[mytype]
1991
1992                         buildtime = mytype in Package._buildtime_keys
1993                         runtime = mytype in Package._runtime_keys
1994                         token_class = None
1995                         if mytype.endswith("DEPEND"):
1996                                 token_class = portage.dep.Atom
1997
1998                         try:
1999                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
2000                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
2001                         except portage.exception.InvalidDependString as e:
2002                                 atoms = None
2003                                 badsyntax.append(str(e))
2004
2005                         if atoms and mytype.endswith("DEPEND"):
2006                                 if runtime and \
2007                                         "test?" in mydepstr.split():
2008                                         stats[mytype + '.suspect'] += 1
2009                                         fails[mytype + '.suspect'].append(relative_path + \
2010                                                 ": 'test?' USE conditional in %s" % mytype)
2011
2012                                 for atom in atoms:
2013                                         if atom == "||":
2014                                                 continue
2015
2016                                         # Skip dependency.unknown for blockers, so that we
2017                                         # don't encourage people to remove necessary blockers,
2018                                         # as discussed in bug #382407.
2019                                         if atom.blocker is None and \
2020                                                 not portdb.xmatch("match-all", atom) and \
2021                                                 not atom.cp.startswith("virtual/"):
2022                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2023
2024                                         is_blocker = atom.blocker
2025
2026                                         if catdir != "virtual":
2027                                                 if not is_blocker and \
2028                                                         atom.cp in suspect_virtual:
2029                                                         stats['virtual.suspect'] += 1
2030                                                         fails['virtual.suspect'].append(
2031                                                                 relative_path +
2032                                                                 ": %s: consider using '%s' instead of '%s'" %
2033                                                                 (mytype, suspect_virtual[atom.cp], atom))
2034
2035                                         if buildtime and \
2036                                                 not is_blocker and \
2037                                                 not inherited_java_eclass and \
2038                                                 atom.cp == "virtual/jdk":
2039                                                 stats['java.eclassesnotused'] += 1
2040                                                 fails['java.eclassesnotused'].append(relative_path)
2041                                         elif buildtime and \
2042                                                 not is_blocker and \
2043                                                 not inherited_wxwidgets_eclass and \
2044                                                 atom.cp == "x11-libs/wxGTK":
2045                                                 stats['wxwidgets.eclassnotused'] += 1
2046                                                 fails['wxwidgets.eclassnotused'].append(
2047                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2048                                                         " without inheriting wxwidgets.eclass") % mytype)
2049                                         elif runtime:
2050                                                 if not is_blocker and \
2051                                                         atom.cp in suspect_rdepend:
2052                                                         stats[mytype + '.suspect'] += 1
2053                                                         fails[mytype + '.suspect'].append(
2054                                                                 relative_path + ": '%s'" % atom)
2055
2056                                         if atom.operator == "~" and \
2057                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2058                                                 qacat = 'dependency.badtilde'
2059                                                 stats[qacat] += 1
2060                                                 fails[qacat].append(
2061                                                         (relative_path + ": %s uses the ~ operator"
2062                                                          " with a non-zero revision:" + \
2063                                                          " '%s'") % (mytype, atom))
2064
2065                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2066
2067                 for m, b in zip(type_list, badsyntax):
2068                         if m.endswith("DEPEND"):
2069                                 qacat = "dependency.syntax"
2070                         else:
2071                                 qacat = m + ".syntax"
2072                         stats[qacat] += 1
2073                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2074
2075                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2076                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2077                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2078                 badlicsyntax = badlicsyntax > 0
2079                 badprovsyntax = badprovsyntax > 0
2080
2081                 # uselist checks - global
2082                 myuse = []
2083                 default_use = []
2084                 for myflag in myaux["IUSE"].split():
2085                         flag_name = myflag.lstrip("+-")
2086                         used_useflags.add(flag_name)
2087                         if myflag != flag_name:
2088                                 default_use.append(myflag)
2089                         if flag_name not in uselist:
2090                                 myuse.append(flag_name)
2091
2092                 # uselist checks - metadata
2093                 for mypos in range(len(myuse)-1, -1, -1):
2094                         if myuse[mypos] and (myuse[mypos] in muselist):
2095                                 del myuse[mypos]
2096
2097                 if default_use and not eapi_has_iuse_defaults(eapi):
2098                         for myflag in default_use:
2099                                 stats['EAPI.incompatible'] += 1
2100                                 fails['EAPI.incompatible'].append(
2101                                         (relative_path + ": IUSE defaults" + \
2102                                         " not supported with EAPI='%s':" + \
2103                                         " '%s'") % (eapi, myflag))
2104
2105                 for mypos in range(len(myuse)):
2106                         stats["IUSE.invalid"] += 1
2107                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2108
2109                 # license checks
2110                 if not badlicsyntax:
2111                         # Parse the LICENSE variable, remove USE conditions and
2112                         # flatten it.
2113                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2114                         # Check each entry to ensure that it exists in PORTDIR's
2115                         # license directory.
2116                         for lic in licenses:
2117                                 # Need to check for "||" manually as no portage
2118                                 # function will remove it without removing values.
2119                                 if lic not in liclist and lic != "||":
2120                                         stats["LICENSE.invalid"] += 1
2121                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2122                                 elif lic in liclist_deprecated:
2123                                         stats["LICENSE.deprecated"] += 1
2124                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2125
2126                 #keyword checks
2127                 myuse = myaux["KEYWORDS"].split()
2128                 for mykey in myuse:
2129                         if mykey not in ("-*", "*", "~*"):
2130                                 myskey = mykey
2131                                 if myskey[:1] == "-":
2132                                         myskey = myskey[1:]
2133                                 if myskey[:1] == "~":
2134                                         myskey = myskey[1:]
2135                                 if myskey not in kwlist:
2136                                         stats["KEYWORDS.invalid"] += 1
2137                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2138                                 elif myskey not in profiles:
2139                                         stats["KEYWORDS.invalid"] += 1
2140                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2141
2142                 #restrict checks
2143                 myrestrict = None
2144                 try:
2145                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2146                 except portage.exception.InvalidDependString as e:
2147                         stats["RESTRICT.syntax"] += 1
2148                         fails["RESTRICT.syntax"].append(
2149                                 "%s: RESTRICT: %s" % (relative_path, e))
2150                         del e
2151                 if myrestrict:
2152                         myrestrict = set(myrestrict)
2153                         mybadrestrict = myrestrict.difference(valid_restrict)
2154                         if mybadrestrict:
2155                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2156                                 for mybad in mybadrestrict:
2157                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2158                 #REQUIRED_USE check
2159                 required_use = myaux["REQUIRED_USE"]
2160                 if required_use:
2161                         if not eapi_has_required_use(eapi):
2162                                 stats['EAPI.incompatible'] += 1
2163                                 fails['EAPI.incompatible'].append(
2164                                         relative_path + ": REQUIRED_USE" + \
2165                                         " not supported with EAPI='%s'" % (eapi,))
2166                         try:
2167                                 portage.dep.check_required_use(required_use, (),
2168                                         pkg.iuse.is_valid_flag, eapi=eapi)
2169                         except portage.exception.InvalidDependString as e:
2170                                 stats["REQUIRED_USE.syntax"] += 1
2171                                 fails["REQUIRED_USE.syntax"].append(
2172                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2173                                 del e
2174
2175                 # Syntax Checks
2176                 relative_path = os.path.join(x, y + ".ebuild")
2177                 full_path = os.path.join(repodir, relative_path)
2178                 if not vcs_preserves_mtime:
2179                         if ebuild_path not in new_ebuilds and \
2180                                 ebuild_path not in modified_ebuilds:
2181                                 pkg.mtime = None
2182                 try:
2183                         # All ebuilds should have utf_8 encoding.
2184                         f = io.open(_unicode_encode(full_path,
2185                                 encoding=_encodings['fs'], errors='strict'),
2186                                 mode='r', encoding=_encodings['repo.content'])
2187                         try:
2188                                 for check_name, e in run_checks(f, pkg):
2189                                         stats[check_name] += 1
2190                                         fails[check_name].append(relative_path + ': %s' % e)
2191                         finally:
2192                                 f.close()
2193                 except UnicodeDecodeError:
2194                         # A file.UTF8 failure will have already been recorded above.
2195                         pass
2196
2197                 if options.force:
2198                         # The dep_check() calls are the most expensive QA test. If --force
2199                         # is enabled, there's no point in wasting time on these since the
2200                         # user is intent on forcing the commit anyway.
2201                         continue
2202
2203                 relevant_profiles = []
2204                 for keyword, arch, groups in arches:
2205                         if arch not in profiles:
2206                                 # A missing profile will create an error further down
2207                                 # during the KEYWORDS verification.
2208                                 continue
2209
2210                         if include_arches is not None:
2211                                 if arch not in include_arches:
2212                                         continue
2213
2214                         relevant_profiles.extend((keyword, groups, prof)
2215                                 for prof in profiles[arch])
2216
2217                 def sort_key(item):
2218                         return item[2].sub_path
2219
2220                 relevant_profiles.sort(key=sort_key)
2221
2222                 for keyword, groups, prof in relevant_profiles:
2223
2224                                 if prof.status not in ("stable", "dev") or \
2225                                         prof.status == "dev" and not options.include_dev:
2226                                         continue
2227
2228                                 dep_settings = arch_caches.get(prof.sub_path)
2229                                 if dep_settings is None:
2230                                         dep_settings = portage.config(
2231                                                 config_profile_path=prof.abs_path,
2232                                                 config_incrementals=repoman_incrementals,
2233                                                 config_root=config_root,
2234                                                 local_config=False,
2235                                                 _unmatched_removal=options.unmatched_removal,
2236                                                 env=env, repositories=repoman_settings.repositories)
2237                                         dep_settings.categories = repoman_settings.categories
2238                                         if options.without_mask:
2239                                                 dep_settings._mask_manager_obj = \
2240                                                         copy.deepcopy(dep_settings._mask_manager)
2241                                                 dep_settings._mask_manager._pmaskdict.clear()
2242                                         arch_caches[prof.sub_path] = dep_settings
2243
2244                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2245                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2246                                 if xcache is None:
2247                                         portdb.melt()
2248                                         portdb.freeze()
2249                                         xcache = portdb.xcache
2250                                         xcache.update(shared_xmatch_caches)
2251                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2252
2253                                 trees[root]["porttree"].settings = dep_settings
2254                                 portdb.settings = dep_settings
2255                                 portdb.xcache = xcache
2256
2257                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2258                                 # just in case, prevent config.reset() from nuking these.
2259                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2260
2261                                 # This attribute is used in dbapi._match_use() to apply
2262                                 # use.stable.{mask,force} settings based on the stable
2263                                 # status of the parent package. This is required in order
2264                                 # for USE deps of unstable packages to be resolved correctly,
2265                                 # since otherwise use.stable.{mask,force} settings of
2266                                 # dependencies may conflict (see bug #456342).
2267                                 dep_settings._parent_stable = dep_settings._isStable(pkg)
2268
2269                                 # Handle package.use*.{force,mask) calculation, for use
2270                                 # in dep_check.
2271                                 dep_settings.useforce = dep_settings._use_manager.getUseForce(
2272                                         pkg, stable=dep_settings._parent_stable)
2273                                 dep_settings.usemask = dep_settings._use_manager.getUseMask(
2274                                         pkg, stable=dep_settings._parent_stable)
2275
2276                                 if not baddepsyntax:
2277                                         ismasked = not ebuild_archs or \
2278                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2279                                         if ismasked:
2280                                                 if not have_pmasked:
2281                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2282                                                                 pkg.cpv, pkg._metadata))
2283                                                 if options.ignore_masked:
2284                                                         continue
2285                                                 #we are testing deps for a masked package; give it some lee-way
2286                                                 suffix = "masked"
2287                                                 matchmode = "minimum-all"
2288                                         else:
2289                                                 suffix = ""
2290                                                 matchmode = "minimum-visible"
2291
2292                                         if not have_dev_keywords:
2293                                                 have_dev_keywords = \
2294                                                         bool(dev_keywords.intersection(keywords))
2295
2296                                         if prof.status == "dev":
2297                                                 suffix = suffix + "indev"
2298
2299                                         for mytype in Package._dep_keys:
2300
2301                                                 mykey = "dependency.bad" + suffix
2302                                                 myvalue = myaux[mytype]
2303                                                 if not myvalue:
2304                                                         continue
2305
2306                                                 success, atoms = portage.dep_check(myvalue, portdb,
2307                                                         dep_settings, use="all", mode=matchmode,
2308                                                         trees=trees)
2309
2310                                                 if success:
2311                                                         if atoms:
2312
2313                                                                 # Don't bother with dependency.unknown for
2314                                                                 # cases in which *DEPEND.bad is triggered.
2315                                                                 for atom in atoms:
2316                                                                         # dep_check returns all blockers and they
2317                                                                         # aren't counted for *DEPEND.bad, so we
2318                                                                         # ignore them here.
2319                                                                         if not atom.blocker:
2320                                                                                 unknown_pkgs.discard(
2321                                                                                         (mytype, atom.unevaluated_atom))
2322
2323                                                                 if not prof.sub_path:
2324                                                                         # old-style virtuals currently aren't
2325                                                                         # resolvable with empty profile, since
2326                                                                         # 'virtuals' mappings are unavailable
2327                                                                         # (it would be expensive to search
2328                                                                         # for PROVIDE in all ebuilds)
2329                                                                         atoms = [atom for atom in atoms if not \
2330                                                                                 (atom.cp.startswith('virtual/') and \
2331                                                                                 not portdb.cp_list(atom.cp))]
2332
2333                                                                 #we have some unsolvable deps
2334                                                                 #remove ! deps, which always show up as unsatisfiable
2335                                                                 atoms = [str(atom.unevaluated_atom) \
2336                                                                         for atom in atoms if not atom.blocker]
2337
2338                                                                 #if we emptied out our list, continue:
2339                                                                 if not atoms:
2340                                                                         continue
2341                                                                 stats[mykey] += 1
2342                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2343                                                                         (relative_path, mytype, keyword,
2344                                                                         prof, repr(atoms)))
2345                                                 else:
2346                                                         stats[mykey] += 1
2347                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2348                                                                 (relative_path, mytype, keyword,
2349                                                                 prof, repr(atoms)))
2350
2351                 if not baddepsyntax and unknown_pkgs:
2352                         type_map = {}
2353                         for mytype, atom in unknown_pkgs:
2354                                 type_map.setdefault(mytype, set()).add(atom)
2355                         for mytype, atoms in type_map.items():
2356                                 stats["dependency.unknown"] += 1
2357                                 fails["dependency.unknown"].append("%s: %s: %s" %
2358                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2359
2360         # check if there are unused local USE-descriptions in metadata.xml
2361         # (unless there are any invalids, to avoid noise)
2362         if allvalid:
2363                 for myflag in muselist.difference(used_useflags):
2364                         stats["metadata.warning"] += 1
2365                         fails["metadata.warning"].append(
2366                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2367                                 (x, myflag))
2368
2369 if options.if_modified == "y" and len(effective_scanlist) < 1:
2370         logging.warn("--if-modified is enabled, but no modified packages were found!")
2371
2372 if options.mode == "manifest":
2373         sys.exit(dofail)
2374
2375 # dofail will be set to 1 if we have failed in at least one non-warning category
2376 dofail = 0
2377 # dowarn will be set to 1 if we tripped any warnings
2378 dowarn = 0
2379 # dofull will be set if we should print a "repoman full" informational message
2380 dofull = options.mode != 'full'
2381
2382 for x in qacats:
2383         if not stats[x]:
2384                 continue
2385         dowarn = 1
2386         if x not in qawarnings:
2387                 dofail = 1
2388
2389 if dofail or \
2390         (dowarn and not (options.quiet or options.mode == "scan")):
2391         dofull = 0
2392
2393 # Save QA output so that it can be conveniently displayed
2394 # in $EDITOR while the user creates a commit message.
2395 # Otherwise, the user would not be able to see this output
2396 # once the editor has taken over the screen.
2397 qa_output = io.StringIO()
2398 style_file = ConsoleStyleFile(sys.stdout)
2399 if options.mode == 'commit' and \
2400         (not commitmessage or not commitmessage.strip()):
2401         style_file.write_listener = qa_output
2402 console_writer = StyleWriter(file=style_file, maxcol=9999)
2403 console_writer.style_listener = style_file.new_styles
2404
2405 f = formatter.AbstractFormatter(console_writer)
2406
2407 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2408
2409 style_file.flush()
2410 del console_writer, f, style_file
2411 qa_output = qa_output.getvalue()
2412 qa_output = qa_output.splitlines(True)
2413
2414 suggest_ignore_masked = False
2415 suggest_include_dev = False
2416
2417 if have_pmasked and not (options.without_mask or options.ignore_masked):
2418         suggest_ignore_masked = True
2419 if have_dev_keywords and not options.include_dev:
2420         suggest_include_dev = True
2421
2422 if suggest_ignore_masked or suggest_include_dev:
2423         print()
2424         if suggest_ignore_masked:
2425                 print(bold("Note: use --without-mask to check " + \
2426                         "KEYWORDS on dependencies of masked packages"))
2427
2428         if suggest_include_dev:
2429                 print(bold("Note: use --include-dev (-d) to check " + \
2430                         "dependencies for 'dev' profiles"))
2431         print()
2432
2433 if options.mode != 'commit':
2434         if dofull:
2435                 print(bold("Note: type \"repoman full\" for a complete listing."))
2436         if dowarn and not dofail:
2437                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2438         elif not dofail:
2439                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2440         elif dofail:
2441                 print(bad("Please fix these important QA issues first."))
2442                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2443                 sys.exit(1)
2444 else:
2445         if dofail and can_force and options.force and not options.pretend:
2446                 print(green("RepoMan sez:") + \
2447                         " \"You want to commit even with these QA issues?\n" + \
2448                         "              I'll take it this time, but I'm not happy.\"\n")
2449         elif dofail:
2450                 if options.force and not can_force:
2451                         print(bad("The --force option has been disabled due to extraordinary issues."))
2452                 print(bad("Please fix these important QA issues first."))
2453                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2454                 sys.exit(1)
2455
2456         if options.pretend:
2457                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2458
2459         myunadded = []
2460         if vcs == "cvs":
2461                 try:
2462                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2463                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2464                 except SystemExit as e:
2465                         raise  # TODO propagate this
2466                 except:
2467                         err("Error retrieving CVS tree; exiting.")
2468         if vcs == "svn":
2469                 try:
2470                         with repoman_popen("svn status --no-ignore") as f:
2471                                 svnstatus = f.readlines()
2472                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2473                 except SystemExit as e:
2474                         raise  # TODO propagate this
2475                 except:
2476                         err("Error retrieving SVN info; exiting.")
2477         if vcs == "git":
2478                 # get list of files not under version control or missing
2479                 myf = repoman_popen("git ls-files --others")
2480                 myunadded = ["./" + elem[:-1] for elem in myf]
2481                 myf.close()
2482         if vcs == "bzr":
2483                 try:
2484                         with repoman_popen("bzr status -S .") as f:
2485                                 bzrstatus = f.readlines()
2486                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2487                 except SystemExit as e:
2488                         raise  # TODO propagate this
2489                 except:
2490                         err("Error retrieving bzr info; exiting.")
2491         if vcs == "hg":
2492                 with repoman_popen("hg status --no-status --unknown .") as f:
2493                         myunadded = f.readlines()
2494                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2495
2496                 # Mercurial doesn't handle manually deleted files as removed from
2497                 # the repository, so the user need to remove them before commit,
2498                 # using "hg remove [FILES]"
2499                 with repoman_popen("hg status --no-status --deleted .") as f:
2500                         mydeleted = f.readlines()
2501                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2502
2503
2504         myautoadd = []
2505         if myunadded:
2506                 for x in range(len(myunadded)-1, -1, -1):
2507                         xs = myunadded[x].split("/")
2508                         if xs[-1] == "files":
2509                                 print("!!! files dir is not added! Please correct this.")
2510                                 sys.exit(-1)
2511                         elif xs[-1] == "Manifest":
2512                                 # It's a manifest... auto add
2513                                 myautoadd += [myunadded[x]]
2514                                 del myunadded[x]
2515
2516         if myunadded:
2517                 print(red("!!! The following files are in your local tree but are not added to the master"))
2518                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2519                 for x in myunadded:
2520                         print("   ", x)
2521                 print()
2522                 print()
2523                 sys.exit(1)
2524
2525         if vcs == "hg" and mydeleted:
2526                 print(red("!!! The following files are removed manually from your local tree but are not"))
2527                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2528                 for x in mydeleted:
2529                         print("   ", x)
2530                 print()
2531                 print()
2532                 sys.exit(1)
2533
2534         if vcs == "cvs":
2535                 mycvstree = cvstree.getentries("./", recursive=1)
2536                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2537                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2538                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2539                 bin_blob_pattern = re.compile("^-kb$")
2540                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2541                         recursive=1, basedir="./"))
2542
2543         if vcs == "svn":
2544                 with repoman_popen("svn status") as f:
2545                         svnstatus = f.readlines()
2546                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2547                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2548                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2549
2550                 # Subversion expands keywords specified in svn:keywords properties.
2551                 with repoman_popen("svn propget -R svn:keywords") as f:
2552                         props = f.readlines()
2553                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2554                         for prop in props if " - " in prop)
2555
2556         elif vcs == "git":
2557                 with repoman_popen("git diff-index --name-only "
2558                         "--relative --diff-filter=M HEAD") as f:
2559                         mychanged = f.readlines()
2560                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2561
2562                 with repoman_popen("git diff-index --name-only "
2563                         "--relative --diff-filter=A HEAD") as f:
2564                         mynew = f.readlines()
2565                 mynew = ["./" + elem[:-1] for elem in mynew]
2566
2567                 with repoman_popen("git diff-index --name-only "
2568                         "--relative --diff-filter=D HEAD") as f:
2569                         myremoved = f.readlines()
2570                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2571
2572         if vcs == "bzr":
2573                 with repoman_popen("bzr status -S .") as f:
2574                         bzrstatus = f.readlines()
2575                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2576                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2577                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2578                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2579                 # Bazaar expands nothing.
2580
2581         if vcs == "hg":
2582                 with repoman_popen("hg status --no-status --modified .") as f:
2583                         mychanged = f.readlines()
2584                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2585
2586                 with repoman_popen("hg status --no-status --added .") as f:
2587                         mynew = f.readlines()
2588                 mynew = ["./" + elem.rstrip() for elem in mynew]
2589
2590                 with repoman_popen("hg status --no-status --removed .") as f:
2591                         myremoved = f.readlines()
2592                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2593
2594         if vcs:
2595                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2596                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2597                         print()
2598                         print("(Didn't find any changed files...)")
2599                         print()
2600                         sys.exit(1)
2601
2602         # Manifests need to be regenerated after all other commits, so don't commit
2603         # them now even if they have changed.
2604         mymanifests = set()
2605         myupdates = set()
2606         for f in mychanged + mynew:
2607                 if "Manifest" == os.path.basename(f):
2608                         mymanifests.add(f)
2609                 else:
2610                         myupdates.add(f)
2611         myupdates.difference_update(myremoved)
2612         myupdates = list(myupdates)
2613         mymanifests = list(mymanifests)
2614         myheaders = []
2615         mydirty = []
2616
2617         commitmessage = options.commitmsg
2618         if options.commitmsgfile:
2619                 try:
2620                         f = io.open(_unicode_encode(options.commitmsgfile,
2621                         encoding=_encodings['fs'], errors='strict'),
2622                         mode='r', encoding=_encodings['content'], errors='replace')
2623                         commitmessage = f.read()
2624                         f.close()
2625                         del f
2626                 except (IOError, OSError) as e:
2627                         if e.errno == errno.ENOENT:
2628                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2629                         else:
2630                                 raise
2631                 # We've read the content so the file is no longer needed.
2632                 commitmessagefile = None
2633         if not commitmessage or not commitmessage.strip():
2634                 try:
2635                         editor = os.environ.get("EDITOR")
2636                         if editor and utilities.editor_is_executable(editor):
2637                                 commitmessage = utilities.get_commit_message_with_editor(
2638                                         editor, message=qa_output)
2639                         else:
2640                                 commitmessage = utilities.get_commit_message_with_stdin()
2641                 except KeyboardInterrupt:
2642                         exithandler()
2643                 if not commitmessage or not commitmessage.strip():
2644                         print("* no commit message?  aborting commit.")
2645                         sys.exit(1)
2646         commitmessage = commitmessage.rstrip()
2647         changelog_msg = commitmessage
2648         portage_version = getattr(portage, "VERSION", None)
2649         gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
2650         dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
2651         if portage_version is None:
2652                 sys.stderr.write("Failed to insert portage version in message!\n")
2653                 sys.stderr.flush()
2654                 portage_version = "Unknown"
2655
2656         report_options = []
2657         if options.force:
2658                 report_options.append("--force")
2659         if options.ignore_arches:
2660                 report_options.append("--ignore-arches")
2661         if include_arches is not None:
2662                 report_options.append("--include-arches=\"%s\"" %
2663                         " ".join(sorted(include_arches)))
2664
2665         if vcs == "git":
2666                 # Use new footer only for git (see bug #438364).
2667                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2668                 if report_options:
2669                         commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
2670                 if sign_manifests:
2671                         commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
2672                 if dco_sob:
2673                         commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
2674         else:
2675                 unameout = platform.system() + " "
2676                 if platform.system() in ["Darwin", "SunOS"]:
2677                         unameout += platform.processor()
2678                 else:
2679                         unameout += platform.machine()
2680                 commit_footer = "\n\n"
2681                 if dco_sob:
2682                         commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
2683                 commit_footer += "(Portage version: %s/%s/%s" % \
2684                         (portage_version, vcs, unameout)
2685                 if report_options:
2686                         commit_footer += ", RepoMan options: " + " ".join(report_options)
2687                 if sign_manifests:
2688                         commit_footer += ", signed Manifest commit with key %s" % \
2689                                 (gpg_key, )
2690                 else:
2691                         commit_footer += ", unsigned Manifest commit"
2692                 commit_footer += ")"
2693
2694         commitmessage += commit_footer
2695
2696         if options.echangelog in ('y', 'force'):
2697                 logging.info("checking for unmodified ChangeLog files")
2698                 committer_name = utilities.get_committer_name(env=repoman_settings)
2699                 for x in sorted(vcs_files_to_cps(
2700                         chain(myupdates, mymanifests, myremoved))):
2701                         catdir, pkgdir = x.split("/")
2702                         checkdir = repodir + "/" + x
2703                         checkdir_relative = ""
2704                         if repolevel < 3:
2705                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2706                         if repolevel < 2:
2707                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2708                         checkdir_relative = os.path.join(".", checkdir_relative)
2709
2710                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2711                         changelog_modified = changelog_path in modified_changelogs
2712                         if changelog_modified and options.echangelog != 'force':
2713                                 continue
2714
2715                         # get changes for this package
2716                         cdrlen = len(checkdir_relative)
2717                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2718                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2719                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2720
2721                         # Skip ChangeLog generation if only the Manifest was modified,
2722                         # as discussed in bug #398009.
2723                         nontrivial_cl_files = set()
2724                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2725                         nontrivial_cl_files.difference_update(['Manifest'])
2726                         if not nontrivial_cl_files and options.echangelog != 'force':
2727                                 continue
2728
2729                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2730                                 committer_name, changelog_msg,
2731                                 os.path.join(repodir, 'skel.ChangeLog'),
2732                                 catdir, pkgdir,
2733                                 new=clnew, removed=clremoved, changed=clchanged,
2734                                 pretend=options.pretend)
2735                         if new_changelog is None:
2736                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2737                                         level=logging.ERROR, noiselevel=-1)
2738                                 sys.exit(1)
2739
2740                         # if the ChangeLog was just created, add it to vcs
2741                         if new_changelog:
2742                                 myautoadd.append(changelog_path)
2743                                 # myautoadd is appended to myupdates below
2744                         else:
2745                                 myupdates.append(changelog_path)
2746
2747                         if options.ask and not options.pretend:
2748                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2749                                 repoman_settings["O"] = checkdir
2750                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2751
2752         if myautoadd:
2753                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2754                 add_cmd = [vcs, "add"]
2755                 add_cmd += myautoadd
2756                 if options.pretend:
2757                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2758                                 noiselevel=-1)
2759                 else:
2760
2761                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2762                                 not os.path.isabs(add_cmd[0]):
2763                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2764                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2765                                 fullname = find_binary(add_cmd[0])
2766                                 if fullname is None:
2767                                         raise portage.exception.CommandNotFound(add_cmd[0])
2768                                 add_cmd[0] = fullname
2769
2770                         add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2771                         retcode = subprocess.call(add_cmd)
2772                         if retcode != os.EX_OK:
2773                                 logging.error(
2774                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2775                                 sys.exit(retcode)
2776
2777                 myupdates += myautoadd
2778
2779         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2780
2781         if vcs not in ('cvs', 'svn'):
2782                 # With git, bzr and hg, there's never any keyword expansion, so
2783                 # there's no need to regenerate manifests and all files will be
2784                 # committed in one big commit at the end.
2785                 print()
2786         elif not repo_config.thin_manifest:
2787                 if vcs == 'cvs':
2788                         headerstring = "'\$(Header|Id).*\$'"
2789                 elif vcs == "svn":
2790                         svn_keywords = dict((k.lower(), k) for k in [
2791                                         "Rev",
2792                                         "Revision",
2793                                         "LastChangedRevision",
2794                                         "Date",
2795                                         "LastChangedDate",
2796                                         "Author",
2797                                         "LastChangedBy",
2798                                         "URL",
2799                                         "HeadURL",
2800                                         "Id",
2801                                         "Header",
2802                         ])
2803
2804                 for myfile in myupdates:
2805
2806                         # for CVS, no_expansion contains files that are excluded from expansion
2807                         if vcs == "cvs":
2808                                 if myfile in no_expansion:
2809                                         continue
2810
2811                         # for SVN, expansion contains files that are included in expansion
2812                         elif vcs == "svn":
2813                                 if myfile not in expansion:
2814                                         continue
2815
2816                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2817                                 enabled_keywords = []
2818                                 for k in expansion[myfile]:
2819                                         keyword = svn_keywords.get(k.lower())
2820                                         if keyword is not None:
2821                                                 enabled_keywords.append(keyword)
2822
2823                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2824
2825                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2826                                 portage._shell_quote(myfile))
2827                         if myout[0] == 0:
2828                                 myheaders.append(myfile)
2829
2830                 print("%s have headers that will change." % green(str(len(myheaders))))
2831                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2832
2833         logging.info("myupdates: %s", myupdates)
2834         logging.info("myheaders: %s", myheaders)
2835
2836         if options.ask and userquery('Commit changes?', True) != 'Yes':
2837                 print("* aborting commit.")
2838                 sys.exit(128 + signal.SIGINT)
2839
2840         # Handle the case where committed files have keywords which
2841         # will change and need a priming commit before the Manifest
2842         # can be committed.
2843         if (myupdates or myremoved) and myheaders:
2844                 myfiles = myupdates + myremoved
2845                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2846                 mymsg = os.fdopen(fd, "wb")
2847                 mymsg.write(_unicode_encode(commitmessage))
2848                 mymsg.close()
2849
2850                 print()
2851                 print(green("Using commit message:"))
2852                 print(green("------------------------------------------------------------------------------"))
2853                 print(commitmessage)
2854                 print(green("------------------------------------------------------------------------------"))
2855                 print()
2856
2857                 # Having a leading ./ prefix on file paths can trigger a bug in
2858                 # the cvs server when committing files to multiple directories,
2859                 # so strip the prefix.
2860                 myfiles = [f.lstrip("./") for f in myfiles]
2861
2862                 commit_cmd = [vcs]
2863                 commit_cmd.extend(vcs_global_opts)
2864                 commit_cmd.append("commit")
2865                 commit_cmd.extend(vcs_local_opts)
2866                 commit_cmd.extend(["-F", commitmessagefile])
2867                 commit_cmd.extend(myfiles)
2868
2869                 try:
2870                         if options.pretend:
2871                                 print("(%s)" % (" ".join(commit_cmd),))
2872                         else:
2873                                 retval = spawn(commit_cmd, env=commit_env)
2874                                 if retval != os.EX_OK:
2875                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2876                                                 "error code: %s\n") % (vcs, retval),
2877                                                 level=logging.ERROR, noiselevel=-1)
2878                                         sys.exit(retval)
2879                 finally:
2880                         try:
2881                                 os.unlink(commitmessagefile)
2882                         except OSError:
2883                                 pass
2884
2885         # Setup the GPG commands
2886         def gpgsign(filename):
2887                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2888                 if gpgcmd is None:
2889                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2890                                 " Is make.globals missing?")
2891                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2892                         "PORTAGE_GPG_KEY" not in repoman_settings:
2893                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2894                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2895                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2896                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2897                                         os.path.expanduser("~/.gnupg")
2898                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2899                                         % repoman_settings["PORTAGE_GPG_DIR"])
2900                         else:
2901                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2902                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2903                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2904                                 raise portage.exception.InvalidLocation(
2905                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2906                                         repoman_settings["PORTAGE_GPG_DIR"])
2907                 gpgvars = {"FILE": filename}
2908                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2909                         v = repoman_settings.get(k)
2910                         if v is not None:
2911                                 gpgvars[k] = v
2912                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2913                 if options.pretend:
2914                         print("(" + gpgcmd + ")")
2915                 else:
2916                         # Encode unicode manually for bug #310789.
2917                         gpgcmd = portage.util.shlex_split(gpgcmd)
2918
2919                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2920                                 not os.path.isabs(gpgcmd[0]):
2921                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2922                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2923                                 fullname = find_binary(gpgcmd[0])
2924                                 if fullname is None:
2925                                         raise portage.exception.CommandNotFound(gpgcmd[0])
2926                                 gpgcmd[0] = fullname
2927
2928                         gpgcmd = [_unicode_encode(arg,
2929                                 encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2930                         rValue = subprocess.call(gpgcmd)
2931                         if rValue == os.EX_OK:
2932                                 os.rename(filename + ".asc", filename)
2933                         else:
2934                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2935
2936         def need_signature(filename):
2937                 try:
2938                         with open(_unicode_encode(filename,
2939                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2940                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2941                 except IOError as e:
2942                         if e.errno in (errno.ENOENT, errno.ESTALE):
2943                                 return False
2944                         raise
2945
2946         # When files are removed and re-added, the cvs server will put /Attic/
2947         # inside the $Header path. This code detects the problem and corrects it
2948         # so that the Manifest will generate correctly. See bug #169500.
2949         # Use binary mode in order to avoid potential character encoding issues.
2950         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2951         attic_str = b'/Attic/'
2952         attic_replace = b'/'
2953         for x in myheaders:
2954                 f = open(_unicode_encode(x,
2955                         encoding=_encodings['fs'], errors='strict'),
2956                         mode='rb')
2957                 mylines = f.readlines()
2958                 f.close()
2959                 modified = False
2960                 for i, line in enumerate(mylines):
2961                         if cvs_header_re.match(line) is not None and \
2962                                 attic_str in line:
2963                                 mylines[i] = line.replace(attic_str, attic_replace)
2964                                 modified = True
2965                 if modified:
2966                         portage.util.write_atomic(x, b''.join(mylines),
2967                                 mode='wb')
2968
2969         if repolevel == 1:
2970                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2971                         "doing the entire repository.\"\n")
2972
2973         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2974
2975                 for x in sorted(vcs_files_to_cps(
2976                         chain(myupdates, myremoved, mymanifests))):
2977                         repoman_settings["O"] = os.path.join(repodir, x)
2978                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2979
2980         signed = False
2981         if sign_manifests:
2982                 signed = True
2983                 try:
2984                         for x in sorted(vcs_files_to_cps(
2985                                 chain(myupdates, myremoved, mymanifests))):
2986                                 repoman_settings["O"] = os.path.join(repodir, x)
2987                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2988                                 if not need_signature(manifest_path):
2989                                         continue
2990                                 gpgsign(manifest_path)
2991                 except portage.exception.PortageException as e:
2992                         portage.writemsg("!!! %s\n" % str(e))
2993                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2994                         signed = False
2995
2996         if vcs == 'git':
2997                 # It's not safe to use the git commit -a option since there might
2998                 # be some modified files elsewhere in the working tree that the
2999                 # user doesn't want to commit. Therefore, call git update-index
3000                 # in order to ensure that the index is updated with the latest
3001                 # versions of all new and modified files in the relevant portion
3002                 # of the working tree.
3003                 myfiles = mymanifests + myupdates
3004                 myfiles.sort()
3005                 update_index_cmd = ["git", "update-index"]
3006                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
3007                 if options.pretend:
3008                         print("(%s)" % (" ".join(update_index_cmd),))
3009                 else:
3010                         retval = spawn(update_index_cmd, env=os.environ)
3011                         if retval != os.EX_OK:
3012                                 writemsg_level(("!!! Exiting on %s (shell) " + \
3013                                         "error code: %s\n") % (vcs, retval),
3014                                         level=logging.ERROR, noiselevel=-1)
3015                                 sys.exit(retval)
3016
3017         if True:
3018                 myfiles = mymanifests[:]
3019                 # If there are no header (SVN/CVS keywords) changes in
3020                 # the files, this Manifest commit must include the
3021                 # other (yet uncommitted) files.
3022                 if not myheaders:
3023                         myfiles += myupdates
3024                         myfiles += myremoved
3025                 myfiles.sort()
3026
3027                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3028                 mymsg = os.fdopen(fd, "wb")
3029                 mymsg.write(_unicode_encode(commitmessage))
3030                 mymsg.close()
3031
3032                 commit_cmd = []
3033                 if options.pretend and vcs is None:
3034                         # substitute a bogus value for pretend output
3035                         commit_cmd.append("cvs")
3036                 else:
3037                         commit_cmd.append(vcs)
3038                 commit_cmd.extend(vcs_global_opts)
3039                 commit_cmd.append("commit")
3040                 commit_cmd.extend(vcs_local_opts)
3041                 if vcs == "hg":
3042                         commit_cmd.extend(["--logfile", commitmessagefile])
3043                         commit_cmd.extend(myfiles)
3044                 else:
3045                         commit_cmd.extend(["-F", commitmessagefile])
3046                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3047
3048                 try:
3049                         if options.pretend:
3050                                 print("(%s)" % (" ".join(commit_cmd),))
3051                         else:
3052                                 retval = spawn(commit_cmd, env=commit_env)
3053                                 if retval != os.EX_OK:
3054                                         if repo_config.sign_commit and vcs == 'git' and \
3055                                                 not git_supports_gpg_sign():
3056                                                 # Inform user that newer git is needed (bug #403323).
3057                                                 logging.error(
3058                                                         "Git >=1.7.9 is required for signed commits!")
3059
3060                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3061                                                 "error code: %s\n") % (vcs, retval),
3062                                                 level=logging.ERROR, noiselevel=-1)
3063                                         sys.exit(retval)
3064                 finally:
3065                         try:
3066                                 os.unlink(commitmessagefile)
3067                         except OSError:
3068                                 pass
3069
3070         print()
3071         if vcs:
3072                 print("Commit complete.")
3073         else:
3074                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3075         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3076 sys.exit(0)