bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import codecs
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27 from itertools import chain
  28 from stat import S_ISDIR
  29
  30 try:
  31         from urllib.parse import urlparse
  32 except ImportError:
  33         from urlparse import urlparse
  34
  35 from os import path as osp
  36 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  37 sys.path.insert(0, pym_path)
  38 import portage
  39 portage._internal_caller = True
  40 portage._disable_legacy_globals()
  41
  42 try:
  43         import xml.etree.ElementTree
  44         from xml.parsers.expat import ExpatError
  45 except (SystemExit, KeyboardInterrupt):
  46         raise
  47 except (ImportError, SystemError, RuntimeError, Exception):
  48         # broken or missing xml support
  49         # http://bugs.python.org/issue14988
  50         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  51         from portage.output import EOutput
  52         out = EOutput()
  53         for line in msg:
  54                 out.eerror(line)
  55         sys.exit(1)
  56
  57 from portage import os
  58 from portage import _encodings
  59 from portage import _unicode_encode
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 import portage.repository.config
  69 from portage import cvstree, normalize_path
  70 from portage import util
  71 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  72         ParseError, PermissionDenied)
  73 from portage.dep import Atom
  74 from portage.process import find_binary, spawn
  75 from portage.output import bold, create_color_func, \
  76         green, nocolor, red
  77 from portage.output import ConsoleStyleFile, StyleWriter
  78 from portage.util import writemsg_level
  79 from portage.package.ebuild.digestgen import digestgen
  80 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  81
  82 if sys.hexversion >= 0x3000000:
  83         basestring = str
  84
  85 util.initialize_logger()
  86
  87 # 14 is the length of DESCRIPTION=""
  88 max_desc_len = 100
  89 allowed_filename_chars="a-zA-Z0-9._-+:"
  90 pv_toolong_re = re.compile(r'[0-9]{19,}')
  91 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  92 bad = create_color_func("BAD")
  93
  94 # A sane umask is needed for files that portage creates.
  95 os.umask(0o22)
  96 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  97 # behave incrementally.
  98 repoman_incrementals = tuple(x for x in \
  99         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 100 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 101 repoman_settings = portage.config(config_root=config_root, local_config=False)
 102
 103 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 104         repoman_settings.get('TERM') == 'dumb' or \
 105         not sys.stdout.isatty():
 106         nocolor()
 107
 108 def warn(txt):
 109         print("repoman: " + txt)
 110
 111 def err(txt):
 112         warn(txt)
 113         sys.exit(1)
 114
 115 def exithandler(signum=None, frame=None):
 116         logging.fatal("Interrupted; exiting...")
 117         if signum is None:
 118                 sys.exit(1)
 119         else:
 120                 sys.exit(128 + signum)
 121
 122 signal.signal(signal.SIGINT, exithandler)
 123
 124 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 125         """Repoman needs its own HelpFormatter for now, because the default ones
 126         murder the help text."""
 127
 128         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 129                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 130
 131         def format_description(self, description):
 132                 return description
 133
 134 class RepomanOptionParser(optparse.OptionParser):
 135         """Add the on_tail function, ruby has it, optionParser should too
 136         """
 137
 138         def __init__(self, *args, **kwargs):
 139                 optparse.OptionParser.__init__(self, *args, **kwargs)
 140                 self.tail = ""
 141
 142         def on_tail(self, description):
 143                 self.tail += description
 144
 145         def format_help(self, formatter=None):
 146                 result = optparse.OptionParser.format_help(self, formatter)
 147                 result += self.tail
 148                 return result
 149
 150
 151 def ParseArgs(argv, qahelp):
 152         """This function uses a customized optionParser to parse command line arguments for repoman
 153         Args:
 154           argv - a sequence of command line arguments
 155                 qahelp - a dict of qa warning to help message
 156         Returns:
 157           (opts, args), just like a call to parser.parse_args()
 158         """
 159
 160         argv = portage._decode_argv(argv)
 161
 162         modes = {
 163                 'commit' : 'Run a scan then commit changes',
 164                 'ci' : 'Run a scan then commit changes',
 165                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 166                 'full' : 'Scan directory tree and print all issues (not a summary)',
 167                 'help' : 'Show this screen',
 168                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 169                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 170                 'scan' : 'Scan directory tree for QA issues'
 171         }
 172
 173         mode_keys = list(modes)
 174         mode_keys.sort()
 175
 176         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 177         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 178         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 179         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 180         parser.description += "\nmodes: " + " | ".join(map(green, mode_keys))
 181
 182         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 183                 help='Request a confirmation before commiting')
 184
 185         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 186                 help='specify a commit message on the command line')
 187
 188         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 189                 help='specify a path to a file that contains a commit message')
 190
 191         parser.add_option('--digest',
 192                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 193                 help='Automatically update Manifest digests for modified files')
 194
 195         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 196                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 197
 198         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 199                 help='do not print unnecessary messages')
 200
 201         parser.add_option(
 202                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 203                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 204                 'regardless of modification if \'force\' is specified)')
 205
 206         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 207                 help='Commit with QA violations')
 208
 209         parser.add_option('--vcs', dest='vcs',
 210                 help='Force using specific VCS instead of autodetection')
 211
 212         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 213                 help='be very verbose in output', default=0)
 214
 215         parser.add_option('-V', '--version', dest='version', action='store_true',
 216                 help='show version info')
 217
 218         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 219                 default=False, help='forces the metadata.xml parse check to be carried out')
 220
 221         parser.add_option(
 222                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 223                 metavar="<y|n>",
 224                 help='only check packages that have uncommitted modifications')
 225
 226         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 227                 default=False, help='ignore arch-specific failures (where arch != host)')
 228
 229         parser.add_option("--ignore-default-opts",
 230                 action="store_true",
 231                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 232
 233         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 234                 default=False, help='ignore masked packages (not allowed with commit mode)')
 235
 236         parser.add_option('--include-arches', dest='include_arches',
 237                 metavar='ARCHES', action='append',
 238                 help='A space separated list of arches used to '
 239                 'filter the selection of profiles for dependency checks')
 240
 241         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 242                 default=False, help='include dev profiles in dependency checks')
 243
 244         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 245                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 246
 247         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 248                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 249
 250         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 251                 help='specify which mode repoman will run in (default=full)')
 252
 253         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 254
 255         for k in mode_keys:
 256                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 257
 258         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 259
 260         sorted_qa = list(qahelp)
 261         sorted_qa.sort()
 262         for k in sorted_qa:
 263                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 264
 265         opts, args = parser.parse_args(argv[1:])
 266
 267         if not opts.ignore_default_opts:
 268                 default_opts = portage.util.shlex_split(
 269                         repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
 270                 if default_opts:
 271                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 272
 273         if opts.mode == 'help':
 274                 parser.print_help(short=False)
 275
 276         for arg in args:
 277                 if arg in modes:
 278                         if not opts.mode:
 279                                 opts.mode = arg
 280                                 break
 281                 else:
 282                         parser.error("invalid mode: %s" % arg)
 283
 284         if not opts.mode:
 285                 opts.mode = 'full'
 286
 287         if opts.mode == 'ci':
 288                 opts.mode = 'commit'  # backwards compat shortcut
 289
 290         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 291         for val in range(opts.verbosity):
 292                 logger = logging.getLogger()
 293                 logger.setLevel(logger.getEffectiveLevel() - 10)
 294
 295         for val in range(opts.quiet):
 296                 logger = logging.getLogger()
 297                 logger.setLevel(logger.getEffectiveLevel() + 10)
 298
 299         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 300                 if opts.ignore_masked:
 301                         opts.ignore_masked = False
 302                         logging.warn('Commit mode automatically disables --ignore-masked')
 303                 if opts.without_mask:
 304                         opts.without_mask = False
 305                         logging.warn('Commit mode automatically disables --without-mask')
 306
 307         return (opts, args)
 308
 309 qahelp = {
 310         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 311         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 312         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 313         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 314         "changelog.missing": "Missing ChangeLog files",
 315         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 316         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 317         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 318         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 319         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 320         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 321         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 322         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 323         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 324         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 325         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 326         "file.size": "Files in the files directory must be under 20 KiB",
 327         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 328         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 329         "file.UTF8": "File is not UTF8 compliant",
 330         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 331         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 332         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 333         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 334         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 335         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 336         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 337         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 338         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 339         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 340         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 341         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 342         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 343         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 344         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 345         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 346         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 347         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 348         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 349         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 350         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 351         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 352         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 353         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 354         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 355         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 356         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 357         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 358         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 359         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 360         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 361         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 362         "variable.readonly": "Assigning a readonly variable",
 363         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 364         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 365         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 366         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 367         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 368         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 369         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 370         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 371         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 372         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 373         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 374         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 375         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 376         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 377         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 378         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 379         "ebuild.badheader": "This ebuild has a malformed header",
 380         "manifest.bad": "Manifest has missing or incorrect digests",
 381         "metadata.missing": "Missing metadata.xml files",
 382         "metadata.bad": "Bad metadata.xml files",
 383         "metadata.warning": "Warnings in metadata.xml files",
 384         "portage.internal": "The ebuild uses an internal Portage function or variable",
 385         "repo.eapi.banned": "The ebuild uses an EAPI which is banned by the repository's metadata/layout.conf settings",
 386         "repo.eapi.deprecated": "The ebuild uses an EAPI which is deprecated by the repository's metadata/layout.conf settings",
 387         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 388         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 389         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 390         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 391 }
 392
 393 qacats = list(qahelp)
 394 qacats.sort()
 395
 396 qawarnings = set((
 397 "changelog.missing",
 398 "changelog.notadded",
 399 "dependency.unknown",
 400 "digest.assumed",
 401 "digest.unused",
 402 "ebuild.notadded",
 403 "ebuild.nesteddie",
 404 "dependency.badmasked",
 405 "dependency.badindev",
 406 "dependency.badmaskedindev",
 407 "dependency.badtilde",
 408 "DESCRIPTION.toolong",
 409 "EAPI.deprecated",
 410 "HOMEPAGE.virtual",
 411 "LICENSE.deprecated",
 412 "LICENSE.virtual",
 413 "KEYWORDS.dropped",
 414 "KEYWORDS.stupid",
 415 "KEYWORDS.missing",
 416 "PDEPEND.suspect",
 417 "RDEPEND.implicit",
 418 "RDEPEND.suspect",
 419 "virtual.suspect",
 420 "RESTRICT.invalid",
 421 "ebuild.minorsyn",
 422 "ebuild.badheader",
 423 "ebuild.patches",
 424 "file.size",
 425 "inherit.unused",
 426 "inherit.deprecated",
 427 "java.eclassesnotused",
 428 "wxwidgets.eclassnotused",
 429 "metadata.warning",
 430 "portage.internal",
 431 "repo.eapi.deprecated",
 432 "usage.obsolete",
 433 "upstream.workaround",
 434 "LIVEVCS.stable",
 435 "LIVEVCS.unmasked",
 436 ))
 437
 438 if portage.const._ENABLE_INHERIT_CHECK:
 439         # This is experimental, so it's non-fatal.
 440         qawarnings.add("inherit.missing")
 441
 442 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 443
 444 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 445 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 446 allvars.update(Package.metadata_keys)
 447 allvars = sorted(allvars)
 448 commitmessage = None
 449 for x in missingvars:
 450         x += ".missing"
 451         if x not in qacats:
 452                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 453                 qacats.append(x)
 454                 qawarnings.add(x)
 455
 456 valid_restrict = frozenset(["binchecks", "bindist",
 457         "fetch", "installsources", "mirror", "preserve-libs",
 458         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 459
 460 live_eclasses = frozenset([
 461         "bzr",
 462         "cvs",
 463         "darcs",
 464         "git",
 465         "git-2",
 466         "mercurial",
 467         "subversion",
 468         "tla",
 469 ])
 470
 471 suspect_rdepend = frozenset([
 472         "app-arch/cabextract",
 473         "app-arch/rpm2targz",
 474         "app-doc/doxygen",
 475         "dev-lang/nasm",
 476         "dev-lang/swig",
 477         "dev-lang/yasm",
 478         "dev-perl/extutils-pkgconfig",
 479         "dev-util/byacc",
 480         "dev-util/cmake",
 481         "dev-util/ftjam",
 482         "dev-util/gperf",
 483         "dev-util/gtk-doc",
 484         "dev-util/gtk-doc-am",
 485         "dev-util/intltool",
 486         "dev-util/jam",
 487         "dev-util/pkg-config-lite",
 488         "dev-util/pkgconf",
 489         "dev-util/pkgconfig",
 490         "dev-util/pkgconfig-openbsd",
 491         "dev-util/scons",
 492         "dev-util/unifdef",
 493         "dev-util/yacc",
 494         "media-gfx/ebdftopcf",
 495         "sys-apps/help2man",
 496         "sys-devel/autoconf",
 497         "sys-devel/automake",
 498         "sys-devel/bin86",
 499         "sys-devel/bison",
 500         "sys-devel/dev86",
 501         "sys-devel/flex",
 502         "sys-devel/m4",
 503         "sys-devel/pmake",
 504         "virtual/linux-sources",
 505         "virtual/pkgconfig",
 506         "x11-misc/bdftopcf",
 507         "x11-misc/imake",
 508 ])
 509
 510 suspect_virtual = {
 511         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 512         "dev-util/pkgconf":"virtual/pkgconfig",
 513         "dev-util/pkgconfig":"virtual/pkgconfig",
 514         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 515         "dev-libs/libusb":"virtual/libusb",
 516         "dev-libs/libusbx":"virtual/libusb",
 517         "dev-libs/libusb-compat":"virtual/libusb",
 518 }
 519
 520 metadata_xml_encoding = 'UTF-8'
 521 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 522         (metadata_xml_encoding,)
 523 metadata_doctype_name = 'pkgmetadata'
 524 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 525 # force refetch if the local copy creation time is older than this
 526 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 527
 528 # file.executable
 529 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 530
 531 options, arguments = ParseArgs(sys.argv, qahelp)
 532
 533 if options.version:
 534         print("Portage", portage.VERSION)
 535         sys.exit(0)
 536
 537 # Set this to False when an extraordinary issue (generally
 538 # something other than a QA issue) makes it impossible to
 539 # commit (like if Manifest generation fails).
 540 can_force = True
 541
 542 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 543 if portdir is None:
 544         sys.exit(1)
 545
 546 myreporoot = os.path.basename(portdir_overlay)
 547 myreporoot += mydir[len(portdir_overlay):]
 548
 549 if options.vcs:
 550         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 551                 vcs = options.vcs
 552         else:
 553                 vcs = None
 554 else:
 555         vcses = utilities.FindVCS()
 556         if len(vcses) > 1:
 557                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 558                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 559                 sys.exit(1)
 560         elif vcses:
 561                 vcs = vcses[0]
 562         else:
 563                 vcs = None
 564
 565 if options.if_modified == "y" and vcs is None:
 566         logging.info("Not in a version controlled repository; "
 567                 "disabling --if-modified.")
 568         options.if_modified = "n"
 569
 570 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 571 vcs_preserves_mtime = vcs in ('cvs',)
 572
 573 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 574 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 575 if vcs_global_opts is None:
 576         if vcs in ('cvs', 'svn'):
 577                 vcs_global_opts = "-q"
 578         else:
 579                 vcs_global_opts = ""
 580 vcs_global_opts = vcs_global_opts.split()
 581
 582 if options.mode == 'commit' and not options.pretend and not vcs:
 583         logging.info("Not in a version controlled repository; enabling pretend mode.")
 584         options.pretend = True
 585
 586 # Ensure that current repository is in the list of enabled repositories.
 587 repodir = os.path.realpath(portdir_overlay)
 588 try:
 589         repoman_settings.repositories.get_repo_for_location(repodir)
 590 except KeyError:
 591         repo_name = portage.repository.config.RepoConfig._read_valid_repo_name(portdir_overlay)[0]
 592         layout_conf_data = portage.repository.config.parse_layout_conf(portdir_overlay)[0]
 593         if layout_conf_data['repo-name']:
 594                 repo_name = layout_conf_data['repo-name']
 595         tmp_conf_file = io.StringIO(textwrap.dedent("""
 596                 [%s]
 597                 location = %s
 598                 """) % (repo_name, portdir_overlay))
 599         # Ensure that the repository corresponding to $PWD overrides a
 600         # repository of the same name referenced by the existing PORTDIR
 601         # or PORTDIR_OVERLAY settings.
 602         repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 603                 (repoman_settings.get('PORTDIR_OVERLAY', ''),
 604                 portage._shell_quote(portdir_overlay))
 605         repositories = portage.repository.config.load_repository_config(repoman_settings, extra_files=[tmp_conf_file])
 606         # We have to call the config constructor again so that attributes
 607         # dependent on config.repositories are initialized correctly.
 608         repoman_settings = portage.config(config_root=config_root, local_config=False, repositories=repositories)
 609
 610 root = repoman_settings['EROOT']
 611 trees = {
 612         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 613 }
 614 portdb = trees[root]['porttree'].dbapi
 615
 616 # Constrain dependency resolution to the master(s)
 617 # that are specified in layout.conf.
 618 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 619 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 620 portdir = portdb.porttrees[0]
 621 commit_env = os.environ.copy()
 622 # list() is for iteration on a copy.
 623 for repo in list(repoman_settings.repositories):
 624         # all paths are canonical
 625         if repo.location not in repo_config.eclass_db.porttrees:
 626                 del repoman_settings.repositories[repo.name]
 627
 628 if repo_config.allow_provide_virtual:
 629         qawarnings.add("virtual.oldstyle")
 630
 631 if repo_config.sign_commit:
 632         if vcs == 'git':
 633                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 634                 # the commit arguments. If key_id is unspecified, then it must be
 635                 # configured by `git config user.signingkey key_id`.
 636                 vcs_local_opts.append("--gpg-sign")
 637                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 638                         # Pass GNUPGHOME to git for bug #462362.
 639                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 640
 641 # In order to disable manifest signatures, repos may set
 642 # "sign-manifests = false" in metadata/layout.conf. This
 643 # can be used to prevent merge conflicts like those that
 644 # thin-manifests is designed to prevent.
 645 sign_manifests = "sign" in repoman_settings.features and \
 646         repo_config.sign_manifest
 647
 648 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 649         options.mode in ("commit",) and not sign_manifests:
 650         msg = ("The '%s' repository has manifest signatures enabled, "
 651         "but FEATURES=sign is currently disabled. In order to avoid this "
 652         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 653         "repositories can disable manifest signatures by setting "
 654         "'sign-manifests = false' in metadata/layout.conf.") % \
 655         (repo_config.name,)
 656         for line in textwrap.wrap(msg, 60):
 657                 logging.warn(line)
 658
 659 if sign_manifests and options.mode in ("commit",) and \
 660         repoman_settings.get("PORTAGE_GPG_KEY") and \
 661         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 662         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 663         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 664                 repoman_settings["PORTAGE_GPG_KEY"])
 665         sys.exit(1)
 666
 667 manifest_hashes = repo_config.manifest_hashes
 668 if manifest_hashes is None:
 669         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 670
 671 if options.mode in ("commit", "fix", "manifest"):
 672         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 673                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 674                 "metadata/layout.conf does not contain the '%s' hash which "
 675                 "is required by this portage version. You will have to "
 676                 "upgrade portage if you want to generate valid manifests for "
 677                 "this repository.") % \
 678                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 679                 for line in textwrap.wrap(msg, 70):
 680                         logging.error(line)
 681                 sys.exit(1)
 682
 683         unsupported_hashes = manifest_hashes.difference(
 684                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 685         if unsupported_hashes:
 686                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 687                 "metadata/layout.conf contains one or more hash types '%s' "
 688                 "which are not supported by this portage version. You will "
 689                 "have to upgrade portage if you want to generate valid "
 690                 "manifests for this repository.") % \
 691                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 692                 for line in textwrap.wrap(msg, 70):
 693                         logging.error(line)
 694                 sys.exit(1)
 695
 696 if options.echangelog is None and repo_config.update_changelog:
 697         options.echangelog = 'y'
 698
 699 if vcs is None:
 700         options.echangelog = 'n'
 701
 702 # The --echangelog option causes automatic ChangeLog generation,
 703 # which invalidates changelog.ebuildadded and changelog.missing
 704 # checks.
 705 # Note: Some don't use ChangeLogs in distributed SCMs.
 706 # It will be generated on server side from scm log,
 707 # before package moves to the rsync server.
 708 # This is needed because they try to avoid merge collisions.
 709 # Gentoo's Council decided to always use the ChangeLog file.
 710 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 711 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 712
 713 if 'digest' in repoman_settings.features and options.digest != 'n':
 714         options.digest = 'y'
 715
 716 logging.debug("vcs: %s" % (vcs,))
 717 logging.debug("repo config: %s" % (repo_config,))
 718 logging.debug("options: %s" % (options,))
 719
 720 # It's confusing if these warnings are displayed without the user
 721 # being told which profile they come from, so disable them.
 722 env = os.environ.copy()
 723 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 724
 725 categories = []
 726 for path in repo_config.eclass_db.porttrees:
 727         categories.extend(portage.util.grabfile(
 728                 os.path.join(path, 'profiles', 'categories')))
 729 repoman_settings.categories = frozenset(
 730         portage.util.stack_lists([categories], incremental=1))
 731 categories = repoman_settings.categories
 732
 733 portdb.settings = repoman_settings
 734 root_config = RootConfig(repoman_settings, trees[root], None)
 735 # We really only need to cache the metadata that's necessary for visibility
 736 # filtering. Anything else can be discarded to reduce memory consumption.
 737 portdb._aux_cache_keys.clear()
 738 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 739
 740 reposplit = myreporoot.split(os.path.sep)
 741 repolevel = len(reposplit)
 742
 743 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 744 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 745 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 746 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 747         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 748         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 749         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 750         print(red("***"))
 751         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 752
 753 # Make startdir relative to the canonical repodir, so that we can pass
 754 # it to digestgen and it won't have to be canonicalized again.
 755 if repolevel == 1:
 756         startdir = repodir
 757 else:
 758         startdir = normalize_path(mydir)
 759         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 760
 761 def caterror(mycat):
 762         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 763
 764 def repoman_getstatusoutput(cmd):
 765         """
 766         Implements an interface similar to getstatusoutput(), but with
 767         customized unicode handling (see bug #310789) and without the shell.
 768         """
 769         args = portage.util.shlex_split(cmd)
 770
 771         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 772                 not os.path.isabs(args[0]):
 773                 # Python 3.1 _execvp throws TypeError for non-absolute executable
 774                 # path passed as bytes (see http://bugs.python.org/issue8513).
 775                 fullname = find_binary(args[0])
 776                 if fullname is None:
 777                         raise portage.exception.CommandNotFound(args[0])
 778                 args[0] = fullname
 779
 780         encoding = _encodings['fs']
 781         args = [_unicode_encode(x,
 782                 encoding=encoding, errors='strict') for x in args]
 783         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 784                 stderr=subprocess.STDOUT)
 785         output = portage._unicode_decode(proc.communicate()[0],
 786                 encoding=encoding, errors='strict')
 787         if output and output[-1] == "\n":
 788                 # getstatusoutput strips one newline
 789                 output = output[:-1]
 790         return (proc.wait(), output)
 791
 792 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 793         """
 794         Implements an interface similar to os.popen(), but with customized
 795         unicode handling (see bug #310789) and without the shell.
 796         """
 797
 798         __slots__ = ('_proc', '_stdout')
 799
 800         def __init__(self, cmd):
 801                 args = portage.util.shlex_split(cmd)
 802
 803                 if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 804                         not os.path.isabs(args[0]):
 805                         # Python 3.1 _execvp throws TypeError for non-absolute executable
 806                         # path passed as bytes (see http://bugs.python.org/issue8513).
 807                         fullname = find_binary(args[0])
 808                         if fullname is None:
 809                                 raise portage.exception.CommandNotFound(args[0])
 810                         args[0] = fullname
 811
 812                 encoding = _encodings['fs']
 813                 args = [_unicode_encode(x,
 814                         encoding=encoding, errors='strict') for x in args]
 815                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 816                 object.__setattr__(self, '_proc', proc)
 817                 object.__setattr__(self, '_stdout',
 818                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 819
 820         def _get_target(self):
 821                 return object.__getattribute__(self, '_stdout')
 822
 823         __enter__ = _get_target
 824
 825         def __exit__(self, exc_type, exc_value, traceback):
 826                 proc = object.__getattribute__(self, '_proc')
 827                 proc.wait()
 828                 proc.stdout.close()
 829
 830 class ProfileDesc(object):
 831         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 832         def __init__(self, arch, status, sub_path, tree_path):
 833                 self.arch = arch
 834                 self.status = status
 835                 if sub_path:
 836                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 837                 self.sub_path = sub_path
 838                 self.tree_path = tree_path
 839                 if tree_path:
 840                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 841                 else:
 842                         self.abs_path = tree_path
 843
 844         def __str__(self):
 845                 if self.sub_path:
 846                         return self.sub_path
 847                 return 'empty profile'
 848
 849 profile_list = []
 850 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 851
 852 # get lists of valid keywords, licenses, and use
 853 kwlist = set()
 854 liclist = set()
 855 uselist = set()
 856 global_pmasklines = []
 857
 858 for path in portdb.porttrees:
 859         try:
 860                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 861         except OSError:
 862                 pass
 863         kwlist.update(portage.grabfile(os.path.join(path,
 864                 "profiles", "arch.list")))
 865
 866         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 867         for x in use_desc:
 868                 x = x.split()
 869                 if x:
 870                         uselist.add(x[0])
 871
 872         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 873         try:
 874                 expand_list = os.listdir(expand_desc_dir)
 875         except OSError:
 876                 pass
 877         else:
 878                 for fn in expand_list:
 879                         if not fn[-5:] == '.desc':
 880                                 continue
 881                         use_prefix = fn[:-5].lower() + '_'
 882                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 883                                 x = x.split()
 884                                 if x:
 885                                         uselist.add(use_prefix + x[0])
 886
 887         global_pmasklines.append(portage.util.grabfile_package(
 888                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 889
 890         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 891         try:
 892                 desc_file = io.open(_unicode_encode(desc_path,
 893                         encoding=_encodings['fs'], errors='strict'),
 894                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 895         except EnvironmentError:
 896                 pass
 897         else:
 898                 for i, x in enumerate(desc_file):
 899                         if x[0] == "#":
 900                                 continue
 901                         arch = x.split()
 902                         if len(arch) == 0:
 903                                 continue
 904                         if len(arch) != 3:
 905                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 906                                         desc_path + " line %d" % (i + 1, ))
 907                         elif arch[0] not in kwlist:
 908                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 909                                         desc_path + " line %d" % (i + 1, ))
 910                         elif arch[2] not in valid_profile_types:
 911                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 912                                         desc_path + " line %d" % (i + 1, ))
 913                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 914                         if not os.path.isdir(profile_desc.abs_path):
 915                                 logging.error(
 916                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 917                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 918                                 continue
 919                         if os.path.exists(
 920                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 921                                 continue
 922                         profile_list.append(profile_desc)
 923                 desc_file.close()
 924
 925 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 926 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 927
 928 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 929 global_pmaskdict = {}
 930 for x in global_pmasklines:
 931         global_pmaskdict.setdefault(x.cp, []).append(x)
 932 del global_pmasklines
 933
 934 def has_global_mask(pkg):
 935         mask_atoms = global_pmaskdict.get(pkg.cp)
 936         if mask_atoms:
 937                 pkg_list = [pkg]
 938                 for x in mask_atoms:
 939                         if portage.dep.match_from_list(x, pkg_list):
 940                                 return x
 941         return None
 942
 943 # Ensure that profile sub_path attributes are unique. Process in reverse order
 944 # so that profiles with duplicate sub_path from overlays will override
 945 # profiles with the same sub_path from parent repos.
 946 profiles = {}
 947 profile_list.reverse()
 948 profile_sub_paths = set()
 949 for prof in profile_list:
 950         if prof.sub_path in profile_sub_paths:
 951                 continue
 952         profile_sub_paths.add(prof.sub_path)
 953         profiles.setdefault(prof.arch, []).append(prof)
 954
 955 # Use an empty profile for checking dependencies of
 956 # packages that have empty KEYWORDS.
 957 prof = ProfileDesc('**', 'stable', '', '')
 958 profiles.setdefault(prof.arch, []).append(prof)
 959
 960 for x in repoman_settings.archlist():
 961         if x[0] == "~":
 962                 continue
 963         if x not in profiles:
 964                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 965                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 966                 print(red("up with the " + x + " team."))
 967                 print()
 968
 969 liclist_deprecated = set()
 970 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 971         liclist_deprecated.update(
 972                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 973
 974 if not liclist:
 975         logging.fatal("Couldn't find licenses?")
 976         sys.exit(1)
 977
 978 if not kwlist:
 979         logging.fatal("Couldn't read KEYWORDS from arch.list")
 980         sys.exit(1)
 981
 982 if not uselist:
 983         logging.fatal("Couldn't find use.desc?")
 984         sys.exit(1)
 985
 986 scanlist = []
 987 if repolevel == 2:
 988         # we are inside a category directory
 989         catdir = reposplit[-1]
 990         if catdir not in categories:
 991                 caterror(catdir)
 992         mydirlist = os.listdir(startdir)
 993         for x in mydirlist:
 994                 if x == "CVS" or x.startswith("."):
 995                         continue
 996                 if os.path.isdir(startdir + "/" + x):
 997                         scanlist.append(catdir + "/" + x)
 998         repo_subdir = catdir + os.sep
 999 elif repolevel == 1:
1000         for x in categories:
1001                 if not os.path.isdir(startdir + "/" + x):
1002                         continue
1003                 for y in os.listdir(startdir + "/" + x):
1004                         if y == "CVS" or y.startswith("."):
1005                                 continue
1006                         if os.path.isdir(startdir + "/" + x + "/" + y):
1007                                 scanlist.append(x + "/" + y)
1008         repo_subdir = ""
1009 elif repolevel == 3:
1010         catdir = reposplit[-2]
1011         if catdir not in categories:
1012                 caterror(catdir)
1013         scanlist.append(catdir + "/" + reposplit[-1])
1014         repo_subdir = scanlist[-1] + os.sep
1015 else:
1016         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
1017                 ' from the current working directory'
1018         logging.critical(msg)
1019         sys.exit(1)
1020
1021 repo_subdir_len = len(repo_subdir)
1022 scanlist.sort()
1023
1024 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
1025
1026 def vcs_files_to_cps(vcs_file_iter):
1027         """
1028         Iterate over the given modified file paths returned from the vcs,
1029         and return a frozenset containing category/pn strings for each
1030         modified package.
1031         """
1032
1033         modified_cps = []
1034
1035         if repolevel == 3:
1036                 if reposplit[-2] in categories and \
1037                         next(vcs_file_iter, None) is not None:
1038                         modified_cps.append("/".join(reposplit[-2:]))
1039
1040         elif repolevel == 2:
1041                 category = reposplit[-1]
1042                 if category in categories:
1043                         for filename in vcs_file_iter:
1044                                 f_split = filename.split(os.sep)
1045                                 # ['.', pn, ...]
1046                                 if len(f_split) > 2:
1047                                         modified_cps.append(category + "/" + f_split[1])
1048
1049         else:
1050                 # repolevel == 1
1051                 for filename in vcs_file_iter:
1052                         f_split = filename.split(os.sep)
1053                         # ['.', category, pn, ...]
1054                         if len(f_split) > 3 and f_split[1] in categories:
1055                                 modified_cps.append("/".join(f_split[1:3]))
1056
1057         return frozenset(modified_cps)
1058
1059 def git_supports_gpg_sign():
1060         status, cmd_output = \
1061                 repoman_getstatusoutput("git --version")
1062         cmd_output = cmd_output.split()
1063         if cmd_output:
1064                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1065                 if version is not None:
1066                         version = [int(x) for x in version.groups()]
1067                         if version[0] > 1 or \
1068                                 (version[0] == 1 and version[1] > 7) or \
1069                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1070                                 return True
1071         return False
1072
1073 def dev_keywords(profiles):
1074         """
1075         Create a set of KEYWORDS values that exist in 'dev'
1076         profiles. These are used
1077         to trigger a message notifying the user when they might
1078         want to add the --include-dev option.
1079         """
1080         type_arch_map = {}
1081         for arch, arch_profiles in profiles.items():
1082                 for prof in arch_profiles:
1083                         arch_set = type_arch_map.get(prof.status)
1084                         if arch_set is None:
1085                                 arch_set = set()
1086                                 type_arch_map[prof.status] = arch_set
1087                         arch_set.add(arch)
1088
1089         dev_keywords = type_arch_map.get('dev', set())
1090         dev_keywords.update(['~' + arch for arch in dev_keywords])
1091         return frozenset(dev_keywords)
1092
1093 dev_keywords = dev_keywords(profiles)
1094
1095 stats = {}
1096 fails = {}
1097
1098 for x in qacats:
1099         stats[x] = 0
1100         fails[x] = []
1101
1102 xmllint_capable = False
1103 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1104
1105 def fetch_metadata_dtd():
1106         """
1107         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1108         metadata_dtd_ctime_interval.
1109         @rtype: bool
1110         @return: True if successful, otherwise False
1111         """
1112
1113         must_fetch = True
1114         metadata_dtd_st = None
1115         current_time = int(time.time())
1116         try:
1117                 metadata_dtd_st = os.stat(metadata_dtd)
1118         except EnvironmentError as e:
1119                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1120                         raise
1121                 del e
1122         else:
1123                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1124                 if abs(current_time - metadata_dtd_st.st_ctime) \
1125                         < metadata_dtd_ctime_interval:
1126                         must_fetch = False
1127
1128         if must_fetch:
1129                 print()
1130                 print(green("***") + " the local copy of metadata.dtd " + \
1131                         "needs to be refetched, doing that now")
1132                 print()
1133                 parsed_url = urlparse(metadata_dtd_uri)
1134                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
1135                 fcmd = repoman_settings.get(setting)
1136                 if not fcmd:
1137                         fcmd = repoman_settings.get('FETCHCOMMAND')
1138                         if not fcmd:
1139                                 logging.error("FETCHCOMMAND is unset")
1140                                 return False
1141
1142                 destdir = repoman_settings["DISTDIR"]
1143                 fd, metadata_dtd_tmp = tempfile.mkstemp(
1144                         prefix='metadata.dtd.', dir=destdir)
1145                 os.close(fd)
1146
1147                 try:
1148                         if not portage.getbinpkg.file_get(metadata_dtd_uri,
1149                                 destdir, fcmd=fcmd,
1150                                 filename=os.path.basename(metadata_dtd_tmp)):
1151                                 logging.error("failed to fetch metadata.dtd from '%s'" %
1152                                         metadata_dtd_uri)
1153                                 return False
1154
1155                         try:
1156                                 portage.util.apply_secpass_permissions(metadata_dtd_tmp,
1157                                         gid=portage.data.portage_gid, mode=0o664, mask=0o2)
1158                         except portage.exception.PortageException:
1159                                 pass
1160
1161                         os.rename(metadata_dtd_tmp, metadata_dtd)
1162                 finally:
1163                         try:
1164                                 os.unlink(metadata_dtd_tmp)
1165                         except OSError:
1166                                 pass
1167
1168         return True
1169
1170 if options.mode == "manifest":
1171         pass
1172 elif not find_binary('xmllint'):
1173         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1174         if options.xml_parse or repolevel == 3:
1175                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1176                 sys.exit(1)
1177 else:
1178         if not fetch_metadata_dtd():
1179                 sys.exit(1)
1180         # this can be problematic if xmllint changes their output
1181         xmllint_capable = True
1182
1183 if options.mode == 'commit' and vcs:
1184         utilities.detect_vcs_conflicts(options, vcs)
1185
1186 if options.mode == "manifest":
1187         pass
1188 elif options.pretend:
1189         print(green("\nRepoMan does a once-over of the neighborhood..."))
1190 else:
1191         print(green("\nRepoMan scours the neighborhood..."))
1192
1193 new_ebuilds = set()
1194 modified_ebuilds = set()
1195 modified_changelogs = set()
1196 mychanged = []
1197 mynew = []
1198 myremoved = []
1199
1200 if vcs == "cvs":
1201         mycvstree = cvstree.getentries("./", recursive=1)
1202         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1203         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1204         if options.if_modified == "y":
1205                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1206
1207 elif vcs == "svn":
1208         with repoman_popen("svn status") as f:
1209                 svnstatus = f.readlines()
1210         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1211         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1212         if options.if_modified == "y":
1213                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1214
1215 elif vcs == "git":
1216         with repoman_popen("git diff-index --name-only "
1217                 "--relative --diff-filter=M HEAD") as f:
1218                 mychanged = f.readlines()
1219         mychanged = ["./" + elem[:-1] for elem in mychanged]
1220
1221         with repoman_popen("git diff-index --name-only "
1222                 "--relative --diff-filter=A HEAD") as f:
1223                 mynew = f.readlines()
1224         mynew = ["./" + elem[:-1] for elem in mynew]
1225         if options.if_modified == "y":
1226                 with repoman_popen("git diff-index --name-only "
1227                         "--relative --diff-filter=D HEAD") as f:
1228                         myremoved = f.readlines()
1229                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1230
1231 elif vcs == "bzr":
1232         with repoman_popen("bzr status -S .") as f:
1233                 bzrstatus = f.readlines()
1234         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1235         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1236         if options.if_modified == "y":
1237                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1238
1239 elif vcs == "hg":
1240         with repoman_popen("hg status --no-status --modified .") as f:
1241                 mychanged = f.readlines()
1242         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1243         with repoman_popen("hg status --no-status --added .") as f:
1244                 mynew = f.readlines()
1245         mynew = ["./" + elem.rstrip() for elem in mynew]
1246         if options.if_modified == "y":
1247                 with repoman_popen("hg status --no-status --removed .") as f:
1248                         myremoved = f.readlines()
1249                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1250
1251 if vcs:
1252         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1253         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1254         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1255                 if os.path.basename(x) == "ChangeLog")
1256
1257 def vcs_new_changed(relative_path):
1258         for x in chain(mychanged, mynew):
1259                 if x == relative_path:
1260                         return True
1261         return False
1262
1263 have_pmasked = False
1264 have_dev_keywords = False
1265 dofail = 0
1266
1267 # NOTE: match-all caches are not shared due to potential
1268 # differences between profiles in _get_implicit_iuse.
1269 arch_caches = {}
1270 arch_xmatch_caches = {}
1271 shared_xmatch_caches = {"cp-list":{}}
1272
1273 include_arches = None
1274 if options.include_arches:
1275         include_arches = set()
1276         include_arches.update(*[x.split() for x in options.include_arches])
1277
1278 # Disable the "ebuild.notadded" check when not in commit mode and
1279 # running `svn status` in every package dir will be too expensive.
1280
1281 check_ebuild_notadded = not \
1282         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1283
1284 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1285 thirdpartymirrors = {}
1286 for k, v in repoman_settings.thirdpartymirrors().items():
1287         for v in v:
1288                 if not v.endswith("/"):
1289                         v += "/"
1290                 thirdpartymirrors[v] = k
1291
1292 class _XMLParser(xml.etree.ElementTree.XMLParser):
1293
1294         def __init__(self, data, **kwargs):
1295                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1296                 self._portage_data = data
1297                 if hasattr(self, 'parser'):
1298                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1299                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1300                         self._base_StartDoctypeDeclHandler = \
1301                                 self.parser.StartDoctypeDeclHandler
1302                         self.parser.StartDoctypeDeclHandler = \
1303                                 self._portage_StartDoctypeDeclHandler
1304
1305         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1306                 if self._base_XmlDeclHandler is not None:
1307                         self._base_XmlDeclHandler(version, encoding, standalone)
1308                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1309
1310         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1311                 has_internal_subset):
1312                 if self._base_StartDoctypeDeclHandler is not None:
1313                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1314                                 has_internal_subset)
1315                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1316
1317 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1318         """
1319         Implements doctype() as required to avoid deprecation warnings with
1320         >=python-2.7.
1321         """
1322         def doctype(self, name, pubid, system):
1323                 pass
1324
1325 try:
1326         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1327 except (EnvironmentError, ParseError, PermissionDenied) as e:
1328         err(str(e))
1329 except FileNotFound:
1330         # TODO: Download as we do for metadata.dtd, but add a way to
1331         # disable for non-gentoo repoman users who may not have herds.
1332         herd_base = None
1333
1334 effective_scanlist = scanlist
1335 if options.if_modified == "y":
1336         effective_scanlist = sorted(vcs_files_to_cps(
1337                 chain(mychanged, mynew, myremoved)))
1338
1339 for x in effective_scanlist:
1340         #ebuilds and digests added to cvs respectively.
1341         logging.info("checking package %s" % x)
1342         # save memory by discarding xmatch caches from previous package(s)
1343         arch_xmatch_caches.clear()
1344         eadded = []
1345         catdir, pkgdir = x.split("/")
1346         checkdir = repodir + "/" + x
1347         checkdir_relative = ""
1348         if repolevel < 3:
1349                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1350         if repolevel < 2:
1351                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1352         checkdir_relative = os.path.join(".", checkdir_relative)
1353         generated_manifest = False
1354
1355         if options.mode == "manifest" or \
1356           (options.mode != 'manifest-check' and options.digest == 'y') or \
1357           options.mode in ('commit', 'fix') and not options.pretend:
1358                 auto_assumed = set()
1359                 fetchlist_dict = portage.FetchlistDict(checkdir,
1360                         repoman_settings, portdb)
1361                 if options.mode == 'manifest' and options.force:
1362                         portage._doebuild_manifest_exempt_depend += 1
1363                         try:
1364                                 distdir = repoman_settings['DISTDIR']
1365                                 mf = repoman_settings.repositories.get_repo_for_location(
1366                                         os.path.dirname(os.path.dirname(checkdir)))
1367                                 mf = mf.load_manifest(checkdir, distdir,
1368                                         fetchlist_dict=fetchlist_dict)
1369                                 mf.create(requiredDistfiles=None,
1370                                         assumeDistHashesAlways=True)
1371                                 for distfiles in fetchlist_dict.values():
1372                                         for distfile in distfiles:
1373                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1374                                                         mf.fhashdict['DIST'].pop(distfile, None)
1375                                                 else:
1376                                                         auto_assumed.add(distfile)
1377                                 mf.write()
1378                         finally:
1379                                 portage._doebuild_manifest_exempt_depend -= 1
1380
1381                 repoman_settings["O"] = checkdir
1382                 try:
1383                         generated_manifest = digestgen(
1384                                 mysettings=repoman_settings, myportdb=portdb)
1385                 except portage.exception.PermissionDenied as e:
1386                         generated_manifest = False
1387                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1388                                 level=logging.ERROR, noiselevel=-1)
1389
1390                 if not generated_manifest:
1391                         print("Unable to generate manifest.")
1392                         dofail = 1
1393
1394                 if options.mode == "manifest":
1395                         if not dofail and options.force and auto_assumed and \
1396                                 'assume-digests' in repoman_settings.features:
1397                                 # Show which digests were assumed despite the --force option
1398                                 # being given. This output will already have been shown by
1399                                 # digestgen() if assume-digests is not enabled, so only show
1400                                 # it here if assume-digests is enabled.
1401                                 pkgs = list(fetchlist_dict)
1402                                 pkgs.sort()
1403                                 portage.writemsg_stdout("  digest.assumed" + \
1404                                         portage.output.colorize("WARN",
1405                                         str(len(auto_assumed)).rjust(18)) + "\n")
1406                                 for cpv in pkgs:
1407                                         fetchmap = fetchlist_dict[cpv]
1408                                         pf = portage.catsplit(cpv)[1]
1409                                         for distfile in sorted(fetchmap):
1410                                                 if distfile in auto_assumed:
1411                                                         portage.writemsg_stdout(
1412                                                                 "   %s::%s\n" % (pf, distfile))
1413                         continue
1414                 elif dofail:
1415                         sys.exit(1)
1416
1417         if not generated_manifest:
1418                 repoman_settings['O'] = checkdir
1419                 repoman_settings['PORTAGE_QUIET'] = '1'
1420                 if not portage.digestcheck([], repoman_settings, strict=1):
1421                         stats["manifest.bad"] += 1
1422                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1423                 repoman_settings.pop('PORTAGE_QUIET', None)
1424
1425         if options.mode == 'manifest-check':
1426                 continue
1427
1428         checkdirlist = os.listdir(checkdir)
1429         ebuildlist = []
1430         pkgs = {}
1431         allvalid = True
1432         for y in checkdirlist:
1433                 if (y in no_exec or y.endswith(".ebuild")) and \
1434                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1435                                 stats["file.executable"] += 1
1436                                 fails["file.executable"].append(os.path.join(checkdir, y))
1437                 if y.endswith(".ebuild"):
1438                         pf = y[:-7]
1439                         ebuildlist.append(pf)
1440                         cpv = "%s/%s" % (catdir, pf)
1441                         try:
1442                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1443                         except KeyError:
1444                                 allvalid = False
1445                                 stats["ebuild.syntax"] += 1
1446                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1447                                 continue
1448                         except IOError:
1449                                 allvalid = False
1450                                 stats["ebuild.output"] += 1
1451                                 fails["ebuild.output"].append(os.path.join(x, y))
1452                                 continue
1453                         if not portage.eapi_is_supported(myaux["EAPI"]):
1454                                 allvalid = False
1455                                 stats["EAPI.unsupported"] += 1
1456                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1457                                 continue
1458                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1459                                 root_config=root_config, type_name="ebuild")
1460
1461         slot_keywords = {}
1462
1463         if len(pkgs) != len(ebuildlist):
1464                 # If we can't access all the metadata then it's totally unsafe to
1465                 # commit since there's no way to generate a correct Manifest.
1466                 # Do not try to do any more QA checks on this package since missing
1467                 # metadata leads to false positives for several checks, and false
1468                 # positives confuse users.
1469                 can_force = False
1470                 continue
1471
1472         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1473         ebuildlist = sorted(pkgs.values())
1474         ebuildlist = [pkg.pf for pkg in ebuildlist]
1475
1476         for y in checkdirlist:
1477                 index = repo_config.find_invalid_path_char(y)
1478                 if index != -1:
1479                         y_relative = os.path.join(checkdir_relative, y)
1480                         if vcs is not None and not vcs_new_changed(y_relative):
1481                                 # If the file isn't in the VCS new or changed set, then
1482                                 # assume that it's an irrelevant temporary file (Manifest
1483                                 # entries are not generated for file names containing
1484                                 # prohibited characters). See bug #406877.
1485                                 index = -1
1486                 if index != -1:
1487                         stats["file.name"] += 1
1488                         fails["file.name"].append("%s/%s: char '%s'" % \
1489                                 (checkdir, y, y[index]))
1490
1491                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1492                         continue
1493                 f = None
1494                 try:
1495                         line = 1
1496                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1497                                 encoding=_encodings['fs'], errors='strict'),
1498                                 mode='r', encoding=_encodings['repo.content'])
1499                         for l in f:
1500                                 line += 1
1501                 except UnicodeDecodeError as ue:
1502                         stats["file.UTF8"] += 1
1503                         s = ue.object[:ue.start]
1504                         l2 = s.count("\n")
1505                         line += l2
1506                         if l2 != 0:
1507                                 s = s[s.rfind("\n") + 1:]
1508                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1509                 finally:
1510                         if f is not None:
1511                                 f.close()
1512
1513         if vcs in ("git", "hg") and check_ebuild_notadded:
1514                 if vcs == "git":
1515                         myf = repoman_popen("git ls-files --others %s" % \
1516                                 (portage._shell_quote(checkdir_relative),))
1517                 if vcs == "hg":
1518                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1519                                 (portage._shell_quote(checkdir_relative),))
1520                 for l in myf:
1521                         if l[:-1][-7:] == ".ebuild":
1522                                 stats["ebuild.notadded"] += 1
1523                                 fails["ebuild.notadded"].append(
1524                                         os.path.join(x, os.path.basename(l[:-1])))
1525                 myf.close()
1526
1527         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1528                 try:
1529                         if vcs == "cvs":
1530                                 myf = open(checkdir + "/CVS/Entries", "r")
1531                         if vcs == "svn":
1532                                 myf = repoman_popen("svn status --depth=files --verbose " +
1533                                         portage._shell_quote(checkdir))
1534                         if vcs == "bzr":
1535                                 myf = repoman_popen("bzr ls -v --kind=file " +
1536                                         portage._shell_quote(checkdir))
1537                         myl = myf.readlines()
1538                         myf.close()
1539                         for l in myl:
1540                                 if vcs == "cvs":
1541                                         if l[0] != "/":
1542                                                 continue
1543                                         splitl = l[1:].split("/")
1544                                         if not len(splitl):
1545                                                 continue
1546                                         if splitl[0][-7:] == ".ebuild":
1547                                                 eadded.append(splitl[0][:-7])
1548                                 if vcs == "svn":
1549                                         if l[:1] == "?":
1550                                                 continue
1551                                         if l[:7] == '      >':
1552                                                 # tree conflict, new in subversion 1.6
1553                                                 continue
1554                                         l = l.split()[-1]
1555                                         if l[-7:] == ".ebuild":
1556                                                 eadded.append(os.path.basename(l[:-7]))
1557                                 if vcs == "bzr":
1558                                         if l[1:2] == "?":
1559                                                 continue
1560                                         l = l.split()[-1]
1561                                         if l[-7:] == ".ebuild":
1562                                                 eadded.append(os.path.basename(l[:-7]))
1563                         if vcs == "svn":
1564                                 myf = repoman_popen("svn status " +
1565                                         portage._shell_quote(checkdir))
1566                                 myl = myf.readlines()
1567                                 myf.close()
1568                                 for l in myl:
1569                                         if l[0] == "A":
1570                                                 l = l.rstrip().split(' ')[-1]
1571                                                 if l[-7:] == ".ebuild":
1572                                                         eadded.append(os.path.basename(l[:-7]))
1573                 except IOError:
1574                         if vcs == "cvs":
1575                                 stats["CVS/Entries.IO_error"] += 1
1576                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1577                         else:
1578                                 raise
1579                         continue
1580
1581         mf = repoman_settings.repositories.get_repo_for_location(
1582                 os.path.dirname(os.path.dirname(checkdir)))
1583         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1584         mydigests = mf.getTypeDigests("DIST")
1585
1586         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1587         myfiles_all = []
1588         src_uri_error = False
1589         for mykey in fetchlist_dict:
1590                 try:
1591                         myfiles_all.extend(fetchlist_dict[mykey])
1592                 except portage.exception.InvalidDependString as e:
1593                         src_uri_error = True
1594                         try:
1595                                 portdb.aux_get(mykey, ["SRC_URI"])
1596                         except KeyError:
1597                                 # This will be reported as an "ebuild.syntax" error.
1598                                 pass
1599                         else:
1600                                 stats["SRC_URI.syntax"] += 1
1601                                 fails["SRC_URI.syntax"].append(
1602                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1603         del fetchlist_dict
1604         if not src_uri_error:
1605                 # This test can produce false positives if SRC_URI could not
1606                 # be parsed for one or more ebuilds. There's no point in
1607                 # producing a false error here since the root cause will
1608                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1609                 # or "ebuild.sytax".
1610                 myfiles_all = set(myfiles_all)
1611                 for entry in mydigests:
1612                         if entry not in myfiles_all:
1613                                 stats["digest.unused"] += 1
1614                                 fails["digest.unused"].append(checkdir + "::" + entry)
1615                 for entry in myfiles_all:
1616                         if entry not in mydigests:
1617                                 stats["digest.missing"] += 1
1618                                 fails["digest.missing"].append(checkdir + "::" + entry)
1619         del myfiles_all
1620
1621         if os.path.exists(checkdir + "/files"):
1622                 filesdirlist = os.listdir(checkdir + "/files")
1623
1624                 # recurse through files directory
1625                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1626                 while filesdirlist:
1627                         y = filesdirlist.pop(0)
1628                         relative_path = os.path.join(x, "files", y)
1629                         full_path = os.path.join(repodir, relative_path)
1630                         try:
1631                                 mystat = os.stat(full_path)
1632                         except OSError as oe:
1633                                 if oe.errno == 2:
1634                                         # don't worry about it.  it likely was removed via fix above.
1635                                         continue
1636                                 else:
1637                                         raise oe
1638                         if S_ISDIR(mystat.st_mode):
1639                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1640                                 if y == "CVS" or y == ".svn":
1641                                         continue
1642                                 for z in os.listdir(checkdir + "/files/" + y):
1643                                         if z == "CVS" or z == ".svn":
1644                                                 continue
1645                                         filesdirlist.append(y + "/" + z)
1646                         # Current policy is no files over 20 KiB, these are the checks. File size between
1647                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1648                         elif mystat.st_size > 61440:
1649                                 stats["file.size.fatal"] += 1
1650                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1651                         elif mystat.st_size > 20480:
1652                                 stats["file.size"] += 1
1653                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1654
1655                         index = repo_config.find_invalid_path_char(y)
1656                         if index != -1:
1657                                 y_relative = os.path.join(checkdir_relative, "files", y)
1658                                 if vcs is not None and not vcs_new_changed(y_relative):
1659                                         # If the file isn't in the VCS new or changed set, then
1660                                         # assume that it's an irrelevant temporary file (Manifest
1661                                         # entries are not generated for file names containing
1662                                         # prohibited characters). See bug #406877.
1663                                         index = -1
1664                         if index != -1:
1665                                 stats["file.name"] += 1
1666                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1667                                         (checkdir, y, y[index]))
1668         del mydigests
1669
1670         if check_changelog and "ChangeLog" not in checkdirlist:
1671                 stats["changelog.missing"] += 1
1672                 fails["changelog.missing"].append(x + "/ChangeLog")
1673
1674         musedict = {}
1675         # metadata.xml file check
1676         if "metadata.xml" not in checkdirlist:
1677                 stats["metadata.missing"] += 1
1678                 fails["metadata.missing"].append(x + "/metadata.xml")
1679         # metadata.xml parse check
1680         else:
1681                 metadata_bad = False
1682                 xml_info = {}
1683                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1684
1685                 # read metadata.xml into memory
1686                 try:
1687                         _metadata_xml = xml.etree.ElementTree.parse(
1688                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1689                                 encoding=_encodings['fs'], errors='strict'),
1690                                 parser=xml_parser)
1691                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1692                         metadata_bad = True
1693                         stats["metadata.bad"] += 1
1694                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1695                         del e
1696                 else:
1697                         if not hasattr(xml_parser, 'parser') or \
1698                                 sys.hexversion < 0x2070000 or \
1699                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1700                                 # doctype is not parsed with python 2.6 or 3.1
1701                                 pass
1702                         else:
1703                                 if "XML_DECLARATION" not in xml_info:
1704                                         stats["metadata.bad"] += 1
1705                                         fails["metadata.bad"].append("%s/metadata.xml: "
1706                                                 "xml declaration is missing on first line, "
1707                                                 "should be '%s'" % (x, metadata_xml_declaration))
1708                                 else:
1709                                         xml_version, xml_encoding, xml_standalone = \
1710                                                 xml_info["XML_DECLARATION"]
1711                                         if xml_encoding is None or \
1712                                                 xml_encoding.upper() != metadata_xml_encoding:
1713                                                 stats["metadata.bad"] += 1
1714                                                 if xml_encoding is None:
1715                                                         encoding_problem = "but it is undefined"
1716                                                 else:
1717                                                         encoding_problem = "not '%s'" % xml_encoding
1718                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1719                                                         "xml declaration encoding should be '%s', %s" %
1720                                                         (x, metadata_xml_encoding, encoding_problem))
1721
1722                                 if "DOCTYPE" not in xml_info:
1723                                         metadata_bad = True
1724                                         stats["metadata.bad"] += 1
1725                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1726                                                 "DOCTYPE is missing"))
1727                                 else:
1728                                         doctype_name, doctype_system, doctype_pubid = \
1729                                                 xml_info["DOCTYPE"]
1730                                         if doctype_system != metadata_dtd_uri:
1731                                                 stats["metadata.bad"] += 1
1732                                                 if doctype_system is None:
1733                                                         system_problem = "but it is undefined"
1734                                                 else:
1735                                                         system_problem = "not '%s'" % doctype_system
1736                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1737                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1738                                                         (x, metadata_dtd_uri, system_problem))
1739
1740                                         if doctype_name != metadata_doctype_name:
1741                                                 stats["metadata.bad"] += 1
1742                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1743                                                         "DOCTYPE: name should be '%s', not '%s'" %
1744                                                         (x, metadata_doctype_name, doctype_name))
1745
1746                         # load USE flags from metadata.xml
1747                         try:
1748                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1749                         except portage.exception.ParseError as e:
1750                                 metadata_bad = True
1751                                 stats["metadata.bad"] += 1
1752                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1753                         else:
1754                                 for atom in chain(*musedict.values()):
1755                                         if atom is None:
1756                                                 continue
1757                                         try:
1758                                                 atom = Atom(atom)
1759                                         except InvalidAtom as e:
1760                                                 stats["metadata.bad"] += 1
1761                                                 fails["metadata.bad"].append(
1762                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1763                                         else:
1764                                                 if atom.cp != x:
1765                                                         stats["metadata.bad"] += 1
1766                                                         fails["metadata.bad"].append(
1767                                                                 ("%s/metadata.xml: Atom contains "
1768                                                                 "unexpected cat/pn: %s") % (x, atom))
1769
1770                         # Run other metadata.xml checkers
1771                         try:
1772                                 utilities.check_metadata(_metadata_xml, herd_base)
1773                         except (utilities.UnknownHerdsError, ) as e:
1774                                 metadata_bad = True
1775                                 stats["metadata.bad"] += 1
1776                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1777                                 del e
1778
1779                 #Only carry out if in package directory or check forced
1780                 if xmllint_capable and not metadata_bad:
1781                         # xmlint can produce garbage output even on success, so only dump
1782                         # the ouput when it fails.
1783                         st, out = repoman_getstatusoutput(
1784                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1785                                  (portage._shell_quote(metadata_dtd),
1786                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1787                         if st != os.EX_OK:
1788                                 print(red("!!!") + " metadata.xml is invalid:")
1789                                 for z in out.splitlines():
1790                                         print(red("!!! ") + z)
1791                                 stats["metadata.bad"] += 1
1792                                 fails["metadata.bad"].append(x + "/metadata.xml")
1793
1794                 del metadata_bad
1795         muselist = frozenset(musedict)
1796
1797         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1798         changelog_modified = changelog_path in modified_changelogs
1799
1800         # detect unused local USE-descriptions
1801         used_useflags = set()
1802
1803         for y in ebuildlist:
1804                 relative_path = os.path.join(x, y + ".ebuild")
1805                 full_path = os.path.join(repodir, relative_path)
1806                 ebuild_path = y + ".ebuild"
1807                 if repolevel < 3:
1808                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1809                 if repolevel < 2:
1810                         ebuild_path = os.path.join(catdir, ebuild_path)
1811                 ebuild_path = os.path.join(".", ebuild_path)
1812                 if check_changelog and not changelog_modified \
1813                         and ebuild_path in new_ebuilds:
1814                         stats['changelog.ebuildadded'] += 1
1815                         fails['changelog.ebuildadded'].append(relative_path)
1816
1817                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1818                         #ebuild not added to vcs
1819                         stats["ebuild.notadded"] += 1
1820                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1821                 myesplit = portage.pkgsplit(y)
1822                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1823                             or pv_toolong_re.search(myesplit[1]) \
1824                             or pv_toolong_re.search(myesplit[2]):
1825                         stats["ebuild.invalidname"] += 1
1826                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1827                         continue
1828                 elif myesplit[0] != pkgdir:
1829                         print(pkgdir, myesplit[0])
1830                         stats["ebuild.namenomatch"] += 1
1831                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1832                         continue
1833
1834                 pkg = pkgs[y]
1835
1836                 if pkg.invalid:
1837                         allvalid = False
1838                         for k, msgs in pkg.invalid.items():
1839                                 for msg in msgs:
1840                                         stats[k] += 1
1841                                         fails[k].append("%s: %s" % (relative_path, msg))
1842                         continue
1843
1844                 myaux = pkg._metadata
1845                 eapi = myaux["EAPI"]
1846                 inherited = pkg.inherited
1847                 live_ebuild = live_eclasses.intersection(inherited)
1848
1849                 if repo_config.eapi_is_banned(eapi):
1850                         stats["repo.eapi.banned"] += 1
1851                         fails["repo.eapi.banned"].append(
1852                                 "%s: %s" % (relative_path, eapi))
1853
1854                 elif repo_config.eapi_is_deprecated(eapi):
1855                         stats["repo.eapi.deprecated"] += 1
1856                         fails["repo.eapi.deprecated"].append(
1857                                 "%s: %s" % (relative_path, eapi))
1858
1859                 for k, v in myaux.items():
1860                         if not isinstance(v, basestring):
1861                                 continue
1862                         m = non_ascii_re.search(v)
1863                         if m is not None:
1864                                 stats["variable.invalidchar"] += 1
1865                                 fails["variable.invalidchar"].append(
1866                                         ("%s: %s variable contains non-ASCII " + \
1867                                         "character at position %s") % \
1868                                         (relative_path, k, m.start() + 1))
1869
1870                 if not src_uri_error:
1871                         # Check that URIs don't reference a server from thirdpartymirrors.
1872                         for uri in portage.dep.use_reduce( \
1873                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1874                                 contains_mirror = False
1875                                 for mirror, mirror_alias in thirdpartymirrors.items():
1876                                         if uri.startswith(mirror):
1877                                                 contains_mirror = True
1878                                                 break
1879                                 if not contains_mirror:
1880                                         continue
1881
1882                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1883                                 stats["SRC_URI.mirror"] += 1
1884                                 fails["SRC_URI.mirror"].append(
1885                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1886                                         (relative_path, mirror, new_uri))
1887
1888                 if myaux.get("PROVIDE"):
1889                         stats["virtual.oldstyle"] += 1
1890                         fails["virtual.oldstyle"].append(relative_path)
1891
1892                 for pos, missing_var in enumerate(missingvars):
1893                         if not myaux.get(missing_var):
1894                                 if catdir == "virtual" and \
1895                                         missing_var in ("HOMEPAGE", "LICENSE"):
1896                                         continue
1897                                 if live_ebuild and missing_var == "KEYWORDS":
1898                                         continue
1899                                 myqakey = missingvars[pos] + ".missing"
1900                                 stats[myqakey] += 1
1901                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1902
1903                 if catdir == "virtual":
1904                         for var in ("HOMEPAGE", "LICENSE"):
1905                                 if myaux.get(var):
1906                                         myqakey = var + ".virtual"
1907                                         stats[myqakey] += 1
1908                                         fails[myqakey].append(relative_path)
1909
1910                 # 14 is the length of DESCRIPTION=""
1911                 if len(myaux['DESCRIPTION']) > max_desc_len:
1912                         stats['DESCRIPTION.toolong'] += 1
1913                         fails['DESCRIPTION.toolong'].append(
1914                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1915                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1916
1917                 keywords = myaux["KEYWORDS"].split()
1918                 stable_keywords = []
1919                 for keyword in keywords:
1920                         if not keyword.startswith("~") and \
1921                                 not keyword.startswith("-"):
1922                                 stable_keywords.append(keyword)
1923                 if stable_keywords:
1924                         if ebuild_path in new_ebuilds and catdir != "virtual":
1925                                 stable_keywords.sort()
1926                                 stats["KEYWORDS.stable"] += 1
1927                                 fails["KEYWORDS.stable"].append(
1928                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1929                                                 " ".join(stable_keywords))
1930
1931                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1932                         if not kw.startswith("-"))
1933
1934                 previous_keywords = slot_keywords.get(pkg.slot)
1935                 if previous_keywords is None:
1936                         slot_keywords[pkg.slot] = set()
1937                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1938                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1939                         if dropped_keywords:
1940                                 stats["KEYWORDS.dropped"] += 1
1941                                 fails["KEYWORDS.dropped"].append(
1942                                         relative_path + ": %s" % \
1943                                         " ".join(sorted(dropped_keywords)))
1944
1945                 slot_keywords[pkg.slot].update(ebuild_archs)
1946
1947                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1948                 if "-*" in keywords:
1949                         haskeyword = False
1950                         for kw in keywords:
1951                                 if kw[0] == "~":
1952                                         kw = kw[1:]
1953                                 if kw in kwlist:
1954                                         haskeyword = True
1955                         if not haskeyword:
1956                                 stats["KEYWORDS.stupid"] += 1
1957                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1958
1959                 """
1960                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1961                 not be allowed to be marked stable
1962                 """
1963                 if live_ebuild and repo_config.name == "gentoo":
1964                         bad_stable_keywords = []
1965                         for keyword in keywords:
1966                                 if not keyword.startswith("~") and \
1967                                         not keyword.startswith("-"):
1968                                         bad_stable_keywords.append(keyword)
1969                                 del keyword
1970                         if bad_stable_keywords:
1971                                 stats["LIVEVCS.stable"] += 1
1972                                 fails["LIVEVCS.stable"].append(
1973                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1974                                                 bad_stable_keywords)
1975                         del bad_stable_keywords
1976
1977                         if keywords and not has_global_mask(pkg):
1978                                 stats["LIVEVCS.unmasked"] += 1
1979                                 fails["LIVEVCS.unmasked"].append(relative_path)
1980
1981                 if options.ignore_arches:
1982                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1983                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1984                 else:
1985                         arches = set()
1986                         for keyword in keywords:
1987                                 if keyword[0] == "-":
1988                                         continue
1989                                 elif keyword[0] == "~":
1990                                         arch = keyword[1:]
1991                                         if arch == "*":
1992                                                 for expanded_arch in profiles:
1993                                                         if expanded_arch == "**":
1994                                                                 continue
1995                                                         arches.add((keyword, expanded_arch,
1996                                                                 (expanded_arch, "~" + expanded_arch)))
1997                                         else:
1998                                                 arches.add((keyword, arch, (arch, keyword)))
1999                                 else:
2000                                         if keyword == "*":
2001                                                 for expanded_arch in profiles:
2002                                                         if expanded_arch == "**":
2003                                                                 continue
2004                                                         arches.add((keyword, expanded_arch,
2005                                                                 (expanded_arch,)))
2006                                         else:
2007                                                 arches.add((keyword, keyword, (keyword,)))
2008                         if not arches:
2009                                 # Use an empty profile for checking dependencies of
2010                                 # packages that have empty KEYWORDS.
2011                                 arches.add(('**', '**', ('**',)))
2012
2013                 unknown_pkgs = set()
2014                 baddepsyntax = False
2015                 badlicsyntax = False
2016                 badprovsyntax = False
2017                 catpkg = catdir + "/" + y
2018
2019                 inherited_java_eclass = "java-pkg-2" in inherited or \
2020                         "java-pkg-opt-2" in inherited
2021                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
2022                 operator_tokens = set(["||", "(", ")"])
2023                 type_list, badsyntax = [], []
2024                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
2025                         mydepstr = myaux[mytype]
2026
2027                         buildtime = mytype in Package._buildtime_keys
2028                         runtime = mytype in Package._runtime_keys
2029                         token_class = None
2030                         if mytype.endswith("DEPEND"):
2031                                 token_class = portage.dep.Atom
2032
2033                         try:
2034                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
2035                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
2036                         except portage.exception.InvalidDependString as e:
2037                                 atoms = None
2038                                 badsyntax.append(str(e))
2039
2040                         if atoms and mytype.endswith("DEPEND"):
2041                                 if runtime and \
2042                                         "test?" in mydepstr.split():
2043                                         stats[mytype + '.suspect'] += 1
2044                                         fails[mytype + '.suspect'].append(relative_path + \
2045                                                 ": 'test?' USE conditional in %s" % mytype)
2046
2047                                 for atom in atoms:
2048                                         if atom == "||":
2049                                                 continue
2050
2051                                         # Skip dependency.unknown for blockers, so that we
2052                                         # don't encourage people to remove necessary blockers,
2053                                         # as discussed in bug #382407.
2054                                         if atom.blocker is None and \
2055                                                 not portdb.xmatch("match-all", atom) and \
2056                                                 not atom.cp.startswith("virtual/"):
2057                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2058
2059                                         is_blocker = atom.blocker
2060
2061                                         if catdir != "virtual":
2062                                                 if not is_blocker and \
2063                                                         atom.cp in suspect_virtual:
2064                                                         stats['virtual.suspect'] += 1
2065                                                         fails['virtual.suspect'].append(
2066                                                                 relative_path +
2067                                                                 ": %s: consider using '%s' instead of '%s'" %
2068                                                                 (mytype, suspect_virtual[atom.cp], atom))
2069
2070                                         if buildtime and \
2071                                                 not is_blocker and \
2072                                                 not inherited_java_eclass and \
2073                                                 atom.cp == "virtual/jdk":
2074                                                 stats['java.eclassesnotused'] += 1
2075                                                 fails['java.eclassesnotused'].append(relative_path)
2076                                         elif buildtime and \
2077                                                 not is_blocker and \
2078                                                 not inherited_wxwidgets_eclass and \
2079                                                 atom.cp == "x11-libs/wxGTK":
2080                                                 stats['wxwidgets.eclassnotused'] += 1
2081                                                 fails['wxwidgets.eclassnotused'].append(
2082                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2083                                                         " without inheriting wxwidgets.eclass") % mytype)
2084                                         elif runtime:
2085                                                 if not is_blocker and \
2086                                                         atom.cp in suspect_rdepend:
2087                                                         stats[mytype + '.suspect'] += 1
2088                                                         fails[mytype + '.suspect'].append(
2089                                                                 relative_path + ": '%s'" % atom)
2090
2091                                         if atom.operator == "~" and \
2092                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2093                                                 qacat = 'dependency.badtilde'
2094                                                 stats[qacat] += 1
2095                                                 fails[qacat].append(
2096                                                         (relative_path + ": %s uses the ~ operator"
2097                                                          " with a non-zero revision:" + \
2098                                                          " '%s'") % (mytype, atom))
2099
2100                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2101
2102                 for m, b in zip(type_list, badsyntax):
2103                         if m.endswith("DEPEND"):
2104                                 qacat = "dependency.syntax"
2105                         else:
2106                                 qacat = m + ".syntax"
2107                         stats[qacat] += 1
2108                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2109
2110                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2111                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2112                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2113                 badlicsyntax = badlicsyntax > 0
2114                 badprovsyntax = badprovsyntax > 0
2115
2116                 # uselist checks - global
2117                 myuse = []
2118                 default_use = []
2119                 for myflag in myaux["IUSE"].split():
2120                         flag_name = myflag.lstrip("+-")
2121                         used_useflags.add(flag_name)
2122                         if myflag != flag_name:
2123                                 default_use.append(myflag)
2124                         if flag_name not in uselist:
2125                                 myuse.append(flag_name)
2126
2127                 # uselist checks - metadata
2128                 for mypos in range(len(myuse)-1, -1, -1):
2129                         if myuse[mypos] and (myuse[mypos] in muselist):
2130                                 del myuse[mypos]
2131
2132                 if default_use and not eapi_has_iuse_defaults(eapi):
2133                         for myflag in default_use:
2134                                 stats['EAPI.incompatible'] += 1
2135                                 fails['EAPI.incompatible'].append(
2136                                         (relative_path + ": IUSE defaults" + \
2137                                         " not supported with EAPI='%s':" + \
2138                                         " '%s'") % (eapi, myflag))
2139
2140                 for mypos in range(len(myuse)):
2141                         stats["IUSE.invalid"] += 1
2142                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2143
2144                 # license checks
2145                 if not badlicsyntax:
2146                         # Parse the LICENSE variable, remove USE conditions and
2147                         # flatten it.
2148                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2149                         # Check each entry to ensure that it exists in PORTDIR's
2150                         # license directory.
2151                         for lic in licenses:
2152                                 # Need to check for "||" manually as no portage
2153                                 # function will remove it without removing values.
2154                                 if lic not in liclist and lic != "||":
2155                                         stats["LICENSE.invalid"] += 1
2156                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2157                                 elif lic in liclist_deprecated:
2158                                         stats["LICENSE.deprecated"] += 1
2159                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2160
2161                 #keyword checks
2162                 myuse = myaux["KEYWORDS"].split()
2163                 for mykey in myuse:
2164                         if mykey not in ("-*", "*", "~*"):
2165                                 myskey = mykey
2166                                 if myskey[:1] == "-":
2167                                         myskey = myskey[1:]
2168                                 if myskey[:1] == "~":
2169                                         myskey = myskey[1:]
2170                                 if myskey not in kwlist:
2171                                         stats["KEYWORDS.invalid"] += 1
2172                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2173                                 elif myskey not in profiles:
2174                                         stats["KEYWORDS.invalid"] += 1
2175                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2176
2177                 #restrict checks
2178                 myrestrict = None
2179                 try:
2180                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2181                 except portage.exception.InvalidDependString as e:
2182                         stats["RESTRICT.syntax"] += 1
2183                         fails["RESTRICT.syntax"].append(
2184                                 "%s: RESTRICT: %s" % (relative_path, e))
2185                         del e
2186                 if myrestrict:
2187                         myrestrict = set(myrestrict)
2188                         mybadrestrict = myrestrict.difference(valid_restrict)
2189                         if mybadrestrict:
2190                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2191                                 for mybad in mybadrestrict:
2192                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2193                 #REQUIRED_USE check
2194                 required_use = myaux["REQUIRED_USE"]
2195                 if required_use:
2196                         if not eapi_has_required_use(eapi):
2197                                 stats['EAPI.incompatible'] += 1
2198                                 fails['EAPI.incompatible'].append(
2199                                         relative_path + ": REQUIRED_USE" + \
2200                                         " not supported with EAPI='%s'" % (eapi,))
2201                         try:
2202                                 portage.dep.check_required_use(required_use, (),
2203                                         pkg.iuse.is_valid_flag, eapi=eapi)
2204                         except portage.exception.InvalidDependString as e:
2205                                 stats["REQUIRED_USE.syntax"] += 1
2206                                 fails["REQUIRED_USE.syntax"].append(
2207                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2208                                 del e
2209
2210                 # Syntax Checks
2211                 relative_path = os.path.join(x, y + ".ebuild")
2212                 full_path = os.path.join(repodir, relative_path)
2213                 if not vcs_preserves_mtime:
2214                         if ebuild_path not in new_ebuilds and \
2215                                 ebuild_path not in modified_ebuilds:
2216                                 pkg.mtime = None
2217                 try:
2218                         # All ebuilds should have utf_8 encoding.
2219                         f = io.open(_unicode_encode(full_path,
2220                                 encoding=_encodings['fs'], errors='strict'),
2221                                 mode='r', encoding=_encodings['repo.content'])
2222                         try:
2223                                 for check_name, e in run_checks(f, pkg):
2224                                         stats[check_name] += 1
2225                                         fails[check_name].append(relative_path + ': %s' % e)
2226                         finally:
2227                                 f.close()
2228                 except UnicodeDecodeError:
2229                         # A file.UTF8 failure will have already been recorded above.
2230                         pass
2231
2232                 if options.force:
2233                         # The dep_check() calls are the most expensive QA test. If --force
2234                         # is enabled, there's no point in wasting time on these since the
2235                         # user is intent on forcing the commit anyway.
2236                         continue
2237
2238                 relevant_profiles = []
2239                 for keyword, arch, groups in arches:
2240                         if arch not in profiles:
2241                                 # A missing profile will create an error further down
2242                                 # during the KEYWORDS verification.
2243                                 continue
2244
2245                         if include_arches is not None:
2246                                 if arch not in include_arches:
2247                                         continue
2248
2249                         relevant_profiles.extend((keyword, groups, prof)
2250                                 for prof in profiles[arch])
2251
2252                 def sort_key(item):
2253                         return item[2].sub_path
2254
2255                 relevant_profiles.sort(key=sort_key)
2256
2257                 for keyword, groups, prof in relevant_profiles:
2258
2259                                 if prof.status not in ("stable", "dev") or \
2260                                         prof.status == "dev" and not options.include_dev:
2261                                         continue
2262
2263                                 dep_settings = arch_caches.get(prof.sub_path)
2264                                 if dep_settings is None:
2265                                         dep_settings = portage.config(
2266                                                 config_profile_path=prof.abs_path,
2267                                                 config_incrementals=repoman_incrementals,
2268                                                 config_root=config_root,
2269                                                 local_config=False,
2270                                                 _unmatched_removal=options.unmatched_removal,
2271                                                 env=env, repositories=repoman_settings.repositories)
2272                                         dep_settings.categories = repoman_settings.categories
2273                                         if options.without_mask:
2274                                                 dep_settings._mask_manager_obj = \
2275                                                         copy.deepcopy(dep_settings._mask_manager)
2276                                                 dep_settings._mask_manager._pmaskdict.clear()
2277                                         arch_caches[prof.sub_path] = dep_settings
2278
2279                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2280                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2281                                 if xcache is None:
2282                                         portdb.melt()
2283                                         portdb.freeze()
2284                                         xcache = portdb.xcache
2285                                         xcache.update(shared_xmatch_caches)
2286                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2287
2288                                 trees[root]["porttree"].settings = dep_settings
2289                                 portdb.settings = dep_settings
2290                                 portdb.xcache = xcache
2291
2292                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2293                                 # just in case, prevent config.reset() from nuking these.
2294                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2295
2296                                 # This attribute is used in dbapi._match_use() to apply
2297                                 # use.stable.{mask,force} settings based on the stable
2298                                 # status of the parent package. This is required in order
2299                                 # for USE deps of unstable packages to be resolved correctly,
2300                                 # since otherwise use.stable.{mask,force} settings of
2301                                 # dependencies may conflict (see bug #456342).
2302                                 dep_settings._parent_stable = dep_settings._isStable(pkg)
2303
2304                                 # Handle package.use*.{force,mask) calculation, for use
2305                                 # in dep_check.
2306                                 dep_settings.useforce = dep_settings._use_manager.getUseForce(
2307                                         pkg, stable=dep_settings._parent_stable)
2308                                 dep_settings.usemask = dep_settings._use_manager.getUseMask(
2309                                         pkg, stable=dep_settings._parent_stable)
2310
2311                                 if not baddepsyntax:
2312                                         ismasked = not ebuild_archs or \
2313                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2314                                         if ismasked:
2315                                                 if not have_pmasked:
2316                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2317                                                                 pkg.cpv, pkg._metadata))
2318                                                 if options.ignore_masked:
2319                                                         continue
2320                                                 #we are testing deps for a masked package; give it some lee-way
2321                                                 suffix = "masked"
2322                                                 matchmode = "minimum-all"
2323                                         else:
2324                                                 suffix = ""
2325                                                 matchmode = "minimum-visible"
2326
2327                                         if not have_dev_keywords:
2328                                                 have_dev_keywords = \
2329                                                         bool(dev_keywords.intersection(keywords))
2330
2331                                         if prof.status == "dev":
2332                                                 suffix = suffix + "indev"
2333
2334                                         for mytype in Package._dep_keys:
2335
2336                                                 mykey = "dependency.bad" + suffix
2337                                                 myvalue = myaux[mytype]
2338                                                 if not myvalue:
2339                                                         continue
2340
2341                                                 success, atoms = portage.dep_check(myvalue, portdb,
2342                                                         dep_settings, use="all", mode=matchmode,
2343                                                         trees=trees)
2344
2345                                                 if success:
2346                                                         if atoms:
2347
2348                                                                 # Don't bother with dependency.unknown for
2349                                                                 # cases in which *DEPEND.bad is triggered.
2350                                                                 for atom in atoms:
2351                                                                         # dep_check returns all blockers and they
2352                                                                         # aren't counted for *DEPEND.bad, so we
2353                                                                         # ignore them here.
2354                                                                         if not atom.blocker:
2355                                                                                 unknown_pkgs.discard(
2356                                                                                         (mytype, atom.unevaluated_atom))
2357
2358                                                                 if not prof.sub_path:
2359                                                                         # old-style virtuals currently aren't
2360                                                                         # resolvable with empty profile, since
2361                                                                         # 'virtuals' mappings are unavailable
2362                                                                         # (it would be expensive to search
2363                                                                         # for PROVIDE in all ebuilds)
2364                                                                         atoms = [atom for atom in atoms if not \
2365                                                                                 (atom.cp.startswith('virtual/') and \
2366                                                                                 not portdb.cp_list(atom.cp))]
2367
2368                                                                 #we have some unsolvable deps
2369                                                                 #remove ! deps, which always show up as unsatisfiable
2370                                                                 atoms = [str(atom.unevaluated_atom) \
2371                                                                         for atom in atoms if not atom.blocker]
2372
2373                                                                 #if we emptied out our list, continue:
2374                                                                 if not atoms:
2375                                                                         continue
2376                                                                 stats[mykey] += 1
2377                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2378                                                                         (relative_path, mytype, keyword,
2379                                                                         prof, repr(atoms)))
2380                                                 else:
2381                                                         stats[mykey] += 1
2382                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2383                                                                 (relative_path, mytype, keyword,
2384                                                                 prof, repr(atoms)))
2385
2386                 if not baddepsyntax and unknown_pkgs:
2387                         type_map = {}
2388                         for mytype, atom in unknown_pkgs:
2389                                 type_map.setdefault(mytype, set()).add(atom)
2390                         for mytype, atoms in type_map.items():
2391                                 stats["dependency.unknown"] += 1
2392                                 fails["dependency.unknown"].append("%s: %s: %s" %
2393                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2394
2395         # check if there are unused local USE-descriptions in metadata.xml
2396         # (unless there are any invalids, to avoid noise)
2397         if allvalid:
2398                 for myflag in muselist.difference(used_useflags):
2399                         stats["metadata.warning"] += 1
2400                         fails["metadata.warning"].append(
2401                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2402                                 (x, myflag))
2403
2404 if options.if_modified == "y" and len(effective_scanlist) < 1:
2405         logging.warn("--if-modified is enabled, but no modified packages were found!")
2406
2407 if options.mode == "manifest":
2408         sys.exit(dofail)
2409
2410 # dofail will be set to 1 if we have failed in at least one non-warning category
2411 dofail = 0
2412 # dowarn will be set to 1 if we tripped any warnings
2413 dowarn = 0
2414 # dofull will be set if we should print a "repoman full" informational message
2415 dofull = options.mode != 'full'
2416
2417 for x in qacats:
2418         if not stats[x]:
2419                 continue
2420         dowarn = 1
2421         if x not in qawarnings:
2422                 dofail = 1
2423
2424 if dofail or \
2425         (dowarn and not (options.quiet or options.mode == "scan")):
2426         dofull = 0
2427
2428 # Save QA output so that it can be conveniently displayed
2429 # in $EDITOR while the user creates a commit message.
2430 # Otherwise, the user would not be able to see this output
2431 # once the editor has taken over the screen.
2432 qa_output = io.StringIO()
2433 style_file = ConsoleStyleFile(sys.stdout)
2434 if options.mode == 'commit' and \
2435         (not commitmessage or not commitmessage.strip()):
2436         style_file.write_listener = qa_output
2437 console_writer = StyleWriter(file=style_file, maxcol=9999)
2438 console_writer.style_listener = style_file.new_styles
2439
2440 f = formatter.AbstractFormatter(console_writer)
2441
2442 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2443
2444 style_file.flush()
2445 del console_writer, f, style_file
2446 qa_output = qa_output.getvalue()
2447 qa_output = qa_output.splitlines(True)
2448
2449 suggest_ignore_masked = False
2450 suggest_include_dev = False
2451
2452 if have_pmasked and not (options.without_mask or options.ignore_masked):
2453         suggest_ignore_masked = True
2454 if have_dev_keywords and not options.include_dev:
2455         suggest_include_dev = True
2456
2457 if suggest_ignore_masked or suggest_include_dev:
2458         print()
2459         if suggest_ignore_masked:
2460                 print(bold("Note: use --without-mask to check " + \
2461                         "KEYWORDS on dependencies of masked packages"))
2462
2463         if suggest_include_dev:
2464                 print(bold("Note: use --include-dev (-d) to check " + \
2465                         "dependencies for 'dev' profiles"))
2466         print()
2467
2468 if options.mode != 'commit':
2469         if dofull:
2470                 print(bold("Note: type \"repoman full\" for a complete listing."))
2471         if dowarn and not dofail:
2472                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2473         elif not dofail:
2474                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2475         elif dofail:
2476                 print(bad("Please fix these important QA issues first."))
2477                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2478                 sys.exit(1)
2479 else:
2480         if dofail and can_force and options.force and not options.pretend:
2481                 print(green("RepoMan sez:") + \
2482                         " \"You want to commit even with these QA issues?\n" + \
2483                         "              I'll take it this time, but I'm not happy.\"\n")
2484         elif dofail:
2485                 if options.force and not can_force:
2486                         print(bad("The --force option has been disabled due to extraordinary issues."))
2487                 print(bad("Please fix these important QA issues first."))
2488                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2489                 sys.exit(1)
2490
2491         if options.pretend:
2492                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2493
2494         myunadded = []
2495         if vcs == "cvs":
2496                 try:
2497                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2498                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2499                 except SystemExit as e:
2500                         raise  # TODO propagate this
2501                 except:
2502                         err("Error retrieving CVS tree; exiting.")
2503         if vcs == "svn":
2504                 try:
2505                         with repoman_popen("svn status --no-ignore") as f:
2506                                 svnstatus = f.readlines()
2507                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2508                 except SystemExit as e:
2509                         raise  # TODO propagate this
2510                 except:
2511                         err("Error retrieving SVN info; exiting.")
2512         if vcs == "git":
2513                 # get list of files not under version control or missing
2514                 myf = repoman_popen("git ls-files --others")
2515                 myunadded = ["./" + elem[:-1] for elem in myf]
2516                 myf.close()
2517         if vcs == "bzr":
2518                 try:
2519                         with repoman_popen("bzr status -S .") as f:
2520                                 bzrstatus = f.readlines()
2521                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2522                 except SystemExit as e:
2523                         raise  # TODO propagate this
2524                 except:
2525                         err("Error retrieving bzr info; exiting.")
2526         if vcs == "hg":
2527                 with repoman_popen("hg status --no-status --unknown .") as f:
2528                         myunadded = f.readlines()
2529                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2530
2531                 # Mercurial doesn't handle manually deleted files as removed from
2532                 # the repository, so the user need to remove them before commit,
2533                 # using "hg remove [FILES]"
2534                 with repoman_popen("hg status --no-status --deleted .") as f:
2535                         mydeleted = f.readlines()
2536                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2537
2538
2539         myautoadd = []
2540         if myunadded:
2541                 for x in range(len(myunadded)-1, -1, -1):
2542                         xs = myunadded[x].split("/")
2543                         if xs[-1] == "files":
2544                                 print("!!! files dir is not added! Please correct this.")
2545                                 sys.exit(-1)
2546                         elif xs[-1] == "Manifest":
2547                                 # It's a manifest... auto add
2548                                 myautoadd += [myunadded[x]]
2549                                 del myunadded[x]
2550
2551         if myunadded:
2552                 print(red("!!! The following files are in your local tree but are not added to the master"))
2553                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2554                 for x in myunadded:
2555                         print("   ", x)
2556                 print()
2557                 print()
2558                 sys.exit(1)
2559
2560         if vcs == "hg" and mydeleted:
2561                 print(red("!!! The following files are removed manually from your local tree but are not"))
2562                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2563                 for x in mydeleted:
2564                         print("   ", x)
2565                 print()
2566                 print()
2567                 sys.exit(1)
2568
2569         if vcs == "cvs":
2570                 mycvstree = cvstree.getentries("./", recursive=1)
2571                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2572                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2573                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2574                 bin_blob_pattern = re.compile("^-kb$")
2575                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2576                         recursive=1, basedir="./"))
2577
2578         if vcs == "svn":
2579                 with repoman_popen("svn status") as f:
2580                         svnstatus = f.readlines()
2581                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2582                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2583                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2584
2585                 # Subversion expands keywords specified in svn:keywords properties.
2586                 with repoman_popen("svn propget -R svn:keywords") as f:
2587                         props = f.readlines()
2588                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2589                         for prop in props if " - " in prop)
2590
2591         elif vcs == "git":
2592                 with repoman_popen("git diff-index --name-only "
2593                         "--relative --diff-filter=M HEAD") as f:
2594                         mychanged = f.readlines()
2595                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2596
2597                 with repoman_popen("git diff-index --name-only "
2598                         "--relative --diff-filter=A HEAD") as f:
2599                         mynew = f.readlines()
2600                 mynew = ["./" + elem[:-1] for elem in mynew]
2601
2602                 with repoman_popen("git diff-index --name-only "
2603                         "--relative --diff-filter=D HEAD") as f:
2604                         myremoved = f.readlines()
2605                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2606
2607         if vcs == "bzr":
2608                 with repoman_popen("bzr status -S .") as f:
2609                         bzrstatus = f.readlines()
2610                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2611                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2612                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2613                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2614                 # Bazaar expands nothing.
2615
2616         if vcs == "hg":
2617                 with repoman_popen("hg status --no-status --modified .") as f:
2618                         mychanged = f.readlines()
2619                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2620
2621                 with repoman_popen("hg status --no-status --added .") as f:
2622                         mynew = f.readlines()
2623                 mynew = ["./" + elem.rstrip() for elem in mynew]
2624
2625                 with repoman_popen("hg status --no-status --removed .") as f:
2626                         myremoved = f.readlines()
2627                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2628
2629         if vcs:
2630                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2631                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2632                         print()
2633                         print("(Didn't find any changed files...)")
2634                         print()
2635                         sys.exit(1)
2636
2637         # Manifests need to be regenerated after all other commits, so don't commit
2638         # them now even if they have changed.
2639         mymanifests = set()
2640         myupdates = set()
2641         for f in mychanged + mynew:
2642                 if "Manifest" == os.path.basename(f):
2643                         mymanifests.add(f)
2644                 else:
2645                         myupdates.add(f)
2646         myupdates.difference_update(myremoved)
2647         myupdates = list(myupdates)
2648         mymanifests = list(mymanifests)
2649         myheaders = []
2650         mydirty = []
2651
2652         commitmessage = options.commitmsg
2653         if options.commitmsgfile:
2654                 try:
2655                         f = io.open(_unicode_encode(options.commitmsgfile,
2656                         encoding=_encodings['fs'], errors='strict'),
2657                         mode='r', encoding=_encodings['content'], errors='replace')
2658                         commitmessage = f.read()
2659                         f.close()
2660                         del f
2661                 except (IOError, OSError) as e:
2662                         if e.errno == errno.ENOENT:
2663                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2664                         else:
2665                                 raise
2666                 # We've read the content so the file is no longer needed.
2667                 commitmessagefile = None
2668         if not commitmessage or not commitmessage.strip():
2669                 try:
2670                         editor = os.environ.get("EDITOR")
2671                         if editor and utilities.editor_is_executable(editor):
2672                                 commitmessage = utilities.get_commit_message_with_editor(
2673                                         editor, message=qa_output)
2674                         else:
2675                                 commitmessage = utilities.get_commit_message_with_stdin()
2676                 except KeyboardInterrupt:
2677                         exithandler()
2678                 if not commitmessage or not commitmessage.strip():
2679                         print("* no commit message?  aborting commit.")
2680                         sys.exit(1)
2681         commitmessage = commitmessage.rstrip()
2682         changelog_msg = commitmessage
2683         portage_version = getattr(portage, "VERSION", None)
2684         gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
2685         dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
2686         if portage_version is None:
2687                 sys.stderr.write("Failed to insert portage version in message!\n")
2688                 sys.stderr.flush()
2689                 portage_version = "Unknown"
2690
2691         report_options = []
2692         if options.force:
2693                 report_options.append("--force")
2694         if options.ignore_arches:
2695                 report_options.append("--ignore-arches")
2696         if include_arches is not None:
2697                 report_options.append("--include-arches=\"%s\"" %
2698                         " ".join(sorted(include_arches)))
2699
2700         if vcs == "git":
2701                 # Use new footer only for git (see bug #438364).
2702                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2703                 if report_options:
2704                         commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
2705                 if sign_manifests:
2706                         commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
2707                 if dco_sob:
2708                         commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
2709         else:
2710                 unameout = platform.system() + " "
2711                 if platform.system() in ["Darwin", "SunOS"]:
2712                         unameout += platform.processor()
2713                 else:
2714                         unameout += platform.machine()
2715                 commit_footer = "\n\n"
2716                 if dco_sob:
2717                         commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
2718                 commit_footer += "(Portage version: %s/%s/%s" % \
2719                         (portage_version, vcs, unameout)
2720                 if report_options:
2721                         commit_footer += ", RepoMan options: " + " ".join(report_options)
2722                 if sign_manifests:
2723                         commit_footer += ", signed Manifest commit with key %s" % \
2724                                 (gpg_key, )
2725                 else:
2726                         commit_footer += ", unsigned Manifest commit"
2727                 commit_footer += ")"
2728
2729         commitmessage += commit_footer
2730
2731         if options.echangelog in ('y', 'force'):
2732                 logging.info("checking for unmodified ChangeLog files")
2733                 committer_name = utilities.get_committer_name(env=repoman_settings)
2734                 for x in sorted(vcs_files_to_cps(
2735                         chain(myupdates, mymanifests, myremoved))):
2736                         catdir, pkgdir = x.split("/")
2737                         checkdir = repodir + "/" + x
2738                         checkdir_relative = ""
2739                         if repolevel < 3:
2740                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2741                         if repolevel < 2:
2742                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2743                         checkdir_relative = os.path.join(".", checkdir_relative)
2744
2745                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2746                         changelog_modified = changelog_path in modified_changelogs
2747                         if changelog_modified and options.echangelog != 'force':
2748                                 continue
2749
2750                         # get changes for this package
2751                         cdrlen = len(checkdir_relative)
2752                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2753                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2754                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2755
2756                         # Skip ChangeLog generation if only the Manifest was modified,
2757                         # as discussed in bug #398009.
2758                         nontrivial_cl_files = set()
2759                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2760                         nontrivial_cl_files.difference_update(['Manifest'])
2761                         if not nontrivial_cl_files and options.echangelog != 'force':
2762                                 continue
2763
2764                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2765                                 committer_name, changelog_msg,
2766                                 os.path.join(repodir, 'skel.ChangeLog'),
2767                                 catdir, pkgdir,
2768                                 new=clnew, removed=clremoved, changed=clchanged,
2769                                 pretend=options.pretend)
2770                         if new_changelog is None:
2771                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2772                                         level=logging.ERROR, noiselevel=-1)
2773                                 sys.exit(1)
2774
2775                         # if the ChangeLog was just created, add it to vcs
2776                         if new_changelog:
2777                                 myautoadd.append(changelog_path)
2778                                 # myautoadd is appended to myupdates below
2779                         else:
2780                                 myupdates.append(changelog_path)
2781
2782                         if options.ask and not options.pretend:
2783                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2784                                 repoman_settings["O"] = checkdir
2785                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2786
2787         if myautoadd:
2788                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2789                 add_cmd = [vcs, "add"]
2790                 add_cmd += myautoadd
2791                 if options.pretend:
2792                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2793                                 noiselevel=-1)
2794                 else:
2795
2796                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2797                                 not os.path.isabs(add_cmd[0]):
2798                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2799                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2800                                 fullname = find_binary(add_cmd[0])
2801                                 if fullname is None:
2802                                         raise portage.exception.CommandNotFound(add_cmd[0])
2803                                 add_cmd[0] = fullname
2804
2805                         add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2806                         retcode = subprocess.call(add_cmd)
2807                         if retcode != os.EX_OK:
2808                                 logging.error(
2809                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2810                                 sys.exit(retcode)
2811
2812                 myupdates += myautoadd
2813
2814         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2815
2816         if vcs not in ('cvs', 'svn'):
2817                 # With git, bzr and hg, there's never any keyword expansion, so
2818                 # there's no need to regenerate manifests and all files will be
2819                 # committed in one big commit at the end.
2820                 print()
2821         elif not repo_config.thin_manifest:
2822                 if vcs == 'cvs':
2823                         headerstring = "'\$(Header|Id).*\$'"
2824                 elif vcs == "svn":
2825                         svn_keywords = dict((k.lower(), k) for k in [
2826                                         "Rev",
2827                                         "Revision",
2828                                         "LastChangedRevision",
2829                                         "Date",
2830                                         "LastChangedDate",
2831                                         "Author",
2832                                         "LastChangedBy",
2833                                         "URL",
2834                                         "HeadURL",
2835                                         "Id",
2836                                         "Header",
2837                         ])
2838
2839                 for myfile in myupdates:
2840
2841                         # for CVS, no_expansion contains files that are excluded from expansion
2842                         if vcs == "cvs":
2843                                 if myfile in no_expansion:
2844                                         continue
2845
2846                         # for SVN, expansion contains files that are included in expansion
2847                         elif vcs == "svn":
2848                                 if myfile not in expansion:
2849                                         continue
2850
2851                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2852                                 enabled_keywords = []
2853                                 for k in expansion[myfile]:
2854                                         keyword = svn_keywords.get(k.lower())
2855                                         if keyword is not None:
2856                                                 enabled_keywords.append(keyword)
2857
2858                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2859
2860                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2861                                 portage._shell_quote(myfile))
2862                         if myout[0] == 0:
2863                                 myheaders.append(myfile)
2864
2865                 print("%s have headers that will change." % green(str(len(myheaders))))
2866                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2867
2868         logging.info("myupdates: %s", myupdates)
2869         logging.info("myheaders: %s", myheaders)
2870
2871         if options.ask and userquery('Commit changes?', True) != 'Yes':
2872                 print("* aborting commit.")
2873                 sys.exit(128 + signal.SIGINT)
2874
2875         # Handle the case where committed files have keywords which
2876         # will change and need a priming commit before the Manifest
2877         # can be committed.
2878         if (myupdates or myremoved) and myheaders:
2879                 myfiles = myupdates + myremoved
2880                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2881                 mymsg = os.fdopen(fd, "wb")
2882                 mymsg.write(_unicode_encode(commitmessage))
2883                 mymsg.close()
2884
2885                 print()
2886                 print(green("Using commit message:"))
2887                 print(green("------------------------------------------------------------------------------"))
2888                 print(commitmessage)
2889                 print(green("------------------------------------------------------------------------------"))
2890                 print()
2891
2892                 # Having a leading ./ prefix on file paths can trigger a bug in
2893                 # the cvs server when committing files to multiple directories,
2894                 # so strip the prefix.
2895                 myfiles = [f.lstrip("./") for f in myfiles]
2896
2897                 commit_cmd = [vcs]
2898                 commit_cmd.extend(vcs_global_opts)
2899                 commit_cmd.append("commit")
2900                 commit_cmd.extend(vcs_local_opts)
2901                 commit_cmd.extend(["-F", commitmessagefile])
2902                 commit_cmd.extend(myfiles)
2903
2904                 try:
2905                         if options.pretend:
2906                                 print("(%s)" % (" ".join(commit_cmd),))
2907                         else:
2908                                 retval = spawn(commit_cmd, env=commit_env)
2909                                 if retval != os.EX_OK:
2910                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2911                                                 "error code: %s\n") % (vcs, retval),
2912                                                 level=logging.ERROR, noiselevel=-1)
2913                                         sys.exit(retval)
2914                 finally:
2915                         try:
2916                                 os.unlink(commitmessagefile)
2917                         except OSError:
2918                                 pass
2919
2920         # Setup the GPG commands
2921         def gpgsign(filename):
2922                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2923                 if gpgcmd is None:
2924                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2925                                 " Is make.globals missing?")
2926                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2927                         "PORTAGE_GPG_KEY" not in repoman_settings:
2928                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2929                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2930                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2931                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2932                                         os.path.expanduser("~/.gnupg")
2933                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2934                                         % repoman_settings["PORTAGE_GPG_DIR"])
2935                         else:
2936                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2937                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2938                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2939                                 raise portage.exception.InvalidLocation(
2940                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2941                                         repoman_settings["PORTAGE_GPG_DIR"])
2942                 gpgvars = {"FILE": filename}
2943                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2944                         v = repoman_settings.get(k)
2945                         if v is not None:
2946                                 gpgvars[k] = v
2947                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2948                 if options.pretend:
2949                         print("(" + gpgcmd + ")")
2950                 else:
2951                         # Encode unicode manually for bug #310789.
2952                         gpgcmd = portage.util.shlex_split(gpgcmd)
2953
2954                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2955                                 not os.path.isabs(gpgcmd[0]):
2956                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2957                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2958                                 fullname = find_binary(gpgcmd[0])
2959                                 if fullname is None:
2960                                         raise portage.exception.CommandNotFound(gpgcmd[0])
2961                                 gpgcmd[0] = fullname
2962
2963                         gpgcmd = [_unicode_encode(arg,
2964                                 encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2965                         rValue = subprocess.call(gpgcmd)
2966                         if rValue == os.EX_OK:
2967                                 os.rename(filename + ".asc", filename)
2968                         else:
2969                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2970
2971         def need_signature(filename):
2972                 try:
2973                         with open(_unicode_encode(filename,
2974                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2975                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2976                 except IOError as e:
2977                         if e.errno in (errno.ENOENT, errno.ESTALE):
2978                                 return False
2979                         raise
2980
2981         # When files are removed and re-added, the cvs server will put /Attic/
2982         # inside the $Header path. This code detects the problem and corrects it
2983         # so that the Manifest will generate correctly. See bug #169500.
2984         # Use binary mode in order to avoid potential character encoding issues.
2985         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2986         attic_str = b'/Attic/'
2987         attic_replace = b'/'
2988         for x in myheaders:
2989                 f = open(_unicode_encode(x,
2990                         encoding=_encodings['fs'], errors='strict'),
2991                         mode='rb')
2992                 mylines = f.readlines()
2993                 f.close()
2994                 modified = False
2995                 for i, line in enumerate(mylines):
2996                         if cvs_header_re.match(line) is not None and \
2997                                 attic_str in line:
2998                                 mylines[i] = line.replace(attic_str, attic_replace)
2999                                 modified = True
3000                 if modified:
3001                         portage.util.write_atomic(x, b''.join(mylines),
3002                                 mode='wb')
3003
3004         if repolevel == 1:
3005                 print(green("RepoMan sez:"), "\"You're rather crazy... "
3006                         "doing the entire repository.\"\n")
3007
3008         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
3009
3010                 for x in sorted(vcs_files_to_cps(
3011                         chain(myupdates, myremoved, mymanifests))):
3012                         repoman_settings["O"] = os.path.join(repodir, x)
3013                         digestgen(mysettings=repoman_settings, myportdb=portdb)
3014
3015         signed = False
3016         if sign_manifests:
3017                 signed = True
3018                 try:
3019                         for x in sorted(vcs_files_to_cps(
3020                                 chain(myupdates, myremoved, mymanifests))):
3021                                 repoman_settings["O"] = os.path.join(repodir, x)
3022                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
3023                                 if not need_signature(manifest_path):
3024                                         continue
3025                                 gpgsign(manifest_path)
3026                 except portage.exception.PortageException as e:
3027                         portage.writemsg("!!! %s\n" % str(e))
3028                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
3029                         signed = False
3030
3031         if vcs == 'git':
3032                 # It's not safe to use the git commit -a option since there might
3033                 # be some modified files elsewhere in the working tree that the
3034                 # user doesn't want to commit. Therefore, call git update-index
3035                 # in order to ensure that the index is updated with the latest
3036                 # versions of all new and modified files in the relevant portion
3037                 # of the working tree.
3038                 myfiles = mymanifests + myupdates
3039                 myfiles.sort()
3040                 update_index_cmd = ["git", "update-index"]
3041                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
3042                 if options.pretend:
3043                         print("(%s)" % (" ".join(update_index_cmd),))
3044                 else:
3045                         retval = spawn(update_index_cmd, env=os.environ)
3046                         if retval != os.EX_OK:
3047                                 writemsg_level(("!!! Exiting on %s (shell) " + \
3048                                         "error code: %s\n") % (vcs, retval),
3049                                         level=logging.ERROR, noiselevel=-1)
3050                                 sys.exit(retval)
3051
3052         if True:
3053                 myfiles = mymanifests[:]
3054                 # If there are no header (SVN/CVS keywords) changes in
3055                 # the files, this Manifest commit must include the
3056                 # other (yet uncommitted) files.
3057                 if not myheaders:
3058                         myfiles += myupdates
3059                         myfiles += myremoved
3060                 myfiles.sort()
3061
3062                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3063                 mymsg = os.fdopen(fd, "wb")
3064                 mymsg.write(_unicode_encode(commitmessage))
3065                 mymsg.close()
3066
3067                 commit_cmd = []
3068                 if options.pretend and vcs is None:
3069                         # substitute a bogus value for pretend output
3070                         commit_cmd.append("cvs")
3071                 else:
3072                         commit_cmd.append(vcs)
3073                 commit_cmd.extend(vcs_global_opts)
3074                 commit_cmd.append("commit")
3075                 commit_cmd.extend(vcs_local_opts)
3076                 if vcs == "hg":
3077                         commit_cmd.extend(["--logfile", commitmessagefile])
3078                         commit_cmd.extend(myfiles)
3079                 else:
3080                         commit_cmd.extend(["-F", commitmessagefile])
3081                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3082
3083                 try:
3084                         if options.pretend:
3085                                 print("(%s)" % (" ".join(commit_cmd),))
3086                         else:
3087                                 retval = spawn(commit_cmd, env=commit_env)
3088                                 if retval != os.EX_OK:
3089                                         if repo_config.sign_commit and vcs == 'git' and \
3090                                                 not git_supports_gpg_sign():
3091                                                 # Inform user that newer git is needed (bug #403323).
3092                                                 logging.error(
3093                                                         "Git >=1.7.9 is required for signed commits!")
3094
3095                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3096                                                 "error code: %s\n") % (vcs, retval),
3097                                                 level=logging.ERROR, noiselevel=-1)
3098                                         sys.exit(retval)
3099                 finally:
3100                         try:
3101                                 os.unlink(commitmessagefile)
3102                         except OSError:
3103                                 pass
3104
3105         print()
3106         if vcs:
3107                 print("Commit complete.")
3108         else:
3109                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3110         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3111 sys.exit(0)