bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import codecs
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27 from itertools import chain
  28 from stat import S_ISDIR
  29
  30 try:
  31         from urllib.parse import urlparse
  32 except ImportError:
  33         from urlparse import urlparse
  34
  35 from os import path as osp
  36 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  37 sys.path.insert(0, pym_path)
  38 import portage
  39 portage._internal_caller = True
  40 portage._disable_legacy_globals()
  41
  42 try:
  43         import xml.etree.ElementTree
  44         from xml.parsers.expat import ExpatError
  45 except (SystemExit, KeyboardInterrupt):
  46         raise
  47 except (ImportError, SystemError, RuntimeError, Exception):
  48         # broken or missing xml support
  49         # http://bugs.python.org/issue14988
  50         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  51         from portage.output import EOutput
  52         out = EOutput()
  53         for line in msg:
  54                 out.eerror(line)
  55         sys.exit(1)
  56
  57 from portage import os
  58 from portage import _encodings
  59 from portage import _unicode_encode
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 from portage import cvstree, normalize_path
  69 from portage import util
  70 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  71         ParseError, PermissionDenied)
  72 from portage.dep import Atom
  73 from portage.process import find_binary, spawn
  74 from portage.output import bold, create_color_func, \
  75         green, nocolor, red
  76 from portage.output import ConsoleStyleFile, StyleWriter
  77 from portage.util import writemsg_level
  78 from portage.package.ebuild.digestgen import digestgen
  79 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  80
  81 if sys.hexversion >= 0x3000000:
  82         basestring = str
  83
  84 util.initialize_logger()
  85
  86 # 14 is the length of DESCRIPTION=""
  87 max_desc_len = 100
  88 allowed_filename_chars="a-zA-Z0-9._-+:"
  89 pv_toolong_re = re.compile(r'[0-9]{19,}')
  90 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  91 bad = create_color_func("BAD")
  92
  93 # A sane umask is needed for files that portage creates.
  94 os.umask(0o22)
  95 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  96 # behave incrementally.
  97 repoman_incrementals = tuple(x for x in \
  98         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
  99 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 100 repoman_settings = portage.config(config_root=config_root, local_config=False)
 101
 102 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 103         repoman_settings.get('TERM') == 'dumb' or \
 104         not sys.stdout.isatty():
 105         nocolor()
 106
 107 def warn(txt):
 108         print("repoman: " + txt)
 109
 110 def err(txt):
 111         warn(txt)
 112         sys.exit(1)
 113
 114 def exithandler(signum=None, frame=None):
 115         logging.fatal("Interrupted; exiting...")
 116         if signum is None:
 117                 sys.exit(1)
 118         else:
 119                 sys.exit(128 + signum)
 120
 121 signal.signal(signal.SIGINT, exithandler)
 122
 123 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 124         """Repoman needs its own HelpFormatter for now, because the default ones
 125         murder the help text."""
 126
 127         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 128                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 129
 130         def format_description(self, description):
 131                 return description
 132
 133 class RepomanOptionParser(optparse.OptionParser):
 134         """Add the on_tail function, ruby has it, optionParser should too
 135         """
 136
 137         def __init__(self, *args, **kwargs):
 138                 optparse.OptionParser.__init__(self, *args, **kwargs)
 139                 self.tail = ""
 140
 141         def on_tail(self, description):
 142                 self.tail += description
 143
 144         def format_help(self, formatter=None):
 145                 result = optparse.OptionParser.format_help(self, formatter)
 146                 result += self.tail
 147                 return result
 148
 149
 150 def ParseArgs(argv, qahelp):
 151         """This function uses a customized optionParser to parse command line arguments for repoman
 152         Args:
 153           argv - a sequence of command line arguments
 154                 qahelp - a dict of qa warning to help message
 155         Returns:
 156           (opts, args), just like a call to parser.parse_args()
 157         """
 158
 159         argv = portage._decode_argv(argv)
 160
 161         modes = {
 162                 'commit' : 'Run a scan then commit changes',
 163                 'ci' : 'Run a scan then commit changes',
 164                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 165                 'full' : 'Scan directory tree and print all issues (not a summary)',
 166                 'help' : 'Show this screen',
 167                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 168                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 169                 'scan' : 'Scan directory tree for QA issues'
 170         }
 171
 172         mode_keys = list(modes)
 173         mode_keys.sort()
 174
 175         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 176         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 177         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 178         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 179         parser.description += "\nmodes: " + " | ".join(map(green, mode_keys))
 180
 181         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 182                 help='Request a confirmation before commiting')
 183
 184         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 185                 help='specify a commit message on the command line')
 186
 187         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 188                 help='specify a path to a file that contains a commit message')
 189
 190         parser.add_option('--digest',
 191                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 192                 help='Automatically update Manifest digests for modified files')
 193
 194         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 195                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 196
 197         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 198                 help='do not print unnecessary messages')
 199
 200         parser.add_option(
 201                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 202                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 203                 'regardless of modification if \'force\' is specified)')
 204
 205         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 206                 help='Commit with QA violations')
 207
 208         parser.add_option('--vcs', dest='vcs',
 209                 help='Force using specific VCS instead of autodetection')
 210
 211         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 212                 help='be very verbose in output', default=0)
 213
 214         parser.add_option('-V', '--version', dest='version', action='store_true',
 215                 help='show version info')
 216
 217         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 218                 default=False, help='forces the metadata.xml parse check to be carried out')
 219
 220         parser.add_option(
 221                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 222                 metavar="<y|n>",
 223                 help='only check packages that have uncommitted modifications')
 224
 225         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 226                 default=False, help='ignore arch-specific failures (where arch != host)')
 227
 228         parser.add_option("--ignore-default-opts",
 229                 action="store_true",
 230                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 231
 232         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 233                 default=False, help='ignore masked packages (not allowed with commit mode)')
 234
 235         parser.add_option('--include-arches', dest='include_arches',
 236                 metavar='ARCHES', action='append',
 237                 help='A space separated list of arches used to '
 238                 'filter the selection of profiles for dependency checks')
 239
 240         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 241                 default=False, help='include dev profiles in dependency checks')
 242
 243         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 244                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 245
 246         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 247                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 248
 249         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 250                 help='specify which mode repoman will run in (default=full)')
 251
 252         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 253
 254         for k in mode_keys:
 255                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 256
 257         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 258
 259         sorted_qa = list(qahelp)
 260         sorted_qa.sort()
 261         for k in sorted_qa:
 262                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 263
 264         opts, args = parser.parse_args(argv[1:])
 265
 266         if not opts.ignore_default_opts:
 267                 default_opts = portage.util.shlex_split(
 268                         repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
 269                 if default_opts:
 270                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 271
 272         if opts.mode == 'help':
 273                 parser.print_help(short=False)
 274
 275         for arg in args:
 276                 if arg in modes:
 277                         if not opts.mode:
 278                                 opts.mode = arg
 279                                 break
 280                 else:
 281                         parser.error("invalid mode: %s" % arg)
 282
 283         if not opts.mode:
 284                 opts.mode = 'full'
 285
 286         if opts.mode == 'ci':
 287                 opts.mode = 'commit'  # backwards compat shortcut
 288
 289         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 290                 if opts.ignore_masked:
 291                         parser.error('Commit mode and --ignore-masked are not compatible')
 292                 if opts.without_mask:
 293                         parser.error('Commit mode and --without-mask are not compatible')
 294
 295         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 296         for val in range(opts.verbosity):
 297                 logger = logging.getLogger()
 298                 logger.setLevel(logger.getEffectiveLevel() - 10)
 299
 300         for val in range(opts.quiet):
 301                 logger = logging.getLogger()
 302                 logger.setLevel(logger.getEffectiveLevel() + 10)
 303
 304         return (opts, args)
 305
 306 qahelp = {
 307         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 308         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 309         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 310         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 311         "changelog.missing": "Missing ChangeLog files",
 312         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 313         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 314         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 315         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 316         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 317         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 318         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 319         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 320         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 321         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 322         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 323         "file.size": "Files in the files directory must be under 20 KiB",
 324         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 325         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 326         "file.UTF8": "File is not UTF8 compliant",
 327         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 328         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 329         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 330         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 331         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 332         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 333         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 334         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 335         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 336         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 337         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 338         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 339         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 340         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 341         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 342         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 343         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 344         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 345         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 346         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 347         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 348         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 349         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 350         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 351         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 352         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 353         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 354         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 355         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 356         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 357         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 358         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 359         "variable.readonly": "Assigning a readonly variable",
 360         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 361         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 362         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 363         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 364         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 365         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 366         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 367         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 368         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 369         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 370         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 371         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 372         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 373         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 374         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 375         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 376         "ebuild.badheader": "This ebuild has a malformed header",
 377         "manifest.bad": "Manifest has missing or incorrect digests",
 378         "metadata.missing": "Missing metadata.xml files",
 379         "metadata.bad": "Bad metadata.xml files",
 380         "metadata.warning": "Warnings in metadata.xml files",
 381         "portage.internal": "The ebuild uses an internal Portage function or variable",
 382         "repo.eapi.banned": "The ebuild uses an EAPI which is banned by the repository's metadata/layout.conf settings",
 383         "repo.eapi.deprecated": "The ebuild uses an EAPI which is deprecated by the repository's metadata/layout.conf settings",
 384         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 385         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 386         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 387         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 388 }
 389
 390 qacats = list(qahelp)
 391 qacats.sort()
 392
 393 qawarnings = set((
 394 "changelog.missing",
 395 "changelog.notadded",
 396 "dependency.unknown",
 397 "digest.assumed",
 398 "digest.unused",
 399 "ebuild.notadded",
 400 "ebuild.nesteddie",
 401 "dependency.badmasked",
 402 "dependency.badindev",
 403 "dependency.badmaskedindev",
 404 "dependency.badtilde",
 405 "DESCRIPTION.toolong",
 406 "EAPI.deprecated",
 407 "HOMEPAGE.virtual",
 408 "LICENSE.deprecated",
 409 "LICENSE.virtual",
 410 "KEYWORDS.dropped",
 411 "KEYWORDS.stupid",
 412 "KEYWORDS.missing",
 413 "PDEPEND.suspect",
 414 "RDEPEND.implicit",
 415 "RDEPEND.suspect",
 416 "virtual.suspect",
 417 "RESTRICT.invalid",
 418 "ebuild.minorsyn",
 419 "ebuild.badheader",
 420 "ebuild.patches",
 421 "file.size",
 422 "inherit.unused",
 423 "inherit.deprecated",
 424 "java.eclassesnotused",
 425 "wxwidgets.eclassnotused",
 426 "metadata.warning",
 427 "portage.internal",
 428 "repo.eapi.deprecated",
 429 "usage.obsolete",
 430 "upstream.workaround",
 431 "LIVEVCS.stable",
 432 "LIVEVCS.unmasked",
 433 ))
 434
 435 if portage.const._ENABLE_INHERIT_CHECK:
 436         # This is experimental, so it's non-fatal.
 437         qawarnings.add("inherit.missing")
 438
 439 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 440
 441 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 442 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 443 allvars.update(Package.metadata_keys)
 444 allvars = sorted(allvars)
 445 commitmessage = None
 446 for x in missingvars:
 447         x += ".missing"
 448         if x not in qacats:
 449                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 450                 qacats.append(x)
 451                 qawarnings.add(x)
 452
 453 valid_restrict = frozenset(["binchecks", "bindist",
 454         "fetch", "installsources", "mirror", "preserve-libs",
 455         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 456
 457 live_eclasses = frozenset([
 458         "bzr",
 459         "cvs",
 460         "darcs",
 461         "git",
 462         "git-2",
 463         "mercurial",
 464         "subversion",
 465         "tla",
 466 ])
 467
 468 suspect_rdepend = frozenset([
 469         "app-arch/cabextract",
 470         "app-arch/rpm2targz",
 471         "app-doc/doxygen",
 472         "dev-lang/nasm",
 473         "dev-lang/swig",
 474         "dev-lang/yasm",
 475         "dev-perl/extutils-pkgconfig",
 476         "dev-util/byacc",
 477         "dev-util/cmake",
 478         "dev-util/ftjam",
 479         "dev-util/gperf",
 480         "dev-util/gtk-doc",
 481         "dev-util/gtk-doc-am",
 482         "dev-util/intltool",
 483         "dev-util/jam",
 484         "dev-util/pkg-config-lite",
 485         "dev-util/pkgconf",
 486         "dev-util/pkgconfig",
 487         "dev-util/pkgconfig-openbsd",
 488         "dev-util/scons",
 489         "dev-util/unifdef",
 490         "dev-util/yacc",
 491         "media-gfx/ebdftopcf",
 492         "sys-apps/help2man",
 493         "sys-devel/autoconf",
 494         "sys-devel/automake",
 495         "sys-devel/bin86",
 496         "sys-devel/bison",
 497         "sys-devel/dev86",
 498         "sys-devel/flex",
 499         "sys-devel/m4",
 500         "sys-devel/pmake",
 501         "virtual/linux-sources",
 502         "virtual/pkgconfig",
 503         "x11-misc/bdftopcf",
 504         "x11-misc/imake",
 505 ])
 506
 507 suspect_virtual = {
 508         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 509         "dev-util/pkgconf":"virtual/pkgconfig",
 510         "dev-util/pkgconfig":"virtual/pkgconfig",
 511         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 512         "dev-libs/libusb":"virtual/libusb",
 513         "dev-libs/libusbx":"virtual/libusb",
 514         "dev-libs/libusb-compat":"virtual/libusb",
 515 }
 516
 517 metadata_xml_encoding = 'UTF-8'
 518 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 519         (metadata_xml_encoding,)
 520 metadata_doctype_name = 'pkgmetadata'
 521 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 522 # force refetch if the local copy creation time is older than this
 523 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 524
 525 # file.executable
 526 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 527
 528 options, arguments = ParseArgs(sys.argv, qahelp)
 529
 530 if options.version:
 531         print("Portage", portage.VERSION)
 532         sys.exit(0)
 533
 534 # Set this to False when an extraordinary issue (generally
 535 # something other than a QA issue) makes it impossible to
 536 # commit (like if Manifest generation fails).
 537 can_force = True
 538
 539 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 540 if portdir is None:
 541         sys.exit(1)
 542
 543 myreporoot = os.path.basename(portdir_overlay)
 544 myreporoot += mydir[len(portdir_overlay):]
 545
 546 if options.vcs:
 547         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 548                 vcs = options.vcs
 549         else:
 550                 vcs = None
 551 else:
 552         vcses = utilities.FindVCS()
 553         if len(vcses) > 1:
 554                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 555                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 556                 sys.exit(1)
 557         elif vcses:
 558                 vcs = vcses[0]
 559         else:
 560                 vcs = None
 561
 562 if options.if_modified == "y" and vcs is None:
 563         logging.info("Not in a version controlled repository; "
 564                 "disabling --if-modified.")
 565         options.if_modified = "n"
 566
 567 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 568 vcs_preserves_mtime = vcs in ('cvs',)
 569
 570 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 571 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 572 if vcs_global_opts is None:
 573         if vcs in ('cvs', 'svn'):
 574                 vcs_global_opts = "-q"
 575         else:
 576                 vcs_global_opts = ""
 577 vcs_global_opts = vcs_global_opts.split()
 578
 579 if options.mode == 'commit' and not options.pretend and not vcs:
 580         logging.info("Not in a version controlled repository; enabling pretend mode.")
 581         options.pretend = True
 582
 583 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 584 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 585         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 586         portage._shell_quote(portdir_overlay))
 587 # We have to call the config constructor again so
 588 # that config.repositories is initialized correctly.
 589 repoman_settings = portage.config(config_root=config_root, local_config=False,
 590         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 591
 592 root = repoman_settings['EROOT']
 593 trees = {
 594         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 595 }
 596 portdb = trees[root]['porttree'].dbapi
 597
 598 # Constrain dependency resolution to the master(s)
 599 # that are specified in layout.conf.
 600 repodir = os.path.realpath(portdir_overlay)
 601 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 602 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 603 portdir = portdb.porttrees[0]
 604 commit_env = os.environ.copy()
 605
 606 if repo_config.allow_provide_virtual:
 607         qawarnings.add("virtual.oldstyle")
 608
 609 if repo_config.sign_commit:
 610         if vcs == 'git':
 611                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 612                 # the commit arguments. If key_id is unspecified, then it must be
 613                 # configured by `git config user.signingkey key_id`.
 614                 vcs_local_opts.append("--gpg-sign")
 615                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 616                         # Pass GNUPGHOME to git for bug #462362.
 617                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 618
 619 # In order to disable manifest signatures, repos may set
 620 # "sign-manifests = false" in metadata/layout.conf. This
 621 # can be used to prevent merge conflicts like those that
 622 # thin-manifests is designed to prevent.
 623 sign_manifests = "sign" in repoman_settings.features and \
 624         repo_config.sign_manifest
 625
 626 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 627         options.mode in ("commit",) and not sign_manifests:
 628         msg = ("The '%s' repository has manifest signatures enabled, "
 629         "but FEATURES=sign is currently disabled. In order to avoid this "
 630         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 631         "repositories can disable manifest signatures by setting "
 632         "'sign-manifests = false' in metadata/layout.conf.") % \
 633         (repo_config.name,)
 634         for line in textwrap.wrap(msg, 60):
 635                 logging.warn(line)
 636
 637 if sign_manifests and options.mode in ("commit",) and \
 638         repoman_settings.get("PORTAGE_GPG_KEY") and \
 639         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 640         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 641         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 642                 repoman_settings["PORTAGE_GPG_KEY"])
 643         sys.exit(1)
 644
 645 manifest_hashes = repo_config.manifest_hashes
 646 if manifest_hashes is None:
 647         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 648
 649 if options.mode in ("commit", "fix", "manifest"):
 650         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 651                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 652                 "metadata/layout.conf does not contain the '%s' hash which "
 653                 "is required by this portage version. You will have to "
 654                 "upgrade portage if you want to generate valid manifests for "
 655                 "this repository.") % \
 656                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 657                 for line in textwrap.wrap(msg, 70):
 658                         logging.error(line)
 659                 sys.exit(1)
 660
 661         unsupported_hashes = manifest_hashes.difference(
 662                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 663         if unsupported_hashes:
 664                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 665                 "metadata/layout.conf contains one or more hash types '%s' "
 666                 "which are not supported by this portage version. You will "
 667                 "have to upgrade portage if you want to generate valid "
 668                 "manifests for this repository.") % \
 669                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 670                 for line in textwrap.wrap(msg, 70):
 671                         logging.error(line)
 672                 sys.exit(1)
 673
 674 if options.echangelog is None and repo_config.update_changelog:
 675         options.echangelog = 'y'
 676
 677 if vcs is None:
 678         options.echangelog = 'n'
 679
 680 # The --echangelog option causes automatic ChangeLog generation,
 681 # which invalidates changelog.ebuildadded and changelog.missing
 682 # checks.
 683 # Note: Some don't use ChangeLogs in distributed SCMs.
 684 # It will be generated on server side from scm log,
 685 # before package moves to the rsync server.
 686 # This is needed because they try to avoid merge collisions.
 687 # Gentoo's Council decided to always use the ChangeLog file.
 688 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 689 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 690
 691 if 'digest' in repoman_settings.features and options.digest != 'n':
 692         options.digest = 'y'
 693
 694 logging.debug("vcs: %s" % (vcs,))
 695 logging.debug("repo config: %s" % (repo_config,))
 696 logging.debug("options: %s" % (options,))
 697
 698 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 699 # profile-specific config constructor calls.
 700 env = os.environ.copy()
 701 env['PORTDIR'] = portdir
 702 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 703
 704 logging.info('Setting paths:')
 705 logging.info('PORTDIR = "' + portdir + '"')
 706 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 707
 708 # It's confusing if these warnings are displayed without the user
 709 # being told which profile they come from, so disable them.
 710 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 711
 712 categories = []
 713 for path in repo_config.eclass_db.porttrees:
 714         categories.extend(portage.util.grabfile(
 715                 os.path.join(path, 'profiles', 'categories')))
 716 repoman_settings.categories = frozenset(
 717         portage.util.stack_lists([categories], incremental=1))
 718 categories = repoman_settings.categories
 719
 720 portdb.settings = repoman_settings
 721 root_config = RootConfig(repoman_settings, trees[root], None)
 722 # We really only need to cache the metadata that's necessary for visibility
 723 # filtering. Anything else can be discarded to reduce memory consumption.
 724 portdb._aux_cache_keys.clear()
 725 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 726
 727 reposplit = myreporoot.split(os.path.sep)
 728 repolevel = len(reposplit)
 729
 730 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 731 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 732 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 733 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 734         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 735         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 736         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 737         print(red("***"))
 738         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 739
 740 # Make startdir relative to the canonical repodir, so that we can pass
 741 # it to digestgen and it won't have to be canonicalized again.
 742 if repolevel == 1:
 743         startdir = repodir
 744 else:
 745         startdir = normalize_path(mydir)
 746         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 747
 748 def caterror(mycat):
 749         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 750
 751 def repoman_getstatusoutput(cmd):
 752         """
 753         Implements an interface similar to getstatusoutput(), but with
 754         customized unicode handling (see bug #310789) and without the shell.
 755         """
 756         args = portage.util.shlex_split(cmd)
 757         encoding = _encodings['fs']
 758         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 759                 # Python 3.1 does not support bytes in Popen args.
 760                 args = [_unicode_encode(x,
 761                         encoding=encoding, errors='strict') for x in args]
 762         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 763                 stderr=subprocess.STDOUT)
 764         output = portage._unicode_decode(proc.communicate()[0],
 765                 encoding=encoding, errors='strict')
 766         if output and output[-1] == "\n":
 767                 # getstatusoutput strips one newline
 768                 output = output[:-1]
 769         return (proc.wait(), output)
 770
 771 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 772         """
 773         Implements an interface similar to os.popen(), but with customized
 774         unicode handling (see bug #310789) and without the shell.
 775         """
 776
 777         __slots__ = ('_proc', '_stdout')
 778
 779         def __init__(self, cmd):
 780                 args = portage.util.shlex_split(cmd)
 781                 encoding = _encodings['fs']
 782                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 783                         # Python 3.1 does not support bytes in Popen args.
 784                         args = [_unicode_encode(x,
 785                                 encoding=encoding, errors='strict') for x in args]
 786                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 787                 object.__setattr__(self, '_proc', proc)
 788                 object.__setattr__(self, '_stdout',
 789                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 790
 791         def _get_target(self):
 792                 return object.__getattribute__(self, '_stdout')
 793
 794         __enter__ = _get_target
 795
 796         def __exit__(self, exc_type, exc_value, traceback):
 797                 proc = object.__getattribute__(self, '_proc')
 798                 proc.wait()
 799                 proc.stdout.close()
 800
 801 class ProfileDesc(object):
 802         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 803         def __init__(self, arch, status, sub_path, tree_path):
 804                 self.arch = arch
 805                 self.status = status
 806                 if sub_path:
 807                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 808                 self.sub_path = sub_path
 809                 self.tree_path = tree_path
 810                 if tree_path:
 811                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 812                 else:
 813                         self.abs_path = tree_path
 814
 815         def __str__(self):
 816                 if self.sub_path:
 817                         return self.sub_path
 818                 return 'empty profile'
 819
 820 profile_list = []
 821 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 822
 823 # get lists of valid keywords, licenses, and use
 824 kwlist = set()
 825 liclist = set()
 826 uselist = set()
 827 global_pmasklines = []
 828
 829 for path in portdb.porttrees:
 830         try:
 831                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 832         except OSError:
 833                 pass
 834         kwlist.update(portage.grabfile(os.path.join(path,
 835                 "profiles", "arch.list")))
 836
 837         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 838         for x in use_desc:
 839                 x = x.split()
 840                 if x:
 841                         uselist.add(x[0])
 842
 843         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 844         try:
 845                 expand_list = os.listdir(expand_desc_dir)
 846         except OSError:
 847                 pass
 848         else:
 849                 for fn in expand_list:
 850                         if not fn[-5:] == '.desc':
 851                                 continue
 852                         use_prefix = fn[:-5].lower() + '_'
 853                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 854                                 x = x.split()
 855                                 if x:
 856                                         uselist.add(use_prefix + x[0])
 857
 858         global_pmasklines.append(portage.util.grabfile_package(
 859                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 860
 861         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 862         try:
 863                 desc_file = io.open(_unicode_encode(desc_path,
 864                         encoding=_encodings['fs'], errors='strict'),
 865                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 866         except EnvironmentError:
 867                 pass
 868         else:
 869                 for i, x in enumerate(desc_file):
 870                         if x[0] == "#":
 871                                 continue
 872                         arch = x.split()
 873                         if len(arch) == 0:
 874                                 continue
 875                         if len(arch) != 3:
 876                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 877                                         desc_path + " line %d" % (i + 1, ))
 878                         elif arch[0] not in kwlist:
 879                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 880                                         desc_path + " line %d" % (i + 1, ))
 881                         elif arch[2] not in valid_profile_types:
 882                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 883                                         desc_path + " line %d" % (i + 1, ))
 884                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 885                         if not os.path.isdir(profile_desc.abs_path):
 886                                 logging.error(
 887                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 888                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 889                                 continue
 890                         if os.path.exists(
 891                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 892                                 continue
 893                         profile_list.append(profile_desc)
 894                 desc_file.close()
 895
 896 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 897 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 898
 899 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 900 global_pmaskdict = {}
 901 for x in global_pmasklines:
 902         global_pmaskdict.setdefault(x.cp, []).append(x)
 903 del global_pmasklines
 904
 905 def has_global_mask(pkg):
 906         mask_atoms = global_pmaskdict.get(pkg.cp)
 907         if mask_atoms:
 908                 pkg_list = [pkg]
 909                 for x in mask_atoms:
 910                         if portage.dep.match_from_list(x, pkg_list):
 911                                 return x
 912         return None
 913
 914 # Ensure that profile sub_path attributes are unique. Process in reverse order
 915 # so that profiles with duplicate sub_path from overlays will override
 916 # profiles with the same sub_path from parent repos.
 917 profiles = {}
 918 profile_list.reverse()
 919 profile_sub_paths = set()
 920 for prof in profile_list:
 921         if prof.sub_path in profile_sub_paths:
 922                 continue
 923         profile_sub_paths.add(prof.sub_path)
 924         profiles.setdefault(prof.arch, []).append(prof)
 925
 926 # Use an empty profile for checking dependencies of
 927 # packages that have empty KEYWORDS.
 928 prof = ProfileDesc('**', 'stable', '', '')
 929 profiles.setdefault(prof.arch, []).append(prof)
 930
 931 for x in repoman_settings.archlist():
 932         if x[0] == "~":
 933                 continue
 934         if x not in profiles:
 935                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 936                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 937                 print(red("up with the " + x + " team."))
 938                 print()
 939
 940 liclist_deprecated = set()
 941 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 942         liclist_deprecated.update(
 943                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 944
 945 if not liclist:
 946         logging.fatal("Couldn't find licenses?")
 947         sys.exit(1)
 948
 949 if not kwlist:
 950         logging.fatal("Couldn't read KEYWORDS from arch.list")
 951         sys.exit(1)
 952
 953 if not uselist:
 954         logging.fatal("Couldn't find use.desc?")
 955         sys.exit(1)
 956
 957 scanlist = []
 958 if repolevel == 2:
 959         # we are inside a category directory
 960         catdir = reposplit[-1]
 961         if catdir not in categories:
 962                 caterror(catdir)
 963         mydirlist = os.listdir(startdir)
 964         for x in mydirlist:
 965                 if x == "CVS" or x.startswith("."):
 966                         continue
 967                 if os.path.isdir(startdir + "/" + x):
 968                         scanlist.append(catdir + "/" + x)
 969         repo_subdir = catdir + os.sep
 970 elif repolevel == 1:
 971         for x in categories:
 972                 if not os.path.isdir(startdir + "/" + x):
 973                         continue
 974                 for y in os.listdir(startdir + "/" + x):
 975                         if y == "CVS" or y.startswith("."):
 976                                 continue
 977                         if os.path.isdir(startdir + "/" + x + "/" + y):
 978                                 scanlist.append(x + "/" + y)
 979         repo_subdir = ""
 980 elif repolevel == 3:
 981         catdir = reposplit[-2]
 982         if catdir not in categories:
 983                 caterror(catdir)
 984         scanlist.append(catdir + "/" + reposplit[-1])
 985         repo_subdir = scanlist[-1] + os.sep
 986 else:
 987         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 988                 ' from the current working directory'
 989         logging.critical(msg)
 990         sys.exit(1)
 991
 992 repo_subdir_len = len(repo_subdir)
 993 scanlist.sort()
 994
 995 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 996
 997 def vcs_files_to_cps(vcs_file_iter):
 998         """
 999         Iterate over the given modified file paths returned from the vcs,
1000         and return a frozenset containing category/pn strings for each
1001         modified package.
1002         """
1003
1004         modified_cps = []
1005
1006         if repolevel == 3:
1007                 if reposplit[-2] in categories and \
1008                         next(vcs_file_iter, None) is not None:
1009                         modified_cps.append("/".join(reposplit[-2:]))
1010
1011         elif repolevel == 2:
1012                 category = reposplit[-1]
1013                 if category in categories:
1014                         for filename in vcs_file_iter:
1015                                 f_split = filename.split(os.sep)
1016                                 # ['.', pn, ...]
1017                                 if len(f_split) > 2:
1018                                         modified_cps.append(category + "/" + f_split[1])
1019
1020         else:
1021                 # repolevel == 1
1022                 for filename in vcs_file_iter:
1023                         f_split = filename.split(os.sep)
1024                         # ['.', category, pn, ...]
1025                         if len(f_split) > 3 and f_split[1] in categories:
1026                                 modified_cps.append("/".join(f_split[1:3]))
1027
1028         return frozenset(modified_cps)
1029
1030 def git_supports_gpg_sign():
1031         status, cmd_output = \
1032                 repoman_getstatusoutput("git --version")
1033         cmd_output = cmd_output.split()
1034         if cmd_output:
1035                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1036                 if version is not None:
1037                         version = [int(x) for x in version.groups()]
1038                         if version[0] > 1 or \
1039                                 (version[0] == 1 and version[1] > 7) or \
1040                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1041                                 return True
1042         return False
1043
1044 def dev_keywords(profiles):
1045         """
1046         Create a set of KEYWORDS values that exist in 'dev'
1047         profiles. These are used
1048         to trigger a message notifying the user when they might
1049         want to add the --include-dev option.
1050         """
1051         type_arch_map = {}
1052         for arch, arch_profiles in profiles.items():
1053                 for prof in arch_profiles:
1054                         arch_set = type_arch_map.get(prof.status)
1055                         if arch_set is None:
1056                                 arch_set = set()
1057                                 type_arch_map[prof.status] = arch_set
1058                         arch_set.add(arch)
1059
1060         dev_keywords = type_arch_map.get('dev', set())
1061         dev_keywords.update(['~' + arch for arch in dev_keywords])
1062         return frozenset(dev_keywords)
1063
1064 dev_keywords = dev_keywords(profiles)
1065
1066 stats = {}
1067 fails = {}
1068
1069 for x in qacats:
1070         stats[x] = 0
1071         fails[x] = []
1072
1073 xmllint_capable = False
1074 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1075
1076 def fetch_metadata_dtd():
1077         """
1078         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1079         metadata_dtd_ctime_interval.
1080         @rtype: bool
1081         @return: True if successful, otherwise False
1082         """
1083
1084         must_fetch = True
1085         metadata_dtd_st = None
1086         current_time = int(time.time())
1087         try:
1088                 metadata_dtd_st = os.stat(metadata_dtd)
1089         except EnvironmentError as e:
1090                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1091                         raise
1092                 del e
1093         else:
1094                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1095                 if abs(current_time - metadata_dtd_st.st_ctime) \
1096                         < metadata_dtd_ctime_interval:
1097                         must_fetch = False
1098
1099         if must_fetch:
1100                 print()
1101                 print(green("***") + " the local copy of metadata.dtd " + \
1102                         "needs to be refetched, doing that now")
1103                 print()
1104                 parsed_url = urlparse(metadata_dtd_uri)
1105                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
1106                 fcmd = repoman_settings.get(setting)
1107                 if not fcmd:
1108                         fcmd = repoman_settings.get('FETCHCOMMAND')
1109                         if not fcmd:
1110                                 logging.error("FETCHCOMMAND is unset")
1111                                 return False
1112
1113                 destdir = repoman_settings["DISTDIR"]
1114                 fd, metadata_dtd_tmp = tempfile.mkstemp(
1115                         prefix='metadata.dtd.', dir=destdir)
1116                 os.close(fd)
1117
1118                 try:
1119                         if not portage.getbinpkg.file_get(metadata_dtd_uri,
1120                                 destdir, fcmd=fcmd,
1121                                 filename=os.path.basename(metadata_dtd_tmp)):
1122                                 logging.error("failed to fetch metadata.dtd from '%s'" %
1123                                         metadata_dtd_uri)
1124                                 return False
1125
1126                         try:
1127                                 portage.util.apply_secpass_permissions(metadata_dtd_tmp,
1128                                         gid=portage.data.portage_gid, mode=0o664, mask=0o2)
1129                         except portage.exception.PortageException:
1130                                 pass
1131
1132                         os.rename(metadata_dtd_tmp, metadata_dtd)
1133                 finally:
1134                         try:
1135                                 os.unlink(metadata_dtd_tmp)
1136                         except OSError:
1137                                 pass
1138
1139         return True
1140
1141 if options.mode == "manifest":
1142         pass
1143 elif not find_binary('xmllint'):
1144         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1145         if options.xml_parse or repolevel == 3:
1146                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1147                 sys.exit(1)
1148 else:
1149         if not fetch_metadata_dtd():
1150                 sys.exit(1)
1151         # this can be problematic if xmllint changes their output
1152         xmllint_capable = True
1153
1154 if options.mode == 'commit' and vcs:
1155         utilities.detect_vcs_conflicts(options, vcs)
1156
1157 if options.mode == "manifest":
1158         pass
1159 elif options.pretend:
1160         print(green("\nRepoMan does a once-over of the neighborhood..."))
1161 else:
1162         print(green("\nRepoMan scours the neighborhood..."))
1163
1164 new_ebuilds = set()
1165 modified_ebuilds = set()
1166 modified_changelogs = set()
1167 mychanged = []
1168 mynew = []
1169 myremoved = []
1170
1171 if vcs == "cvs":
1172         mycvstree = cvstree.getentries("./", recursive=1)
1173         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1174         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1175         if options.if_modified == "y":
1176                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1177
1178 elif vcs == "svn":
1179         with repoman_popen("svn status") as f:
1180                 svnstatus = f.readlines()
1181         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1182         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1183         if options.if_modified == "y":
1184                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1185
1186 elif vcs == "git":
1187         with repoman_popen("git diff-index --name-only "
1188                 "--relative --diff-filter=M HEAD") as f:
1189                 mychanged = f.readlines()
1190         mychanged = ["./" + elem[:-1] for elem in mychanged]
1191
1192         with repoman_popen("git diff-index --name-only "
1193                 "--relative --diff-filter=A HEAD") as f:
1194                 mynew = f.readlines()
1195         mynew = ["./" + elem[:-1] for elem in mynew]
1196         if options.if_modified == "y":
1197                 with repoman_popen("git diff-index --name-only "
1198                         "--relative --diff-filter=D HEAD") as f:
1199                         myremoved = f.readlines()
1200                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1201
1202 elif vcs == "bzr":
1203         with repoman_popen("bzr status -S .") as f:
1204                 bzrstatus = f.readlines()
1205         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1206         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1207         if options.if_modified == "y":
1208                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1209
1210 elif vcs == "hg":
1211         with repoman_popen("hg status --no-status --modified .") as f:
1212                 mychanged = f.readlines()
1213         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1214         with repoman_popen("hg status --no-status --added .") as f:
1215                 mynew = f.readlines()
1216         mynew = ["./" + elem.rstrip() for elem in mynew]
1217         if options.if_modified == "y":
1218                 with repoman_popen("hg status --no-status --removed .") as f:
1219                         myremoved = f.readlines()
1220                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1221
1222 if vcs:
1223         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1224         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1225         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1226                 if os.path.basename(x) == "ChangeLog")
1227
1228 def vcs_new_changed(relative_path):
1229         for x in chain(mychanged, mynew):
1230                 if x == relative_path:
1231                         return True
1232         return False
1233
1234 have_pmasked = False
1235 have_dev_keywords = False
1236 dofail = 0
1237
1238 # NOTE: match-all caches are not shared due to potential
1239 # differences between profiles in _get_implicit_iuse.
1240 arch_caches = {}
1241 arch_xmatch_caches = {}
1242 shared_xmatch_caches = {"cp-list":{}}
1243
1244 include_arches = None
1245 if options.include_arches:
1246         include_arches = set()
1247         include_arches.update(*[x.split() for x in options.include_arches])
1248
1249 # Disable the "ebuild.notadded" check when not in commit mode and
1250 # running `svn status` in every package dir will be too expensive.
1251
1252 check_ebuild_notadded = not \
1253         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1254
1255 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1256 thirdpartymirrors = {}
1257 for k, v in repoman_settings.thirdpartymirrors().items():
1258         for v in v:
1259                 if not v.endswith("/"):
1260                         v += "/"
1261                 thirdpartymirrors[v] = k
1262
1263 class _XMLParser(xml.etree.ElementTree.XMLParser):
1264
1265         def __init__(self, data, **kwargs):
1266                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1267                 self._portage_data = data
1268                 if hasattr(self, 'parser'):
1269                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1270                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1271                         self._base_StartDoctypeDeclHandler = \
1272                                 self.parser.StartDoctypeDeclHandler
1273                         self.parser.StartDoctypeDeclHandler = \
1274                                 self._portage_StartDoctypeDeclHandler
1275
1276         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1277                 if self._base_XmlDeclHandler is not None:
1278                         self._base_XmlDeclHandler(version, encoding, standalone)
1279                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1280
1281         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1282                 has_internal_subset):
1283                 if self._base_StartDoctypeDeclHandler is not None:
1284                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1285                                 has_internal_subset)
1286                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1287
1288 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1289         """
1290         Implements doctype() as required to avoid deprecation warnings with
1291         >=python-2.7.
1292         """
1293         def doctype(self, name, pubid, system):
1294                 pass
1295
1296 try:
1297         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1298 except (EnvironmentError, ParseError, PermissionDenied) as e:
1299         err(str(e))
1300 except FileNotFound:
1301         # TODO: Download as we do for metadata.dtd, but add a way to
1302         # disable for non-gentoo repoman users who may not have herds.
1303         herd_base = None
1304
1305 effective_scanlist = scanlist
1306 if options.if_modified == "y":
1307         effective_scanlist = sorted(vcs_files_to_cps(
1308                 chain(mychanged, mynew, myremoved)))
1309
1310 for x in effective_scanlist:
1311         #ebuilds and digests added to cvs respectively.
1312         logging.info("checking package %s" % x)
1313         # save memory by discarding xmatch caches from previous package(s)
1314         arch_xmatch_caches.clear()
1315         eadded = []
1316         catdir, pkgdir = x.split("/")
1317         checkdir = repodir + "/" + x
1318         checkdir_relative = ""
1319         if repolevel < 3:
1320                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1321         if repolevel < 2:
1322                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1323         checkdir_relative = os.path.join(".", checkdir_relative)
1324         generated_manifest = False
1325
1326         if options.mode == "manifest" or \
1327           (options.mode != 'manifest-check' and options.digest == 'y') or \
1328           options.mode in ('commit', 'fix') and not options.pretend:
1329                 auto_assumed = set()
1330                 fetchlist_dict = portage.FetchlistDict(checkdir,
1331                         repoman_settings, portdb)
1332                 if options.mode == 'manifest' and options.force:
1333                         portage._doebuild_manifest_exempt_depend += 1
1334                         try:
1335                                 distdir = repoman_settings['DISTDIR']
1336                                 mf = repoman_settings.repositories.get_repo_for_location(
1337                                         os.path.dirname(os.path.dirname(checkdir)))
1338                                 mf = mf.load_manifest(checkdir, distdir,
1339                                         fetchlist_dict=fetchlist_dict)
1340                                 mf.create(requiredDistfiles=None,
1341                                         assumeDistHashesAlways=True)
1342                                 for distfiles in fetchlist_dict.values():
1343                                         for distfile in distfiles:
1344                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1345                                                         mf.fhashdict['DIST'].pop(distfile, None)
1346                                                 else:
1347                                                         auto_assumed.add(distfile)
1348                                 mf.write()
1349                         finally:
1350                                 portage._doebuild_manifest_exempt_depend -= 1
1351
1352                 repoman_settings["O"] = checkdir
1353                 try:
1354                         generated_manifest = digestgen(
1355                                 mysettings=repoman_settings, myportdb=portdb)
1356                 except portage.exception.PermissionDenied as e:
1357                         generated_manifest = False
1358                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1359                                 level=logging.ERROR, noiselevel=-1)
1360
1361                 if not generated_manifest:
1362                         print("Unable to generate manifest.")
1363                         dofail = 1
1364
1365                 if options.mode == "manifest":
1366                         if not dofail and options.force and auto_assumed and \
1367                                 'assume-digests' in repoman_settings.features:
1368                                 # Show which digests were assumed despite the --force option
1369                                 # being given. This output will already have been shown by
1370                                 # digestgen() if assume-digests is not enabled, so only show
1371                                 # it here if assume-digests is enabled.
1372                                 pkgs = list(fetchlist_dict)
1373                                 pkgs.sort()
1374                                 portage.writemsg_stdout("  digest.assumed" + \
1375                                         portage.output.colorize("WARN",
1376                                         str(len(auto_assumed)).rjust(18)) + "\n")
1377                                 for cpv in pkgs:
1378                                         fetchmap = fetchlist_dict[cpv]
1379                                         pf = portage.catsplit(cpv)[1]
1380                                         for distfile in sorted(fetchmap):
1381                                                 if distfile in auto_assumed:
1382                                                         portage.writemsg_stdout(
1383                                                                 "   %s::%s\n" % (pf, distfile))
1384                         continue
1385                 elif dofail:
1386                         sys.exit(1)
1387
1388         if not generated_manifest:
1389                 repoman_settings['O'] = checkdir
1390                 repoman_settings['PORTAGE_QUIET'] = '1'
1391                 if not portage.digestcheck([], repoman_settings, strict=1):
1392                         stats["manifest.bad"] += 1
1393                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1394                 repoman_settings.pop('PORTAGE_QUIET', None)
1395
1396         if options.mode == 'manifest-check':
1397                 continue
1398
1399         checkdirlist = os.listdir(checkdir)
1400         ebuildlist = []
1401         pkgs = {}
1402         allvalid = True
1403         for y in checkdirlist:
1404                 if (y in no_exec or y.endswith(".ebuild")) and \
1405                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1406                                 stats["file.executable"] += 1
1407                                 fails["file.executable"].append(os.path.join(checkdir, y))
1408                 if y.endswith(".ebuild"):
1409                         pf = y[:-7]
1410                         ebuildlist.append(pf)
1411                         cpv = "%s/%s" % (catdir, pf)
1412                         try:
1413                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1414                         except KeyError:
1415                                 allvalid = False
1416                                 stats["ebuild.syntax"] += 1
1417                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1418                                 continue
1419                         except IOError:
1420                                 allvalid = False
1421                                 stats["ebuild.output"] += 1
1422                                 fails["ebuild.output"].append(os.path.join(x, y))
1423                                 continue
1424                         if not portage.eapi_is_supported(myaux["EAPI"]):
1425                                 allvalid = False
1426                                 stats["EAPI.unsupported"] += 1
1427                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1428                                 continue
1429                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1430                                 root_config=root_config, type_name="ebuild")
1431
1432         slot_keywords = {}
1433
1434         if len(pkgs) != len(ebuildlist):
1435                 # If we can't access all the metadata then it's totally unsafe to
1436                 # commit since there's no way to generate a correct Manifest.
1437                 # Do not try to do any more QA checks on this package since missing
1438                 # metadata leads to false positives for several checks, and false
1439                 # positives confuse users.
1440                 can_force = False
1441                 continue
1442
1443         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1444         ebuildlist = sorted(pkgs.values())
1445         ebuildlist = [pkg.pf for pkg in ebuildlist]
1446
1447         for y in checkdirlist:
1448                 index = repo_config.find_invalid_path_char(y)
1449                 if index != -1:
1450                         y_relative = os.path.join(checkdir_relative, y)
1451                         if vcs is not None and not vcs_new_changed(y_relative):
1452                                 # If the file isn't in the VCS new or changed set, then
1453                                 # assume that it's an irrelevant temporary file (Manifest
1454                                 # entries are not generated for file names containing
1455                                 # prohibited characters). See bug #406877.
1456                                 index = -1
1457                 if index != -1:
1458                         stats["file.name"] += 1
1459                         fails["file.name"].append("%s/%s: char '%s'" % \
1460                                 (checkdir, y, y[index]))
1461
1462                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1463                         continue
1464                 f = None
1465                 try:
1466                         line = 1
1467                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1468                                 encoding=_encodings['fs'], errors='strict'),
1469                                 mode='r', encoding=_encodings['repo.content'])
1470                         for l in f:
1471                                 line += 1
1472                 except UnicodeDecodeError as ue:
1473                         stats["file.UTF8"] += 1
1474                         s = ue.object[:ue.start]
1475                         l2 = s.count("\n")
1476                         line += l2
1477                         if l2 != 0:
1478                                 s = s[s.rfind("\n") + 1:]
1479                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1480                 finally:
1481                         if f is not None:
1482                                 f.close()
1483
1484         if vcs in ("git", "hg") and check_ebuild_notadded:
1485                 if vcs == "git":
1486                         myf = repoman_popen("git ls-files --others %s" % \
1487                                 (portage._shell_quote(checkdir_relative),))
1488                 if vcs == "hg":
1489                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1490                                 (portage._shell_quote(checkdir_relative),))
1491                 for l in myf:
1492                         if l[:-1][-7:] == ".ebuild":
1493                                 stats["ebuild.notadded"] += 1
1494                                 fails["ebuild.notadded"].append(
1495                                         os.path.join(x, os.path.basename(l[:-1])))
1496                 myf.close()
1497
1498         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1499                 try:
1500                         if vcs == "cvs":
1501                                 myf = open(checkdir + "/CVS/Entries", "r")
1502                         if vcs == "svn":
1503                                 myf = repoman_popen("svn status --depth=files --verbose " +
1504                                         portage._shell_quote(checkdir))
1505                         if vcs == "bzr":
1506                                 myf = repoman_popen("bzr ls -v --kind=file " +
1507                                         portage._shell_quote(checkdir))
1508                         myl = myf.readlines()
1509                         myf.close()
1510                         for l in myl:
1511                                 if vcs == "cvs":
1512                                         if l[0] != "/":
1513                                                 continue
1514                                         splitl = l[1:].split("/")
1515                                         if not len(splitl):
1516                                                 continue
1517                                         if splitl[0][-7:] == ".ebuild":
1518                                                 eadded.append(splitl[0][:-7])
1519                                 if vcs == "svn":
1520                                         if l[:1] == "?":
1521                                                 continue
1522                                         if l[:7] == '      >':
1523                                                 # tree conflict, new in subversion 1.6
1524                                                 continue
1525                                         l = l.split()[-1]
1526                                         if l[-7:] == ".ebuild":
1527                                                 eadded.append(os.path.basename(l[:-7]))
1528                                 if vcs == "bzr":
1529                                         if l[1:2] == "?":
1530                                                 continue
1531                                         l = l.split()[-1]
1532                                         if l[-7:] == ".ebuild":
1533                                                 eadded.append(os.path.basename(l[:-7]))
1534                         if vcs == "svn":
1535                                 myf = repoman_popen("svn status " +
1536                                         portage._shell_quote(checkdir))
1537                                 myl = myf.readlines()
1538                                 myf.close()
1539                                 for l in myl:
1540                                         if l[0] == "A":
1541                                                 l = l.rstrip().split(' ')[-1]
1542                                                 if l[-7:] == ".ebuild":
1543                                                         eadded.append(os.path.basename(l[:-7]))
1544                 except IOError:
1545                         if vcs == "cvs":
1546                                 stats["CVS/Entries.IO_error"] += 1
1547                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1548                         else:
1549                                 raise
1550                         continue
1551
1552         mf = repoman_settings.repositories.get_repo_for_location(
1553                 os.path.dirname(os.path.dirname(checkdir)))
1554         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1555         mydigests = mf.getTypeDigests("DIST")
1556
1557         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1558         myfiles_all = []
1559         src_uri_error = False
1560         for mykey in fetchlist_dict:
1561                 try:
1562                         myfiles_all.extend(fetchlist_dict[mykey])
1563                 except portage.exception.InvalidDependString as e:
1564                         src_uri_error = True
1565                         try:
1566                                 portdb.aux_get(mykey, ["SRC_URI"])
1567                         except KeyError:
1568                                 # This will be reported as an "ebuild.syntax" error.
1569                                 pass
1570                         else:
1571                                 stats["SRC_URI.syntax"] += 1
1572                                 fails["SRC_URI.syntax"].append(
1573                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1574         del fetchlist_dict
1575         if not src_uri_error:
1576                 # This test can produce false positives if SRC_URI could not
1577                 # be parsed for one or more ebuilds. There's no point in
1578                 # producing a false error here since the root cause will
1579                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1580                 # or "ebuild.sytax".
1581                 myfiles_all = set(myfiles_all)
1582                 for entry in mydigests:
1583                         if entry not in myfiles_all:
1584                                 stats["digest.unused"] += 1
1585                                 fails["digest.unused"].append(checkdir + "::" + entry)
1586                 for entry in myfiles_all:
1587                         if entry not in mydigests:
1588                                 stats["digest.missing"] += 1
1589                                 fails["digest.missing"].append(checkdir + "::" + entry)
1590         del myfiles_all
1591
1592         if os.path.exists(checkdir + "/files"):
1593                 filesdirlist = os.listdir(checkdir + "/files")
1594
1595                 # recurse through files directory
1596                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1597                 while filesdirlist:
1598                         y = filesdirlist.pop(0)
1599                         relative_path = os.path.join(x, "files", y)
1600                         full_path = os.path.join(repodir, relative_path)
1601                         try:
1602                                 mystat = os.stat(full_path)
1603                         except OSError as oe:
1604                                 if oe.errno == 2:
1605                                         # don't worry about it.  it likely was removed via fix above.
1606                                         continue
1607                                 else:
1608                                         raise oe
1609                         if S_ISDIR(mystat.st_mode):
1610                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1611                                 if y == "CVS" or y == ".svn":
1612                                         continue
1613                                 for z in os.listdir(checkdir + "/files/" + y):
1614                                         if z == "CVS" or z == ".svn":
1615                                                 continue
1616                                         filesdirlist.append(y + "/" + z)
1617                         # Current policy is no files over 20 KiB, these are the checks. File size between
1618                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1619                         elif mystat.st_size > 61440:
1620                                 stats["file.size.fatal"] += 1
1621                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1622                         elif mystat.st_size > 20480:
1623                                 stats["file.size"] += 1
1624                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1625
1626                         index = repo_config.find_invalid_path_char(y)
1627                         if index != -1:
1628                                 y_relative = os.path.join(checkdir_relative, "files", y)
1629                                 if vcs is not None and not vcs_new_changed(y_relative):
1630                                         # If the file isn't in the VCS new or changed set, then
1631                                         # assume that it's an irrelevant temporary file (Manifest
1632                                         # entries are not generated for file names containing
1633                                         # prohibited characters). See bug #406877.
1634                                         index = -1
1635                         if index != -1:
1636                                 stats["file.name"] += 1
1637                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1638                                         (checkdir, y, y[index]))
1639         del mydigests
1640
1641         if check_changelog and "ChangeLog" not in checkdirlist:
1642                 stats["changelog.missing"] += 1
1643                 fails["changelog.missing"].append(x + "/ChangeLog")
1644
1645         musedict = {}
1646         # metadata.xml file check
1647         if "metadata.xml" not in checkdirlist:
1648                 stats["metadata.missing"] += 1
1649                 fails["metadata.missing"].append(x + "/metadata.xml")
1650         # metadata.xml parse check
1651         else:
1652                 metadata_bad = False
1653                 xml_info = {}
1654                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1655
1656                 # read metadata.xml into memory
1657                 try:
1658                         _metadata_xml = xml.etree.ElementTree.parse(
1659                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1660                                 encoding=_encodings['fs'], errors='strict'),
1661                                 parser=xml_parser)
1662                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1663                         metadata_bad = True
1664                         stats["metadata.bad"] += 1
1665                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1666                         del e
1667                 else:
1668                         if not hasattr(xml_parser, 'parser') or \
1669                                 sys.hexversion < 0x2070000 or \
1670                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1671                                 # doctype is not parsed with python 2.6 or 3.1
1672                                 pass
1673                         else:
1674                                 if "XML_DECLARATION" not in xml_info:
1675                                         stats["metadata.bad"] += 1
1676                                         fails["metadata.bad"].append("%s/metadata.xml: "
1677                                                 "xml declaration is missing on first line, "
1678                                                 "should be '%s'" % (x, metadata_xml_declaration))
1679                                 else:
1680                                         xml_version, xml_encoding, xml_standalone = \
1681                                                 xml_info["XML_DECLARATION"]
1682                                         if xml_encoding is None or \
1683                                                 xml_encoding.upper() != metadata_xml_encoding:
1684                                                 stats["metadata.bad"] += 1
1685                                                 if xml_encoding is None:
1686                                                         encoding_problem = "but it is undefined"
1687                                                 else:
1688                                                         encoding_problem = "not '%s'" % xml_encoding
1689                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1690                                                         "xml declaration encoding should be '%s', %s" %
1691                                                         (x, metadata_xml_encoding, encoding_problem))
1692
1693                                 if "DOCTYPE" not in xml_info:
1694                                         metadata_bad = True
1695                                         stats["metadata.bad"] += 1
1696                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1697                                                 "DOCTYPE is missing"))
1698                                 else:
1699                                         doctype_name, doctype_system, doctype_pubid = \
1700                                                 xml_info["DOCTYPE"]
1701                                         if doctype_system != metadata_dtd_uri:
1702                                                 stats["metadata.bad"] += 1
1703                                                 if doctype_system is None:
1704                                                         system_problem = "but it is undefined"
1705                                                 else:
1706                                                         system_problem = "not '%s'" % doctype_system
1707                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1708                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1709                                                         (x, metadata_dtd_uri, system_problem))
1710
1711                                         if doctype_name != metadata_doctype_name:
1712                                                 stats["metadata.bad"] += 1
1713                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1714                                                         "DOCTYPE: name should be '%s', not '%s'" %
1715                                                         (x, metadata_doctype_name, doctype_name))
1716
1717                         # load USE flags from metadata.xml
1718                         try:
1719                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1720                         except portage.exception.ParseError as e:
1721                                 metadata_bad = True
1722                                 stats["metadata.bad"] += 1
1723                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1724                         else:
1725                                 for atom in chain(*musedict.values()):
1726                                         if atom is None:
1727                                                 continue
1728                                         try:
1729                                                 atom = Atom(atom)
1730                                         except InvalidAtom as e:
1731                                                 stats["metadata.bad"] += 1
1732                                                 fails["metadata.bad"].append(
1733                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1734                                         else:
1735                                                 if atom.cp != x:
1736                                                         stats["metadata.bad"] += 1
1737                                                         fails["metadata.bad"].append(
1738                                                                 ("%s/metadata.xml: Atom contains "
1739                                                                 "unexpected cat/pn: %s") % (x, atom))
1740
1741                         # Run other metadata.xml checkers
1742                         try:
1743                                 utilities.check_metadata(_metadata_xml, herd_base)
1744                         except (utilities.UnknownHerdsError, ) as e:
1745                                 metadata_bad = True
1746                                 stats["metadata.bad"] += 1
1747                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1748                                 del e
1749
1750                 #Only carry out if in package directory or check forced
1751                 if xmllint_capable and not metadata_bad:
1752                         # xmlint can produce garbage output even on success, so only dump
1753                         # the ouput when it fails.
1754                         st, out = repoman_getstatusoutput(
1755                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1756                                  (portage._shell_quote(metadata_dtd),
1757                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1758                         if st != os.EX_OK:
1759                                 print(red("!!!") + " metadata.xml is invalid:")
1760                                 for z in out.splitlines():
1761                                         print(red("!!! ") + z)
1762                                 stats["metadata.bad"] += 1
1763                                 fails["metadata.bad"].append(x + "/metadata.xml")
1764
1765                 del metadata_bad
1766         muselist = frozenset(musedict)
1767
1768         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1769         changelog_modified = changelog_path in modified_changelogs
1770
1771         # detect unused local USE-descriptions
1772         used_useflags = set()
1773
1774         for y in ebuildlist:
1775                 relative_path = os.path.join(x, y + ".ebuild")
1776                 full_path = os.path.join(repodir, relative_path)
1777                 ebuild_path = y + ".ebuild"
1778                 if repolevel < 3:
1779                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1780                 if repolevel < 2:
1781                         ebuild_path = os.path.join(catdir, ebuild_path)
1782                 ebuild_path = os.path.join(".", ebuild_path)
1783                 if check_changelog and not changelog_modified \
1784                         and ebuild_path in new_ebuilds:
1785                         stats['changelog.ebuildadded'] += 1
1786                         fails['changelog.ebuildadded'].append(relative_path)
1787
1788                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1789                         #ebuild not added to vcs
1790                         stats["ebuild.notadded"] += 1
1791                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1792                 myesplit = portage.pkgsplit(y)
1793                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1794                             or pv_toolong_re.search(myesplit[1]) \
1795                             or pv_toolong_re.search(myesplit[2]):
1796                         stats["ebuild.invalidname"] += 1
1797                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1798                         continue
1799                 elif myesplit[0] != pkgdir:
1800                         print(pkgdir, myesplit[0])
1801                         stats["ebuild.namenomatch"] += 1
1802                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1803                         continue
1804
1805                 pkg = pkgs[y]
1806
1807                 if pkg.invalid:
1808                         allvalid = False
1809                         for k, msgs in pkg.invalid.items():
1810                                 for msg in msgs:
1811                                         stats[k] += 1
1812                                         fails[k].append("%s: %s" % (relative_path, msg))
1813                         continue
1814
1815                 myaux = pkg._metadata
1816                 eapi = myaux["EAPI"]
1817                 inherited = pkg.inherited
1818                 live_ebuild = live_eclasses.intersection(inherited)
1819
1820                 if repo_config.eapi_is_banned(eapi):
1821                         stats["repo.eapi.banned"] += 1
1822                         fails["repo.eapi.banned"].append(
1823                                 "%s: %s" % (relative_path, eapi))
1824
1825                 elif repo_config.eapi_is_deprecated(eapi):
1826                         stats["repo.eapi.deprecated"] += 1
1827                         fails["repo.eapi.deprecated"].append(
1828                                 "%s: %s" % (relative_path, eapi))
1829
1830                 for k, v in myaux.items():
1831                         if not isinstance(v, basestring):
1832                                 continue
1833                         m = non_ascii_re.search(v)
1834                         if m is not None:
1835                                 stats["variable.invalidchar"] += 1
1836                                 fails["variable.invalidchar"].append(
1837                                         ("%s: %s variable contains non-ASCII " + \
1838                                         "character at position %s") % \
1839                                         (relative_path, k, m.start() + 1))
1840
1841                 if not src_uri_error:
1842                         # Check that URIs don't reference a server from thirdpartymirrors.
1843                         for uri in portage.dep.use_reduce( \
1844                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1845                                 contains_mirror = False
1846                                 for mirror, mirror_alias in thirdpartymirrors.items():
1847                                         if uri.startswith(mirror):
1848                                                 contains_mirror = True
1849                                                 break
1850                                 if not contains_mirror:
1851                                         continue
1852
1853                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1854                                 stats["SRC_URI.mirror"] += 1
1855                                 fails["SRC_URI.mirror"].append(
1856                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1857                                         (relative_path, mirror, new_uri))
1858
1859                 if myaux.get("PROVIDE"):
1860                         stats["virtual.oldstyle"] += 1
1861                         fails["virtual.oldstyle"].append(relative_path)
1862
1863                 for pos, missing_var in enumerate(missingvars):
1864                         if not myaux.get(missing_var):
1865                                 if catdir == "virtual" and \
1866                                         missing_var in ("HOMEPAGE", "LICENSE"):
1867                                         continue
1868                                 if live_ebuild and missing_var == "KEYWORDS":
1869                                         continue
1870                                 myqakey = missingvars[pos] + ".missing"
1871                                 stats[myqakey] += 1
1872                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1873
1874                 if catdir == "virtual":
1875                         for var in ("HOMEPAGE", "LICENSE"):
1876                                 if myaux.get(var):
1877                                         myqakey = var + ".virtual"
1878                                         stats[myqakey] += 1
1879                                         fails[myqakey].append(relative_path)
1880
1881                 # 14 is the length of DESCRIPTION=""
1882                 if len(myaux['DESCRIPTION']) > max_desc_len:
1883                         stats['DESCRIPTION.toolong'] += 1
1884                         fails['DESCRIPTION.toolong'].append(
1885                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1886                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1887
1888                 keywords = myaux["KEYWORDS"].split()
1889                 stable_keywords = []
1890                 for keyword in keywords:
1891                         if not keyword.startswith("~") and \
1892                                 not keyword.startswith("-"):
1893                                 stable_keywords.append(keyword)
1894                 if stable_keywords:
1895                         if ebuild_path in new_ebuilds and catdir != "virtual":
1896                                 stable_keywords.sort()
1897                                 stats["KEYWORDS.stable"] += 1
1898                                 fails["KEYWORDS.stable"].append(
1899                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1900                                                 " ".join(stable_keywords))
1901
1902                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1903                         if not kw.startswith("-"))
1904
1905                 previous_keywords = slot_keywords.get(pkg.slot)
1906                 if previous_keywords is None:
1907                         slot_keywords[pkg.slot] = set()
1908                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1909                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1910                         if dropped_keywords:
1911                                 stats["KEYWORDS.dropped"] += 1
1912                                 fails["KEYWORDS.dropped"].append(
1913                                         relative_path + ": %s" % \
1914                                         " ".join(sorted(dropped_keywords)))
1915
1916                 slot_keywords[pkg.slot].update(ebuild_archs)
1917
1918                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1919                 if "-*" in keywords:
1920                         haskeyword = False
1921                         for kw in keywords:
1922                                 if kw[0] == "~":
1923                                         kw = kw[1:]
1924                                 if kw in kwlist:
1925                                         haskeyword = True
1926                         if not haskeyword:
1927                                 stats["KEYWORDS.stupid"] += 1
1928                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1929
1930                 """
1931                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1932                 not be allowed to be marked stable
1933                 """
1934                 if live_ebuild and repo_config.name == "gentoo":
1935                         bad_stable_keywords = []
1936                         for keyword in keywords:
1937                                 if not keyword.startswith("~") and \
1938                                         not keyword.startswith("-"):
1939                                         bad_stable_keywords.append(keyword)
1940                                 del keyword
1941                         if bad_stable_keywords:
1942                                 stats["LIVEVCS.stable"] += 1
1943                                 fails["LIVEVCS.stable"].append(
1944                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1945                                                 bad_stable_keywords)
1946                         del bad_stable_keywords
1947
1948                         if keywords and not has_global_mask(pkg):
1949                                 stats["LIVEVCS.unmasked"] += 1
1950                                 fails["LIVEVCS.unmasked"].append(relative_path)
1951
1952                 if options.ignore_arches:
1953                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1954                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1955                 else:
1956                         arches = set()
1957                         for keyword in keywords:
1958                                 if keyword[0] == "-":
1959                                         continue
1960                                 elif keyword[0] == "~":
1961                                         arch = keyword[1:]
1962                                         if arch == "*":
1963                                                 for expanded_arch in profiles:
1964                                                         if expanded_arch == "**":
1965                                                                 continue
1966                                                         arches.add((keyword, expanded_arch,
1967                                                                 (expanded_arch, "~" + expanded_arch)))
1968                                         else:
1969                                                 arches.add((keyword, arch, (arch, keyword)))
1970                                 else:
1971                                         if keyword == "*":
1972                                                 for expanded_arch in profiles:
1973                                                         if expanded_arch == "**":
1974                                                                 continue
1975                                                         arches.add((keyword, expanded_arch,
1976                                                                 (expanded_arch,)))
1977                                         else:
1978                                                 arches.add((keyword, keyword, (keyword,)))
1979                         if not arches:
1980                                 # Use an empty profile for checking dependencies of
1981                                 # packages that have empty KEYWORDS.
1982                                 arches.add(('**', '**', ('**',)))
1983
1984                 unknown_pkgs = set()
1985                 baddepsyntax = False
1986                 badlicsyntax = False
1987                 badprovsyntax = False
1988                 catpkg = catdir + "/" + y
1989
1990                 inherited_java_eclass = "java-pkg-2" in inherited or \
1991                         "java-pkg-opt-2" in inherited
1992                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1993                 operator_tokens = set(["||", "(", ")"])
1994                 type_list, badsyntax = [], []
1995                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
1996                         mydepstr = myaux[mytype]
1997
1998                         buildtime = mytype in Package._buildtime_keys
1999                         runtime = mytype in Package._runtime_keys
2000                         token_class = None
2001                         if mytype.endswith("DEPEND"):
2002                                 token_class = portage.dep.Atom
2003
2004                         try:
2005                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
2006                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
2007                         except portage.exception.InvalidDependString as e:
2008                                 atoms = None
2009                                 badsyntax.append(str(e))
2010
2011                         if atoms and mytype.endswith("DEPEND"):
2012                                 if runtime and \
2013                                         "test?" in mydepstr.split():
2014                                         stats[mytype + '.suspect'] += 1
2015                                         fails[mytype + '.suspect'].append(relative_path + \
2016                                                 ": 'test?' USE conditional in %s" % mytype)
2017
2018                                 for atom in atoms:
2019                                         if atom == "||":
2020                                                 continue
2021
2022                                         # Skip dependency.unknown for blockers, so that we
2023                                         # don't encourage people to remove necessary blockers,
2024                                         # as discussed in bug #382407.
2025                                         if atom.blocker is None and \
2026                                                 not portdb.xmatch("match-all", atom) and \
2027                                                 not atom.cp.startswith("virtual/"):
2028                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2029
2030                                         is_blocker = atom.blocker
2031
2032                                         if catdir != "virtual":
2033                                                 if not is_blocker and \
2034                                                         atom.cp in suspect_virtual:
2035                                                         stats['virtual.suspect'] += 1
2036                                                         fails['virtual.suspect'].append(
2037                                                                 relative_path +
2038                                                                 ": %s: consider using '%s' instead of '%s'" %
2039                                                                 (mytype, suspect_virtual[atom.cp], atom))
2040
2041                                         if buildtime and \
2042                                                 not is_blocker and \
2043                                                 not inherited_java_eclass and \
2044                                                 atom.cp == "virtual/jdk":
2045                                                 stats['java.eclassesnotused'] += 1
2046                                                 fails['java.eclassesnotused'].append(relative_path)
2047                                         elif buildtime and \
2048                                                 not is_blocker and \
2049                                                 not inherited_wxwidgets_eclass and \
2050                                                 atom.cp == "x11-libs/wxGTK":
2051                                                 stats['wxwidgets.eclassnotused'] += 1
2052                                                 fails['wxwidgets.eclassnotused'].append(
2053                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2054                                                         " without inheriting wxwidgets.eclass") % mytype)
2055                                         elif runtime:
2056                                                 if not is_blocker and \
2057                                                         atom.cp in suspect_rdepend:
2058                                                         stats[mytype + '.suspect'] += 1
2059                                                         fails[mytype + '.suspect'].append(
2060                                                                 relative_path + ": '%s'" % atom)
2061
2062                                         if atom.operator == "~" and \
2063                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2064                                                 qacat = 'dependency.badtilde'
2065                                                 stats[qacat] += 1
2066                                                 fails[qacat].append(
2067                                                         (relative_path + ": %s uses the ~ operator"
2068                                                          " with a non-zero revision:" + \
2069                                                          " '%s'") % (mytype, atom))
2070
2071                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2072
2073                 for m, b in zip(type_list, badsyntax):
2074                         if m.endswith("DEPEND"):
2075                                 qacat = "dependency.syntax"
2076                         else:
2077                                 qacat = m + ".syntax"
2078                         stats[qacat] += 1
2079                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2080
2081                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2082                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2083                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2084                 badlicsyntax = badlicsyntax > 0
2085                 badprovsyntax = badprovsyntax > 0
2086
2087                 # uselist checks - global
2088                 myuse = []
2089                 default_use = []
2090                 for myflag in myaux["IUSE"].split():
2091                         flag_name = myflag.lstrip("+-")
2092                         used_useflags.add(flag_name)
2093                         if myflag != flag_name:
2094                                 default_use.append(myflag)
2095                         if flag_name not in uselist:
2096                                 myuse.append(flag_name)
2097
2098                 # uselist checks - metadata
2099                 for mypos in range(len(myuse)-1, -1, -1):
2100                         if myuse[mypos] and (myuse[mypos] in muselist):
2101                                 del myuse[mypos]
2102
2103                 if default_use and not eapi_has_iuse_defaults(eapi):
2104                         for myflag in default_use:
2105                                 stats['EAPI.incompatible'] += 1
2106                                 fails['EAPI.incompatible'].append(
2107                                         (relative_path + ": IUSE defaults" + \
2108                                         " not supported with EAPI='%s':" + \
2109                                         " '%s'") % (eapi, myflag))
2110
2111                 for mypos in range(len(myuse)):
2112                         stats["IUSE.invalid"] += 1
2113                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2114
2115                 # license checks
2116                 if not badlicsyntax:
2117                         # Parse the LICENSE variable, remove USE conditions and
2118                         # flatten it.
2119                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2120                         # Check each entry to ensure that it exists in PORTDIR's
2121                         # license directory.
2122                         for lic in licenses:
2123                                 # Need to check for "||" manually as no portage
2124                                 # function will remove it without removing values.
2125                                 if lic not in liclist and lic != "||":
2126                                         stats["LICENSE.invalid"] += 1
2127                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2128                                 elif lic in liclist_deprecated:
2129                                         stats["LICENSE.deprecated"] += 1
2130                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2131
2132                 #keyword checks
2133                 myuse = myaux["KEYWORDS"].split()
2134                 for mykey in myuse:
2135                         if mykey not in ("-*", "*", "~*"):
2136                                 myskey = mykey
2137                                 if myskey[:1] == "-":
2138                                         myskey = myskey[1:]
2139                                 if myskey[:1] == "~":
2140                                         myskey = myskey[1:]
2141                                 if myskey not in kwlist:
2142                                         stats["KEYWORDS.invalid"] += 1
2143                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2144                                 elif myskey not in profiles:
2145                                         stats["KEYWORDS.invalid"] += 1
2146                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2147
2148                 #restrict checks
2149                 myrestrict = None
2150                 try:
2151                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2152                 except portage.exception.InvalidDependString as e:
2153                         stats["RESTRICT.syntax"] += 1
2154                         fails["RESTRICT.syntax"].append(
2155                                 "%s: RESTRICT: %s" % (relative_path, e))
2156                         del e
2157                 if myrestrict:
2158                         myrestrict = set(myrestrict)
2159                         mybadrestrict = myrestrict.difference(valid_restrict)
2160                         if mybadrestrict:
2161                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2162                                 for mybad in mybadrestrict:
2163                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2164                 #REQUIRED_USE check
2165                 required_use = myaux["REQUIRED_USE"]
2166                 if required_use:
2167                         if not eapi_has_required_use(eapi):
2168                                 stats['EAPI.incompatible'] += 1
2169                                 fails['EAPI.incompatible'].append(
2170                                         relative_path + ": REQUIRED_USE" + \
2171                                         " not supported with EAPI='%s'" % (eapi,))
2172                         try:
2173                                 portage.dep.check_required_use(required_use, (),
2174                                         pkg.iuse.is_valid_flag, eapi=eapi)
2175                         except portage.exception.InvalidDependString as e:
2176                                 stats["REQUIRED_USE.syntax"] += 1
2177                                 fails["REQUIRED_USE.syntax"].append(
2178                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2179                                 del e
2180
2181                 # Syntax Checks
2182                 relative_path = os.path.join(x, y + ".ebuild")
2183                 full_path = os.path.join(repodir, relative_path)
2184                 if not vcs_preserves_mtime:
2185                         if ebuild_path not in new_ebuilds and \
2186                                 ebuild_path not in modified_ebuilds:
2187                                 pkg.mtime = None
2188                 try:
2189                         # All ebuilds should have utf_8 encoding.
2190                         f = io.open(_unicode_encode(full_path,
2191                                 encoding=_encodings['fs'], errors='strict'),
2192                                 mode='r', encoding=_encodings['repo.content'])
2193                         try:
2194                                 for check_name, e in run_checks(f, pkg):
2195                                         stats[check_name] += 1
2196                                         fails[check_name].append(relative_path + ': %s' % e)
2197                         finally:
2198                                 f.close()
2199                 except UnicodeDecodeError:
2200                         # A file.UTF8 failure will have already been recorded above.
2201                         pass
2202
2203                 if options.force:
2204                         # The dep_check() calls are the most expensive QA test. If --force
2205                         # is enabled, there's no point in wasting time on these since the
2206                         # user is intent on forcing the commit anyway.
2207                         continue
2208
2209                 relevant_profiles = []
2210                 for keyword, arch, groups in arches:
2211                         if arch not in profiles:
2212                                 # A missing profile will create an error further down
2213                                 # during the KEYWORDS verification.
2214                                 continue
2215
2216                         if include_arches is not None:
2217                                 if arch not in include_arches:
2218                                         continue
2219
2220                         relevant_profiles.extend((keyword, groups, prof)
2221                                 for prof in profiles[arch])
2222
2223                 def sort_key(item):
2224                         return item[2].sub_path
2225
2226                 relevant_profiles.sort(key=sort_key)
2227
2228                 for keyword, groups, prof in relevant_profiles:
2229
2230                                 if prof.status not in ("stable", "dev") or \
2231                                         prof.status == "dev" and not options.include_dev:
2232                                         continue
2233
2234                                 dep_settings = arch_caches.get(prof.sub_path)
2235                                 if dep_settings is None:
2236                                         dep_settings = portage.config(
2237                                                 config_profile_path=prof.abs_path,
2238                                                 config_incrementals=repoman_incrementals,
2239                                                 config_root=config_root,
2240                                                 local_config=False,
2241                                                 _unmatched_removal=options.unmatched_removal,
2242                                                 env=env)
2243                                         dep_settings.categories = repoman_settings.categories
2244                                         if options.without_mask:
2245                                                 dep_settings._mask_manager_obj = \
2246                                                         copy.deepcopy(dep_settings._mask_manager)
2247                                                 dep_settings._mask_manager._pmaskdict.clear()
2248                                         arch_caches[prof.sub_path] = dep_settings
2249
2250                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2251                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2252                                 if xcache is None:
2253                                         portdb.melt()
2254                                         portdb.freeze()
2255                                         xcache = portdb.xcache
2256                                         xcache.update(shared_xmatch_caches)
2257                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2258
2259                                 trees[root]["porttree"].settings = dep_settings
2260                                 portdb.settings = dep_settings
2261                                 portdb.xcache = xcache
2262
2263                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2264                                 # just in case, prevent config.reset() from nuking these.
2265                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2266
2267                                 # This attribute is used in dbapi._match_use() to apply
2268                                 # use.stable.{mask,force} settings based on the stable
2269                                 # status of the parent package. This is required in order
2270                                 # for USE deps of unstable packages to be resolved correctly,
2271                                 # since otherwise use.stable.{mask,force} settings of
2272                                 # dependencies may conflict (see bug #456342).
2273                                 dep_settings._parent_stable = dep_settings._isStable(pkg)
2274
2275                                 # Handle package.use*.{force,mask) calculation, for use
2276                                 # in dep_check.
2277                                 dep_settings.useforce = dep_settings._use_manager.getUseForce(
2278                                         pkg, stable=dep_settings._parent_stable)
2279                                 dep_settings.usemask = dep_settings._use_manager.getUseMask(
2280                                         pkg, stable=dep_settings._parent_stable)
2281
2282                                 if not baddepsyntax:
2283                                         ismasked = not ebuild_archs or \
2284                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2285                                         if ismasked:
2286                                                 if not have_pmasked:
2287                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2288                                                                 pkg.cpv, pkg._metadata))
2289                                                 if options.ignore_masked:
2290                                                         continue
2291                                                 #we are testing deps for a masked package; give it some lee-way
2292                                                 suffix = "masked"
2293                                                 matchmode = "minimum-all"
2294                                         else:
2295                                                 suffix = ""
2296                                                 matchmode = "minimum-visible"
2297
2298                                         if not have_dev_keywords:
2299                                                 have_dev_keywords = \
2300                                                         bool(dev_keywords.intersection(keywords))
2301
2302                                         if prof.status == "dev":
2303                                                 suffix = suffix + "indev"
2304
2305                                         for mytype in Package._dep_keys:
2306
2307                                                 mykey = "dependency.bad" + suffix
2308                                                 myvalue = myaux[mytype]
2309                                                 if not myvalue:
2310                                                         continue
2311
2312                                                 success, atoms = portage.dep_check(myvalue, portdb,
2313                                                         dep_settings, use="all", mode=matchmode,
2314                                                         trees=trees)
2315
2316                                                 if success:
2317                                                         if atoms:
2318
2319                                                                 # Don't bother with dependency.unknown for
2320                                                                 # cases in which *DEPEND.bad is triggered.
2321                                                                 for atom in atoms:
2322                                                                         # dep_check returns all blockers and they
2323                                                                         # aren't counted for *DEPEND.bad, so we
2324                                                                         # ignore them here.
2325                                                                         if not atom.blocker:
2326                                                                                 unknown_pkgs.discard(
2327                                                                                         (mytype, atom.unevaluated_atom))
2328
2329                                                                 if not prof.sub_path:
2330                                                                         # old-style virtuals currently aren't
2331                                                                         # resolvable with empty profile, since
2332                                                                         # 'virtuals' mappings are unavailable
2333                                                                         # (it would be expensive to search
2334                                                                         # for PROVIDE in all ebuilds)
2335                                                                         atoms = [atom for atom in atoms if not \
2336                                                                                 (atom.cp.startswith('virtual/') and \
2337                                                                                 not portdb.cp_list(atom.cp))]
2338
2339                                                                 #we have some unsolvable deps
2340                                                                 #remove ! deps, which always show up as unsatisfiable
2341                                                                 atoms = [str(atom.unevaluated_atom) \
2342                                                                         for atom in atoms if not atom.blocker]
2343
2344                                                                 #if we emptied out our list, continue:
2345                                                                 if not atoms:
2346                                                                         continue
2347                                                                 stats[mykey] += 1
2348                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2349                                                                         (relative_path, mytype, keyword,
2350                                                                         prof, repr(atoms)))
2351                                                 else:
2352                                                         stats[mykey] += 1
2353                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2354                                                                 (relative_path, mytype, keyword,
2355                                                                 prof, repr(atoms)))
2356
2357                 if not baddepsyntax and unknown_pkgs:
2358                         type_map = {}
2359                         for mytype, atom in unknown_pkgs:
2360                                 type_map.setdefault(mytype, set()).add(atom)
2361                         for mytype, atoms in type_map.items():
2362                                 stats["dependency.unknown"] += 1
2363                                 fails["dependency.unknown"].append("%s: %s: %s" %
2364                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2365
2366         # check if there are unused local USE-descriptions in metadata.xml
2367         # (unless there are any invalids, to avoid noise)
2368         if allvalid:
2369                 for myflag in muselist.difference(used_useflags):
2370                         stats["metadata.warning"] += 1
2371                         fails["metadata.warning"].append(
2372                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2373                                 (x, myflag))
2374
2375 if options.if_modified == "y" and len(effective_scanlist) < 1:
2376         logging.warn("--if-modified is enabled, but no modified packages were found!")
2377
2378 if options.mode == "manifest":
2379         sys.exit(dofail)
2380
2381 # dofail will be set to 1 if we have failed in at least one non-warning category
2382 dofail = 0
2383 # dowarn will be set to 1 if we tripped any warnings
2384 dowarn = 0
2385 # dofull will be set if we should print a "repoman full" informational message
2386 dofull = options.mode != 'full'
2387
2388 for x in qacats:
2389         if not stats[x]:
2390                 continue
2391         dowarn = 1
2392         if x not in qawarnings:
2393                 dofail = 1
2394
2395 if dofail or \
2396         (dowarn and not (options.quiet or options.mode == "scan")):
2397         dofull = 0
2398
2399 # Save QA output so that it can be conveniently displayed
2400 # in $EDITOR while the user creates a commit message.
2401 # Otherwise, the user would not be able to see this output
2402 # once the editor has taken over the screen.
2403 qa_output = io.StringIO()
2404 style_file = ConsoleStyleFile(sys.stdout)
2405 if options.mode == 'commit' and \
2406         (not commitmessage or not commitmessage.strip()):
2407         style_file.write_listener = qa_output
2408 console_writer = StyleWriter(file=style_file, maxcol=9999)
2409 console_writer.style_listener = style_file.new_styles
2410
2411 f = formatter.AbstractFormatter(console_writer)
2412
2413 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2414
2415 style_file.flush()
2416 del console_writer, f, style_file
2417 qa_output = qa_output.getvalue()
2418 qa_output = qa_output.splitlines(True)
2419
2420 suggest_ignore_masked = False
2421 suggest_include_dev = False
2422
2423 if have_pmasked and not (options.without_mask or options.ignore_masked):
2424         suggest_ignore_masked = True
2425 if have_dev_keywords and not options.include_dev:
2426         suggest_include_dev = True
2427
2428 if suggest_ignore_masked or suggest_include_dev:
2429         print()
2430         if suggest_ignore_masked:
2431                 print(bold("Note: use --without-mask to check " + \
2432                         "KEYWORDS on dependencies of masked packages"))
2433
2434         if suggest_include_dev:
2435                 print(bold("Note: use --include-dev (-d) to check " + \
2436                         "dependencies for 'dev' profiles"))
2437         print()
2438
2439 if options.mode != 'commit':
2440         if dofull:
2441                 print(bold("Note: type \"repoman full\" for a complete listing."))
2442         if dowarn and not dofail:
2443                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2444         elif not dofail:
2445                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2446         elif dofail:
2447                 print(bad("Please fix these important QA issues first."))
2448                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2449                 sys.exit(1)
2450 else:
2451         if dofail and can_force and options.force and not options.pretend:
2452                 print(green("RepoMan sez:") + \
2453                         " \"You want to commit even with these QA issues?\n" + \
2454                         "              I'll take it this time, but I'm not happy.\"\n")
2455         elif dofail:
2456                 if options.force and not can_force:
2457                         print(bad("The --force option has been disabled due to extraordinary issues."))
2458                 print(bad("Please fix these important QA issues first."))
2459                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2460                 sys.exit(1)
2461
2462         if options.pretend:
2463                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2464
2465         myunadded = []
2466         if vcs == "cvs":
2467                 try:
2468                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2469                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2470                 except SystemExit as e:
2471                         raise  # TODO propagate this
2472                 except:
2473                         err("Error retrieving CVS tree; exiting.")
2474         if vcs == "svn":
2475                 try:
2476                         with repoman_popen("svn status --no-ignore") as f:
2477                                 svnstatus = f.readlines()
2478                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2479                 except SystemExit as e:
2480                         raise  # TODO propagate this
2481                 except:
2482                         err("Error retrieving SVN info; exiting.")
2483         if vcs == "git":
2484                 # get list of files not under version control or missing
2485                 myf = repoman_popen("git ls-files --others")
2486                 myunadded = ["./" + elem[:-1] for elem in myf]
2487                 myf.close()
2488         if vcs == "bzr":
2489                 try:
2490                         with repoman_popen("bzr status -S .") as f:
2491                                 bzrstatus = f.readlines()
2492                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2493                 except SystemExit as e:
2494                         raise  # TODO propagate this
2495                 except:
2496                         err("Error retrieving bzr info; exiting.")
2497         if vcs == "hg":
2498                 with repoman_popen("hg status --no-status --unknown .") as f:
2499                         myunadded = f.readlines()
2500                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2501
2502                 # Mercurial doesn't handle manually deleted files as removed from
2503                 # the repository, so the user need to remove them before commit,
2504                 # using "hg remove [FILES]"
2505                 with repoman_popen("hg status --no-status --deleted .") as f:
2506                         mydeleted = f.readlines()
2507                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2508
2509
2510         myautoadd = []
2511         if myunadded:
2512                 for x in range(len(myunadded)-1, -1, -1):
2513                         xs = myunadded[x].split("/")
2514                         if xs[-1] == "files":
2515                                 print("!!! files dir is not added! Please correct this.")
2516                                 sys.exit(-1)
2517                         elif xs[-1] == "Manifest":
2518                                 # It's a manifest... auto add
2519                                 myautoadd += [myunadded[x]]
2520                                 del myunadded[x]
2521
2522         if myunadded:
2523                 print(red("!!! The following files are in your local tree but are not added to the master"))
2524                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2525                 for x in myunadded:
2526                         print("   ", x)
2527                 print()
2528                 print()
2529                 sys.exit(1)
2530
2531         if vcs == "hg" and mydeleted:
2532                 print(red("!!! The following files are removed manually from your local tree but are not"))
2533                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2534                 for x in mydeleted:
2535                         print("   ", x)
2536                 print()
2537                 print()
2538                 sys.exit(1)
2539
2540         if vcs == "cvs":
2541                 mycvstree = cvstree.getentries("./", recursive=1)
2542                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2543                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2544                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2545                 bin_blob_pattern = re.compile("^-kb$")
2546                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2547                         recursive=1, basedir="./"))
2548
2549         if vcs == "svn":
2550                 with repoman_popen("svn status") as f:
2551                         svnstatus = f.readlines()
2552                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2553                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2554                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2555
2556                 # Subversion expands keywords specified in svn:keywords properties.
2557                 with repoman_popen("svn propget -R svn:keywords") as f:
2558                         props = f.readlines()
2559                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2560                         for prop in props if " - " in prop)
2561
2562         elif vcs == "git":
2563                 with repoman_popen("git diff-index --name-only "
2564                         "--relative --diff-filter=M HEAD") as f:
2565                         mychanged = f.readlines()
2566                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2567
2568                 with repoman_popen("git diff-index --name-only "
2569                         "--relative --diff-filter=A HEAD") as f:
2570                         mynew = f.readlines()
2571                 mynew = ["./" + elem[:-1] for elem in mynew]
2572
2573                 with repoman_popen("git diff-index --name-only "
2574                         "--relative --diff-filter=D HEAD") as f:
2575                         myremoved = f.readlines()
2576                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2577
2578         if vcs == "bzr":
2579                 with repoman_popen("bzr status -S .") as f:
2580                         bzrstatus = f.readlines()
2581                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2582                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2583                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2584                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2585                 # Bazaar expands nothing.
2586
2587         if vcs == "hg":
2588                 with repoman_popen("hg status --no-status --modified .") as f:
2589                         mychanged = f.readlines()
2590                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2591
2592                 with repoman_popen("hg status --no-status --added .") as f:
2593                         mynew = f.readlines()
2594                 mynew = ["./" + elem.rstrip() for elem in mynew]
2595
2596                 with repoman_popen("hg status --no-status --removed .") as f:
2597                         myremoved = f.readlines()
2598                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2599
2600         if vcs:
2601                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2602                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2603                         print()
2604                         print("(Didn't find any changed files...)")
2605                         print()
2606                         sys.exit(1)
2607
2608         # Manifests need to be regenerated after all other commits, so don't commit
2609         # them now even if they have changed.
2610         mymanifests = set()
2611         myupdates = set()
2612         for f in mychanged + mynew:
2613                 if "Manifest" == os.path.basename(f):
2614                         mymanifests.add(f)
2615                 else:
2616                         myupdates.add(f)
2617         myupdates.difference_update(myremoved)
2618         myupdates = list(myupdates)
2619         mymanifests = list(mymanifests)
2620         myheaders = []
2621         mydirty = []
2622
2623         commitmessage = options.commitmsg
2624         if options.commitmsgfile:
2625                 try:
2626                         f = io.open(_unicode_encode(options.commitmsgfile,
2627                         encoding=_encodings['fs'], errors='strict'),
2628                         mode='r', encoding=_encodings['content'], errors='replace')
2629                         commitmessage = f.read()
2630                         f.close()
2631                         del f
2632                 except (IOError, OSError) as e:
2633                         if e.errno == errno.ENOENT:
2634                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2635                         else:
2636                                 raise
2637                 # We've read the content so the file is no longer needed.
2638                 commitmessagefile = None
2639         if not commitmessage or not commitmessage.strip():
2640                 try:
2641                         editor = os.environ.get("EDITOR")
2642                         if editor and utilities.editor_is_executable(editor):
2643                                 commitmessage = utilities.get_commit_message_with_editor(
2644                                         editor, message=qa_output)
2645                         else:
2646                                 commitmessage = utilities.get_commit_message_with_stdin()
2647                 except KeyboardInterrupt:
2648                         exithandler()
2649                 if not commitmessage or not commitmessage.strip():
2650                         print("* no commit message?  aborting commit.")
2651                         sys.exit(1)
2652         commitmessage = commitmessage.rstrip()
2653         changelog_msg = commitmessage
2654         portage_version = getattr(portage, "VERSION", None)
2655         gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
2656         dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
2657         if portage_version is None:
2658                 sys.stderr.write("Failed to insert portage version in message!\n")
2659                 sys.stderr.flush()
2660                 portage_version = "Unknown"
2661
2662         report_options = []
2663         if options.force:
2664                 report_options.append("--force")
2665         if options.ignore_arches:
2666                 report_options.append("--ignore-arches")
2667         if include_arches is not None:
2668                 report_options.append("--include-arches=\"%s\"" %
2669                         " ".join(sorted(include_arches)))
2670
2671         if vcs == "git":
2672                 # Use new footer only for git (see bug #438364).
2673                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2674                 if report_options:
2675                         commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
2676                 if sign_manifests:
2677                         commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
2678                 if dco_sob:
2679                         commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
2680         else:
2681                 unameout = platform.system() + " "
2682                 if platform.system() in ["Darwin", "SunOS"]:
2683                         unameout += platform.processor()
2684                 else:
2685                         unameout += platform.machine()
2686                 commit_footer = "\n\n"
2687                 if dco_sob:
2688                         commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
2689                 commit_footer += "(Portage version: %s/%s/%s" % \
2690                         (portage_version, vcs, unameout)
2691                 if report_options:
2692                         commit_footer += ", RepoMan options: " + " ".join(report_options)
2693                 if sign_manifests:
2694                         commit_footer += ", signed Manifest commit with key %s" % \
2695                                 (gpg_key, )
2696                 else:
2697                         commit_footer += ", unsigned Manifest commit"
2698                 commit_footer += ")"
2699
2700         commitmessage += commit_footer
2701
2702         if options.echangelog in ('y', 'force'):
2703                 logging.info("checking for unmodified ChangeLog files")
2704                 committer_name = utilities.get_committer_name(env=repoman_settings)
2705                 for x in sorted(vcs_files_to_cps(
2706                         chain(myupdates, mymanifests, myremoved))):
2707                         catdir, pkgdir = x.split("/")
2708                         checkdir = repodir + "/" + x
2709                         checkdir_relative = ""
2710                         if repolevel < 3:
2711                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2712                         if repolevel < 2:
2713                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2714                         checkdir_relative = os.path.join(".", checkdir_relative)
2715
2716                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2717                         changelog_modified = changelog_path in modified_changelogs
2718                         if changelog_modified and options.echangelog != 'force':
2719                                 continue
2720
2721                         # get changes for this package
2722                         cdrlen = len(checkdir_relative)
2723                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2724                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2725                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2726
2727                         # Skip ChangeLog generation if only the Manifest was modified,
2728                         # as discussed in bug #398009.
2729                         nontrivial_cl_files = set()
2730                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2731                         nontrivial_cl_files.difference_update(['Manifest'])
2732                         if not nontrivial_cl_files and options.echangelog != 'force':
2733                                 continue
2734
2735                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2736                                 committer_name, changelog_msg,
2737                                 os.path.join(repodir, 'skel.ChangeLog'),
2738                                 catdir, pkgdir,
2739                                 new=clnew, removed=clremoved, changed=clchanged,
2740                                 pretend=options.pretend)
2741                         if new_changelog is None:
2742                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2743                                         level=logging.ERROR, noiselevel=-1)
2744                                 sys.exit(1)
2745
2746                         # if the ChangeLog was just created, add it to vcs
2747                         if new_changelog:
2748                                 myautoadd.append(changelog_path)
2749                                 # myautoadd is appended to myupdates below
2750                         else:
2751                                 myupdates.append(changelog_path)
2752
2753                         if options.ask and not options.pretend:
2754                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2755                                 repoman_settings["O"] = checkdir
2756                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2757
2758         if myautoadd:
2759                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2760                 add_cmd = [vcs, "add"]
2761                 add_cmd += myautoadd
2762                 if options.pretend:
2763                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2764                                 noiselevel=-1)
2765                 else:
2766                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2767                                 # Python 3.1 produces the following TypeError if raw bytes are
2768                                 # passed to subprocess.call():
2769                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2770                                 #     errread, errwrite)
2771                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2772                                 #     raise child_exception
2773                                 # TypeError: expected an object with the buffer interface
2774                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2775                         retcode = subprocess.call(add_cmd)
2776                         if retcode != os.EX_OK:
2777                                 logging.error(
2778                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2779                                 sys.exit(retcode)
2780
2781                 myupdates += myautoadd
2782
2783         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2784
2785         if vcs not in ('cvs', 'svn'):
2786                 # With git, bzr and hg, there's never any keyword expansion, so
2787                 # there's no need to regenerate manifests and all files will be
2788                 # committed in one big commit at the end.
2789                 print()
2790         elif not repo_config.thin_manifest:
2791                 if vcs == 'cvs':
2792                         headerstring = "'\$(Header|Id).*\$'"
2793                 elif vcs == "svn":
2794                         svn_keywords = dict((k.lower(), k) for k in [
2795                                         "Rev",
2796                                         "Revision",
2797                                         "LastChangedRevision",
2798                                         "Date",
2799                                         "LastChangedDate",
2800                                         "Author",
2801                                         "LastChangedBy",
2802                                         "URL",
2803                                         "HeadURL",
2804                                         "Id",
2805                                         "Header",
2806                         ])
2807
2808                 for myfile in myupdates:
2809
2810                         # for CVS, no_expansion contains files that are excluded from expansion
2811                         if vcs == "cvs":
2812                                 if myfile in no_expansion:
2813                                         continue
2814
2815                         # for SVN, expansion contains files that are included in expansion
2816                         elif vcs == "svn":
2817                                 if myfile not in expansion:
2818                                         continue
2819
2820                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2821                                 enabled_keywords = []
2822                                 for k in expansion[myfile]:
2823                                         keyword = svn_keywords.get(k.lower())
2824                                         if keyword is not None:
2825                                                 enabled_keywords.append(keyword)
2826
2827                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2828
2829                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2830                                 portage._shell_quote(myfile))
2831                         if myout[0] == 0:
2832                                 myheaders.append(myfile)
2833
2834                 print("%s have headers that will change." % green(str(len(myheaders))))
2835                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2836
2837         logging.info("myupdates: %s", myupdates)
2838         logging.info("myheaders: %s", myheaders)
2839
2840         if options.ask and userquery('Commit changes?', True) != 'Yes':
2841                 print("* aborting commit.")
2842                 sys.exit(128 + signal.SIGINT)
2843
2844         # Handle the case where committed files have keywords which
2845         # will change and need a priming commit before the Manifest
2846         # can be committed.
2847         if (myupdates or myremoved) and myheaders:
2848                 myfiles = myupdates + myremoved
2849                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2850                 mymsg = os.fdopen(fd, "wb")
2851                 mymsg.write(_unicode_encode(commitmessage))
2852                 mymsg.close()
2853
2854                 print()
2855                 print(green("Using commit message:"))
2856                 print(green("------------------------------------------------------------------------------"))
2857                 print(commitmessage)
2858                 print(green("------------------------------------------------------------------------------"))
2859                 print()
2860
2861                 # Having a leading ./ prefix on file paths can trigger a bug in
2862                 # the cvs server when committing files to multiple directories,
2863                 # so strip the prefix.
2864                 myfiles = [f.lstrip("./") for f in myfiles]
2865
2866                 commit_cmd = [vcs]
2867                 commit_cmd.extend(vcs_global_opts)
2868                 commit_cmd.append("commit")
2869                 commit_cmd.extend(vcs_local_opts)
2870                 commit_cmd.extend(["-F", commitmessagefile])
2871                 commit_cmd.extend(myfiles)
2872
2873                 try:
2874                         if options.pretend:
2875                                 print("(%s)" % (" ".join(commit_cmd),))
2876                         else:
2877                                 retval = spawn(commit_cmd, env=commit_env)
2878                                 if retval != os.EX_OK:
2879                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2880                                                 "error code: %s\n") % (vcs, retval),
2881                                                 level=logging.ERROR, noiselevel=-1)
2882                                         sys.exit(retval)
2883                 finally:
2884                         try:
2885                                 os.unlink(commitmessagefile)
2886                         except OSError:
2887                                 pass
2888
2889         # Setup the GPG commands
2890         def gpgsign(filename):
2891                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2892                 if gpgcmd is None:
2893                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2894                                 " Is make.globals missing?")
2895                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2896                         "PORTAGE_GPG_KEY" not in repoman_settings:
2897                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2898                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2899                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2900                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2901                                         os.path.expanduser("~/.gnupg")
2902                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2903                                         % repoman_settings["PORTAGE_GPG_DIR"])
2904                         else:
2905                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2906                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2907                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2908                                 raise portage.exception.InvalidLocation(
2909                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2910                                         repoman_settings["PORTAGE_GPG_DIR"])
2911                 gpgvars = {"FILE": filename}
2912                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2913                         v = repoman_settings.get(k)
2914                         if v is not None:
2915                                 gpgvars[k] = v
2916                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2917                 if options.pretend:
2918                         print("(" + gpgcmd + ")")
2919                 else:
2920                         # Encode unicode manually for bug #310789.
2921                         gpgcmd = portage.util.shlex_split(gpgcmd)
2922                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2923                                 # Python 3.1 does not support bytes in Popen args.
2924                                 gpgcmd = [_unicode_encode(arg,
2925                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2926                         rValue = subprocess.call(gpgcmd)
2927                         if rValue == os.EX_OK:
2928                                 os.rename(filename + ".asc", filename)
2929                         else:
2930                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2931
2932         def need_signature(filename):
2933                 try:
2934                         with open(_unicode_encode(filename,
2935                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2936                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2937                 except IOError as e:
2938                         if e.errno in (errno.ENOENT, errno.ESTALE):
2939                                 return False
2940                         raise
2941
2942         # When files are removed and re-added, the cvs server will put /Attic/
2943         # inside the $Header path. This code detects the problem and corrects it
2944         # so that the Manifest will generate correctly. See bug #169500.
2945         # Use binary mode in order to avoid potential character encoding issues.
2946         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2947         attic_str = b'/Attic/'
2948         attic_replace = b'/'
2949         for x in myheaders:
2950                 f = open(_unicode_encode(x,
2951                         encoding=_encodings['fs'], errors='strict'),
2952                         mode='rb')
2953                 mylines = f.readlines()
2954                 f.close()
2955                 modified = False
2956                 for i, line in enumerate(mylines):
2957                         if cvs_header_re.match(line) is not None and \
2958                                 attic_str in line:
2959                                 mylines[i] = line.replace(attic_str, attic_replace)
2960                                 modified = True
2961                 if modified:
2962                         portage.util.write_atomic(x, b''.join(mylines),
2963                                 mode='wb')
2964
2965         if repolevel == 1:
2966                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2967                         "doing the entire repository.\"\n")
2968
2969         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2970
2971                 for x in sorted(vcs_files_to_cps(
2972                         chain(myupdates, myremoved, mymanifests))):
2973                         repoman_settings["O"] = os.path.join(repodir, x)
2974                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2975
2976         signed = False
2977         if sign_manifests:
2978                 signed = True
2979                 try:
2980                         for x in sorted(vcs_files_to_cps(
2981                                 chain(myupdates, myremoved, mymanifests))):
2982                                 repoman_settings["O"] = os.path.join(repodir, x)
2983                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2984                                 if not need_signature(manifest_path):
2985                                         continue
2986                                 gpgsign(manifest_path)
2987                 except portage.exception.PortageException as e:
2988                         portage.writemsg("!!! %s\n" % str(e))
2989                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2990                         signed = False
2991
2992         if vcs == 'git':
2993                 # It's not safe to use the git commit -a option since there might
2994                 # be some modified files elsewhere in the working tree that the
2995                 # user doesn't want to commit. Therefore, call git update-index
2996                 # in order to ensure that the index is updated with the latest
2997                 # versions of all new and modified files in the relevant portion
2998                 # of the working tree.
2999                 myfiles = mymanifests + myupdates
3000                 myfiles.sort()
3001                 update_index_cmd = ["git", "update-index"]
3002                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
3003                 if options.pretend:
3004                         print("(%s)" % (" ".join(update_index_cmd),))
3005                 else:
3006                         retval = spawn(update_index_cmd, env=os.environ)
3007                         if retval != os.EX_OK:
3008                                 writemsg_level(("!!! Exiting on %s (shell) " + \
3009                                         "error code: %s\n") % (vcs, retval),
3010                                         level=logging.ERROR, noiselevel=-1)
3011                                 sys.exit(retval)
3012
3013         if True:
3014                 myfiles = mymanifests[:]
3015                 # If there are no header (SVN/CVS keywords) changes in
3016                 # the files, this Manifest commit must include the
3017                 # other (yet uncommitted) files.
3018                 if not myheaders:
3019                         myfiles += myupdates
3020                         myfiles += myremoved
3021                 myfiles.sort()
3022
3023                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3024                 mymsg = os.fdopen(fd, "wb")
3025                 mymsg.write(_unicode_encode(commitmessage))
3026                 mymsg.close()
3027
3028                 commit_cmd = []
3029                 if options.pretend and vcs is None:
3030                         # substitute a bogus value for pretend output
3031                         commit_cmd.append("cvs")
3032                 else:
3033                         commit_cmd.append(vcs)
3034                 commit_cmd.extend(vcs_global_opts)
3035                 commit_cmd.append("commit")
3036                 commit_cmd.extend(vcs_local_opts)
3037                 if vcs == "hg":
3038                         commit_cmd.extend(["--logfile", commitmessagefile])
3039                         commit_cmd.extend(myfiles)
3040                 else:
3041                         commit_cmd.extend(["-F", commitmessagefile])
3042                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3043
3044                 try:
3045                         if options.pretend:
3046                                 print("(%s)" % (" ".join(commit_cmd),))
3047                         else:
3048                                 retval = spawn(commit_cmd, env=commit_env)
3049                                 if retval != os.EX_OK:
3050                                         if repo_config.sign_commit and vcs == 'git' and \
3051                                                 not git_supports_gpg_sign():
3052                                                 # Inform user that newer git is needed (bug #403323).
3053                                                 logging.error(
3054                                                         "Git >=1.7.9 is required for signed commits!")
3055
3056                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3057                                                 "error code: %s\n") % (vcs, retval),
3058                                                 level=logging.ERROR, noiselevel=-1)
3059                                         sys.exit(retval)
3060                 finally:
3061                         try:
3062                                 os.unlink(commitmessagefile)
3063                         except OSError:
3064                                 pass
3065
3066         print()
3067         if vcs:
3068                 print("Commit complete.")
3069         else:
3070                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3071         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3072 sys.exit(0)