bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import codecs
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27 from itertools import chain
  28 from stat import S_ISDIR
  29
  30 try:
  31         from urllib.parse import urlparse
  32 except ImportError:
  33         from urlparse import urlparse
  34
  35 from os import path as osp
  36 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  37 sys.path.insert(0, pym_path)
  38 import portage
  39 portage._internal_caller = True
  40 portage._disable_legacy_globals()
  41
  42 try:
  43         import xml.etree.ElementTree
  44         from xml.parsers.expat import ExpatError
  45 except (SystemExit, KeyboardInterrupt):
  46         raise
  47 except (ImportError, SystemError, RuntimeError, Exception):
  48         # broken or missing xml support
  49         # http://bugs.python.org/issue14988
  50         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  51         from portage.output import EOutput
  52         out = EOutput()
  53         for line in msg:
  54                 out.eerror(line)
  55         sys.exit(1)
  56
  57 from portage import os
  58 from portage import _encodings
  59 from portage import _unicode_encode
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 import portage.repository.config
  69 from portage import cvstree, normalize_path
  70 from portage import util
  71 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  72         ParseError, PermissionDenied)
  73 from portage.dep import Atom
  74 from portage.process import find_binary, spawn
  75 from portage.output import bold, create_color_func, \
  76         green, nocolor, red
  77 from portage.output import ConsoleStyleFile, StyleWriter
  78 from portage.util import writemsg_level
  79 from portage.package.ebuild.digestgen import digestgen
  80 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  81
  82 if sys.hexversion >= 0x3000000:
  83         basestring = str
  84
  85 util.initialize_logger()
  86
  87 # 14 is the length of DESCRIPTION=""
  88 max_desc_len = 100
  89 allowed_filename_chars="a-zA-Z0-9._-+:"
  90 pv_toolong_re = re.compile(r'[0-9]{19,}')
  91 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  92 bad = create_color_func("BAD")
  93
  94 # A sane umask is needed for files that portage creates.
  95 os.umask(0o22)
  96 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  97 # behave incrementally.
  98 repoman_incrementals = tuple(x for x in \
  99         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 100 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 101 repoman_settings = portage.config(config_root=config_root, local_config=False)
 102
 103 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 104         repoman_settings.get('TERM') == 'dumb' or \
 105         not sys.stdout.isatty():
 106         nocolor()
 107
 108 def warn(txt):
 109         print("repoman: " + txt)
 110
 111 def err(txt):
 112         warn(txt)
 113         sys.exit(1)
 114
 115 def exithandler(signum=None, frame=None):
 116         logging.fatal("Interrupted; exiting...")
 117         if signum is None:
 118                 sys.exit(1)
 119         else:
 120                 sys.exit(128 + signum)
 121
 122 signal.signal(signal.SIGINT, exithandler)
 123
 124 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 125         """Repoman needs its own HelpFormatter for now, because the default ones
 126         murder the help text."""
 127
 128         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 129                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 130
 131         def format_description(self, description):
 132                 return description
 133
 134 class RepomanOptionParser(optparse.OptionParser):
 135         """Add the on_tail function, ruby has it, optionParser should too
 136         """
 137
 138         def __init__(self, *args, **kwargs):
 139                 optparse.OptionParser.__init__(self, *args, **kwargs)
 140                 self.tail = ""
 141
 142         def on_tail(self, description):
 143                 self.tail += description
 144
 145         def format_help(self, formatter=None):
 146                 result = optparse.OptionParser.format_help(self, formatter)
 147                 result += self.tail
 148                 return result
 149
 150
 151 def ParseArgs(argv, qahelp):
 152         """This function uses a customized optionParser to parse command line arguments for repoman
 153         Args:
 154           argv - a sequence of command line arguments
 155                 qahelp - a dict of qa warning to help message
 156         Returns:
 157           (opts, args), just like a call to parser.parse_args()
 158         """
 159
 160         argv = portage._decode_argv(argv)
 161
 162         modes = {
 163                 'commit' : 'Run a scan then commit changes',
 164                 'ci' : 'Run a scan then commit changes',
 165                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 166                 'full' : 'Scan directory tree and print all issues (not a summary)',
 167                 'help' : 'Show this screen',
 168                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 169                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 170                 'scan' : 'Scan directory tree for QA issues'
 171         }
 172
 173         mode_keys = list(modes)
 174         mode_keys.sort()
 175
 176         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 177         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 178         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 179         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 180         parser.description += "\nmodes: " + " | ".join(map(green, mode_keys))
 181
 182         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 183                 help='Request a confirmation before commiting')
 184
 185         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 186                 help='specify a commit message on the command line')
 187
 188         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 189                 help='specify a path to a file that contains a commit message')
 190
 191         parser.add_option('--digest',
 192                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 193                 help='Automatically update Manifest digests for modified files')
 194
 195         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 196                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 197
 198         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 199                 help='do not print unnecessary messages')
 200
 201         parser.add_option(
 202                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 203                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 204                 'regardless of modification if \'force\' is specified)')
 205
 206         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 207                 help='Commit with QA violations')
 208
 209         parser.add_option('--vcs', dest='vcs',
 210                 help='Force using specific VCS instead of autodetection')
 211
 212         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 213                 help='be very verbose in output', default=0)
 214
 215         parser.add_option('-V', '--version', dest='version', action='store_true',
 216                 help='show version info')
 217
 218         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 219                 default=False, help='forces the metadata.xml parse check to be carried out')
 220
 221         parser.add_option(
 222                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 223                 metavar="<y|n>",
 224                 help='only check packages that have uncommitted modifications')
 225
 226         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 227                 default=False, help='ignore arch-specific failures (where arch != host)')
 228
 229         parser.add_option("--ignore-default-opts",
 230                 action="store_true",
 231                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 232
 233         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 234                 default=False, help='ignore masked packages (not allowed with commit mode)')
 235
 236         parser.add_option('--include-arches', dest='include_arches',
 237                 metavar='ARCHES', action='append',
 238                 help='A space separated list of arches used to '
 239                 'filter the selection of profiles for dependency checks')
 240
 241         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 242                 default=False, help='include dev profiles in dependency checks')
 243
 244         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 245                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 246
 247         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 248                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 249
 250         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 251                 help='specify which mode repoman will run in (default=full)')
 252
 253         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 254
 255         for k in mode_keys:
 256                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 257
 258         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 259
 260         sorted_qa = list(qahelp)
 261         sorted_qa.sort()
 262         for k in sorted_qa:
 263                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 264
 265         opts, args = parser.parse_args(argv[1:])
 266
 267         if not opts.ignore_default_opts:
 268                 default_opts = portage.util.shlex_split(
 269                         repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
 270                 if default_opts:
 271                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 272
 273         if opts.mode == 'help':
 274                 parser.print_help(short=False)
 275
 276         for arg in args:
 277                 if arg in modes:
 278                         if not opts.mode:
 279                                 opts.mode = arg
 280                                 break
 281                 else:
 282                         parser.error("invalid mode: %s" % arg)
 283
 284         if not opts.mode:
 285                 opts.mode = 'full'
 286
 287         if opts.mode == 'ci':
 288                 opts.mode = 'commit'  # backwards compat shortcut
 289
 290         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 291                 if opts.ignore_masked:
 292                         parser.error('Commit mode and --ignore-masked are not compatible')
 293                 if opts.without_mask:
 294                         parser.error('Commit mode and --without-mask are not compatible')
 295
 296         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 297         for val in range(opts.verbosity):
 298                 logger = logging.getLogger()
 299                 logger.setLevel(logger.getEffectiveLevel() - 10)
 300
 301         for val in range(opts.quiet):
 302                 logger = logging.getLogger()
 303                 logger.setLevel(logger.getEffectiveLevel() + 10)
 304
 305         return (opts, args)
 306
 307 qahelp = {
 308         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 309         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 310         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 311         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 312         "changelog.missing": "Missing ChangeLog files",
 313         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 314         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 315         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 316         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 317         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 318         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 319         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 320         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 321         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 322         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 323         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 324         "file.size": "Files in the files directory must be under 20 KiB",
 325         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 326         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 327         "file.UTF8": "File is not UTF8 compliant",
 328         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 329         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 330         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 331         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 332         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 333         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 334         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 335         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 336         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 337         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 338         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 339         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 340         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 341         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 342         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 343         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 344         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 345         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 346         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 347         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 348         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 349         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 350         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 351         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 352         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 353         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 354         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 355         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 356         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 357         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 358         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 359         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 360         "variable.readonly": "Assigning a readonly variable",
 361         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 362         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 363         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 364         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 365         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 366         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 367         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 368         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 369         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 370         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 371         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 372         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 373         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 374         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 375         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 376         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 377         "ebuild.badheader": "This ebuild has a malformed header",
 378         "manifest.bad": "Manifest has missing or incorrect digests",
 379         "metadata.missing": "Missing metadata.xml files",
 380         "metadata.bad": "Bad metadata.xml files",
 381         "metadata.warning": "Warnings in metadata.xml files",
 382         "portage.internal": "The ebuild uses an internal Portage function or variable",
 383         "repo.eapi.banned": "The ebuild uses an EAPI which is banned by the repository's metadata/layout.conf settings",
 384         "repo.eapi.deprecated": "The ebuild uses an EAPI which is deprecated by the repository's metadata/layout.conf settings",
 385         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 386         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 387         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 388         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 389 }
 390
 391 qacats = list(qahelp)
 392 qacats.sort()
 393
 394 qawarnings = set((
 395 "changelog.missing",
 396 "changelog.notadded",
 397 "dependency.unknown",
 398 "digest.assumed",
 399 "digest.unused",
 400 "ebuild.notadded",
 401 "ebuild.nesteddie",
 402 "dependency.badmasked",
 403 "dependency.badindev",
 404 "dependency.badmaskedindev",
 405 "dependency.badtilde",
 406 "DESCRIPTION.toolong",
 407 "EAPI.deprecated",
 408 "HOMEPAGE.virtual",
 409 "LICENSE.deprecated",
 410 "LICENSE.virtual",
 411 "KEYWORDS.dropped",
 412 "KEYWORDS.stupid",
 413 "KEYWORDS.missing",
 414 "PDEPEND.suspect",
 415 "RDEPEND.implicit",
 416 "RDEPEND.suspect",
 417 "virtual.suspect",
 418 "RESTRICT.invalid",
 419 "ebuild.minorsyn",
 420 "ebuild.badheader",
 421 "ebuild.patches",
 422 "file.size",
 423 "inherit.unused",
 424 "inherit.deprecated",
 425 "java.eclassesnotused",
 426 "wxwidgets.eclassnotused",
 427 "metadata.warning",
 428 "portage.internal",
 429 "repo.eapi.deprecated",
 430 "usage.obsolete",
 431 "upstream.workaround",
 432 "LIVEVCS.stable",
 433 "LIVEVCS.unmasked",
 434 ))
 435
 436 if portage.const._ENABLE_INHERIT_CHECK:
 437         # This is experimental, so it's non-fatal.
 438         qawarnings.add("inherit.missing")
 439
 440 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 441
 442 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 443 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 444 allvars.update(Package.metadata_keys)
 445 allvars = sorted(allvars)
 446 commitmessage = None
 447 for x in missingvars:
 448         x += ".missing"
 449         if x not in qacats:
 450                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 451                 qacats.append(x)
 452                 qawarnings.add(x)
 453
 454 valid_restrict = frozenset(["binchecks", "bindist",
 455         "fetch", "installsources", "mirror", "preserve-libs",
 456         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 457
 458 live_eclasses = frozenset([
 459         "bzr",
 460         "cvs",
 461         "darcs",
 462         "git",
 463         "git-2",
 464         "mercurial",
 465         "subversion",
 466         "tla",
 467 ])
 468
 469 suspect_rdepend = frozenset([
 470         "app-arch/cabextract",
 471         "app-arch/rpm2targz",
 472         "app-doc/doxygen",
 473         "dev-lang/nasm",
 474         "dev-lang/swig",
 475         "dev-lang/yasm",
 476         "dev-perl/extutils-pkgconfig",
 477         "dev-util/byacc",
 478         "dev-util/cmake",
 479         "dev-util/ftjam",
 480         "dev-util/gperf",
 481         "dev-util/gtk-doc",
 482         "dev-util/gtk-doc-am",
 483         "dev-util/intltool",
 484         "dev-util/jam",
 485         "dev-util/pkg-config-lite",
 486         "dev-util/pkgconf",
 487         "dev-util/pkgconfig",
 488         "dev-util/pkgconfig-openbsd",
 489         "dev-util/scons",
 490         "dev-util/unifdef",
 491         "dev-util/yacc",
 492         "media-gfx/ebdftopcf",
 493         "sys-apps/help2man",
 494         "sys-devel/autoconf",
 495         "sys-devel/automake",
 496         "sys-devel/bin86",
 497         "sys-devel/bison",
 498         "sys-devel/dev86",
 499         "sys-devel/flex",
 500         "sys-devel/m4",
 501         "sys-devel/pmake",
 502         "virtual/linux-sources",
 503         "virtual/pkgconfig",
 504         "x11-misc/bdftopcf",
 505         "x11-misc/imake",
 506 ])
 507
 508 suspect_virtual = {
 509         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 510         "dev-util/pkgconf":"virtual/pkgconfig",
 511         "dev-util/pkgconfig":"virtual/pkgconfig",
 512         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 513         "dev-libs/libusb":"virtual/libusb",
 514         "dev-libs/libusbx":"virtual/libusb",
 515         "dev-libs/libusb-compat":"virtual/libusb",
 516 }
 517
 518 metadata_xml_encoding = 'UTF-8'
 519 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 520         (metadata_xml_encoding,)
 521 metadata_doctype_name = 'pkgmetadata'
 522 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 523 # force refetch if the local copy creation time is older than this
 524 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 525
 526 # file.executable
 527 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 528
 529 options, arguments = ParseArgs(sys.argv, qahelp)
 530
 531 if options.version:
 532         print("Portage", portage.VERSION)
 533         sys.exit(0)
 534
 535 # Set this to False when an extraordinary issue (generally
 536 # something other than a QA issue) makes it impossible to
 537 # commit (like if Manifest generation fails).
 538 can_force = True
 539
 540 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 541 if portdir is None:
 542         sys.exit(1)
 543
 544 myreporoot = os.path.basename(portdir_overlay)
 545 myreporoot += mydir[len(portdir_overlay):]
 546
 547 if options.vcs:
 548         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 549                 vcs = options.vcs
 550         else:
 551                 vcs = None
 552 else:
 553         vcses = utilities.FindVCS()
 554         if len(vcses) > 1:
 555                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 556                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 557                 sys.exit(1)
 558         elif vcses:
 559                 vcs = vcses[0]
 560         else:
 561                 vcs = None
 562
 563 if options.if_modified == "y" and vcs is None:
 564         logging.info("Not in a version controlled repository; "
 565                 "disabling --if-modified.")
 566         options.if_modified = "n"
 567
 568 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 569 vcs_preserves_mtime = vcs in ('cvs',)
 570
 571 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 572 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 573 if vcs_global_opts is None:
 574         if vcs in ('cvs', 'svn'):
 575                 vcs_global_opts = "-q"
 576         else:
 577                 vcs_global_opts = ""
 578 vcs_global_opts = vcs_global_opts.split()
 579
 580 if options.mode == 'commit' and not options.pretend and not vcs:
 581         logging.info("Not in a version controlled repository; enabling pretend mode.")
 582         options.pretend = True
 583
 584 # Ensure that current repository is in the list of enabled repositories.
 585 repodir = os.path.realpath(portdir_overlay)
 586 try:
 587         repoman_settings.repositories.get_repo_for_location(repodir)
 588 except KeyError:
 589         repo_name = portage.repository.config.RepoConfig._read_valid_repo_name(portdir_overlay)[0]
 590         layout_conf_data = portage.repository.config.parse_layout_conf(portdir_overlay)[0]
 591         if layout_conf_data['repo-name']:
 592                 repo_name = layout_conf_data['repo-name']
 593         tmp_conf_file = io.StringIO(textwrap.dedent("""
 594                 [%s]
 595                 location = %s
 596                 """) % (repo_name, portdir_overlay))
 597         # Ensure that the repository corresponding to $PWD overrides a
 598         # repository of the same name referenced by the existing PORTDIR
 599         # or PORTDIR_OVERLAY settings.
 600         repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 601                 (repoman_settings.get('PORTDIR_OVERLAY', ''),
 602                 portage._shell_quote(portdir_overlay))
 603         repositories = portage.repository.config.load_repository_config(repoman_settings, extra_files=[tmp_conf_file])
 604         # We have to call the config constructor again so that attributes
 605         # dependent on config.repositories are initialized correctly.
 606         repoman_settings = portage.config(config_root=config_root, local_config=False, repositories=repositories)
 607
 608 root = repoman_settings['EROOT']
 609 trees = {
 610         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 611 }
 612 portdb = trees[root]['porttree'].dbapi
 613
 614 # Constrain dependency resolution to the master(s)
 615 # that are specified in layout.conf.
 616 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 617 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 618 portdir = portdb.porttrees[0]
 619 commit_env = os.environ.copy()
 620 # list() is for iteration on a copy.
 621 for repo in list(repoman_settings.repositories):
 622         # all paths are canonical
 623         if repo.location not in repo_config.eclass_db.porttrees:
 624                 del repoman_settings.repositories[repo.name]
 625
 626 if repo_config.allow_provide_virtual:
 627         qawarnings.add("virtual.oldstyle")
 628
 629 if repo_config.sign_commit:
 630         if vcs == 'git':
 631                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 632                 # the commit arguments. If key_id is unspecified, then it must be
 633                 # configured by `git config user.signingkey key_id`.
 634                 vcs_local_opts.append("--gpg-sign")
 635                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 636                         # Pass GNUPGHOME to git for bug #462362.
 637                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 638
 639 # In order to disable manifest signatures, repos may set
 640 # "sign-manifests = false" in metadata/layout.conf. This
 641 # can be used to prevent merge conflicts like those that
 642 # thin-manifests is designed to prevent.
 643 sign_manifests = "sign" in repoman_settings.features and \
 644         repo_config.sign_manifest
 645
 646 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 647         options.mode in ("commit",) and not sign_manifests:
 648         msg = ("The '%s' repository has manifest signatures enabled, "
 649         "but FEATURES=sign is currently disabled. In order to avoid this "
 650         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 651         "repositories can disable manifest signatures by setting "
 652         "'sign-manifests = false' in metadata/layout.conf.") % \
 653         (repo_config.name,)
 654         for line in textwrap.wrap(msg, 60):
 655                 logging.warn(line)
 656
 657 if sign_manifests and options.mode in ("commit",) and \
 658         repoman_settings.get("PORTAGE_GPG_KEY") and \
 659         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 660         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 661         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 662                 repoman_settings["PORTAGE_GPG_KEY"])
 663         sys.exit(1)
 664
 665 manifest_hashes = repo_config.manifest_hashes
 666 if manifest_hashes is None:
 667         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 668
 669 if options.mode in ("commit", "fix", "manifest"):
 670         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 671                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 672                 "metadata/layout.conf does not contain the '%s' hash which "
 673                 "is required by this portage version. You will have to "
 674                 "upgrade portage if you want to generate valid manifests for "
 675                 "this repository.") % \
 676                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 677                 for line in textwrap.wrap(msg, 70):
 678                         logging.error(line)
 679                 sys.exit(1)
 680
 681         unsupported_hashes = manifest_hashes.difference(
 682                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 683         if unsupported_hashes:
 684                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 685                 "metadata/layout.conf contains one or more hash types '%s' "
 686                 "which are not supported by this portage version. You will "
 687                 "have to upgrade portage if you want to generate valid "
 688                 "manifests for this repository.") % \
 689                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 690                 for line in textwrap.wrap(msg, 70):
 691                         logging.error(line)
 692                 sys.exit(1)
 693
 694 if options.echangelog is None and repo_config.update_changelog:
 695         options.echangelog = 'y'
 696
 697 if vcs is None:
 698         options.echangelog = 'n'
 699
 700 # The --echangelog option causes automatic ChangeLog generation,
 701 # which invalidates changelog.ebuildadded and changelog.missing
 702 # checks.
 703 # Note: Some don't use ChangeLogs in distributed SCMs.
 704 # It will be generated on server side from scm log,
 705 # before package moves to the rsync server.
 706 # This is needed because they try to avoid merge collisions.
 707 # Gentoo's Council decided to always use the ChangeLog file.
 708 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 709 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 710
 711 if 'digest' in repoman_settings.features and options.digest != 'n':
 712         options.digest = 'y'
 713
 714 logging.debug("vcs: %s" % (vcs,))
 715 logging.debug("repo config: %s" % (repo_config,))
 716 logging.debug("options: %s" % (options,))
 717
 718 # It's confusing if these warnings are displayed without the user
 719 # being told which profile they come from, so disable them.
 720 env = os.environ.copy()
 721 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 722
 723 categories = []
 724 for path in repo_config.eclass_db.porttrees:
 725         categories.extend(portage.util.grabfile(
 726                 os.path.join(path, 'profiles', 'categories')))
 727 repoman_settings.categories = frozenset(
 728         portage.util.stack_lists([categories], incremental=1))
 729 categories = repoman_settings.categories
 730
 731 portdb.settings = repoman_settings
 732 root_config = RootConfig(repoman_settings, trees[root], None)
 733 # We really only need to cache the metadata that's necessary for visibility
 734 # filtering. Anything else can be discarded to reduce memory consumption.
 735 portdb._aux_cache_keys.clear()
 736 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 737
 738 reposplit = myreporoot.split(os.path.sep)
 739 repolevel = len(reposplit)
 740
 741 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 742 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 743 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 744 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 745         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 746         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 747         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 748         print(red("***"))
 749         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 750
 751 # Make startdir relative to the canonical repodir, so that we can pass
 752 # it to digestgen and it won't have to be canonicalized again.
 753 if repolevel == 1:
 754         startdir = repodir
 755 else:
 756         startdir = normalize_path(mydir)
 757         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 758
 759 def caterror(mycat):
 760         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 761
 762 def repoman_getstatusoutput(cmd):
 763         """
 764         Implements an interface similar to getstatusoutput(), but with
 765         customized unicode handling (see bug #310789) and without the shell.
 766         """
 767         args = portage.util.shlex_split(cmd)
 768
 769         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 770                 not os.path.isabs(args[0]):
 771                 # Python 3.1 _execvp throws TypeError for non-absolute executable
 772                 # path passed as bytes (see http://bugs.python.org/issue8513).
 773                 fullname = find_binary(args[0])
 774                 if fullname is None:
 775                         raise portage.exception.CommandNotFound(args[0])
 776                 args[0] = fullname
 777
 778         encoding = _encodings['fs']
 779         args = [_unicode_encode(x,
 780                 encoding=encoding, errors='strict') for x in args]
 781         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 782                 stderr=subprocess.STDOUT)
 783         output = portage._unicode_decode(proc.communicate()[0],
 784                 encoding=encoding, errors='strict')
 785         if output and output[-1] == "\n":
 786                 # getstatusoutput strips one newline
 787                 output = output[:-1]
 788         return (proc.wait(), output)
 789
 790 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 791         """
 792         Implements an interface similar to os.popen(), but with customized
 793         unicode handling (see bug #310789) and without the shell.
 794         """
 795
 796         __slots__ = ('_proc', '_stdout')
 797
 798         def __init__(self, cmd):
 799                 args = portage.util.shlex_split(cmd)
 800
 801                 if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
 802                         not os.path.isabs(args[0]):
 803                         # Python 3.1 _execvp throws TypeError for non-absolute executable
 804                         # path passed as bytes (see http://bugs.python.org/issue8513).
 805                         fullname = find_binary(args[0])
 806                         if fullname is None:
 807                                 raise portage.exception.CommandNotFound(args[0])
 808                         args[0] = fullname
 809
 810                 encoding = _encodings['fs']
 811                 args = [_unicode_encode(x,
 812                         encoding=encoding, errors='strict') for x in args]
 813                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 814                 object.__setattr__(self, '_proc', proc)
 815                 object.__setattr__(self, '_stdout',
 816                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 817
 818         def _get_target(self):
 819                 return object.__getattribute__(self, '_stdout')
 820
 821         __enter__ = _get_target
 822
 823         def __exit__(self, exc_type, exc_value, traceback):
 824                 proc = object.__getattribute__(self, '_proc')
 825                 proc.wait()
 826                 proc.stdout.close()
 827
 828 class ProfileDesc(object):
 829         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 830         def __init__(self, arch, status, sub_path, tree_path):
 831                 self.arch = arch
 832                 self.status = status
 833                 if sub_path:
 834                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 835                 self.sub_path = sub_path
 836                 self.tree_path = tree_path
 837                 if tree_path:
 838                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 839                 else:
 840                         self.abs_path = tree_path
 841
 842         def __str__(self):
 843                 if self.sub_path:
 844                         return self.sub_path
 845                 return 'empty profile'
 846
 847 profile_list = []
 848 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 849
 850 # get lists of valid keywords, licenses, and use
 851 kwlist = set()
 852 liclist = set()
 853 uselist = set()
 854 global_pmasklines = []
 855
 856 for path in portdb.porttrees:
 857         try:
 858                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 859         except OSError:
 860                 pass
 861         kwlist.update(portage.grabfile(os.path.join(path,
 862                 "profiles", "arch.list")))
 863
 864         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 865         for x in use_desc:
 866                 x = x.split()
 867                 if x:
 868                         uselist.add(x[0])
 869
 870         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 871         try:
 872                 expand_list = os.listdir(expand_desc_dir)
 873         except OSError:
 874                 pass
 875         else:
 876                 for fn in expand_list:
 877                         if not fn[-5:] == '.desc':
 878                                 continue
 879                         use_prefix = fn[:-5].lower() + '_'
 880                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 881                                 x = x.split()
 882                                 if x:
 883                                         uselist.add(use_prefix + x[0])
 884
 885         global_pmasklines.append(portage.util.grabfile_package(
 886                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 887
 888         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 889         try:
 890                 desc_file = io.open(_unicode_encode(desc_path,
 891                         encoding=_encodings['fs'], errors='strict'),
 892                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 893         except EnvironmentError:
 894                 pass
 895         else:
 896                 for i, x in enumerate(desc_file):
 897                         if x[0] == "#":
 898                                 continue
 899                         arch = x.split()
 900                         if len(arch) == 0:
 901                                 continue
 902                         if len(arch) != 3:
 903                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 904                                         desc_path + " line %d" % (i + 1, ))
 905                         elif arch[0] not in kwlist:
 906                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 907                                         desc_path + " line %d" % (i + 1, ))
 908                         elif arch[2] not in valid_profile_types:
 909                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 910                                         desc_path + " line %d" % (i + 1, ))
 911                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 912                         if not os.path.isdir(profile_desc.abs_path):
 913                                 logging.error(
 914                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 915                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 916                                 continue
 917                         if os.path.exists(
 918                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 919                                 continue
 920                         profile_list.append(profile_desc)
 921                 desc_file.close()
 922
 923 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 924 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 925
 926 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 927 global_pmaskdict = {}
 928 for x in global_pmasklines:
 929         global_pmaskdict.setdefault(x.cp, []).append(x)
 930 del global_pmasklines
 931
 932 def has_global_mask(pkg):
 933         mask_atoms = global_pmaskdict.get(pkg.cp)
 934         if mask_atoms:
 935                 pkg_list = [pkg]
 936                 for x in mask_atoms:
 937                         if portage.dep.match_from_list(x, pkg_list):
 938                                 return x
 939         return None
 940
 941 # Ensure that profile sub_path attributes are unique. Process in reverse order
 942 # so that profiles with duplicate sub_path from overlays will override
 943 # profiles with the same sub_path from parent repos.
 944 profiles = {}
 945 profile_list.reverse()
 946 profile_sub_paths = set()
 947 for prof in profile_list:
 948         if prof.sub_path in profile_sub_paths:
 949                 continue
 950         profile_sub_paths.add(prof.sub_path)
 951         profiles.setdefault(prof.arch, []).append(prof)
 952
 953 # Use an empty profile for checking dependencies of
 954 # packages that have empty KEYWORDS.
 955 prof = ProfileDesc('**', 'stable', '', '')
 956 profiles.setdefault(prof.arch, []).append(prof)
 957
 958 for x in repoman_settings.archlist():
 959         if x[0] == "~":
 960                 continue
 961         if x not in profiles:
 962                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 963                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 964                 print(red("up with the " + x + " team."))
 965                 print()
 966
 967 liclist_deprecated = set()
 968 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 969         liclist_deprecated.update(
 970                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 971
 972 if not liclist:
 973         logging.fatal("Couldn't find licenses?")
 974         sys.exit(1)
 975
 976 if not kwlist:
 977         logging.fatal("Couldn't read KEYWORDS from arch.list")
 978         sys.exit(1)
 979
 980 if not uselist:
 981         logging.fatal("Couldn't find use.desc?")
 982         sys.exit(1)
 983
 984 scanlist = []
 985 if repolevel == 2:
 986         # we are inside a category directory
 987         catdir = reposplit[-1]
 988         if catdir not in categories:
 989                 caterror(catdir)
 990         mydirlist = os.listdir(startdir)
 991         for x in mydirlist:
 992                 if x == "CVS" or x.startswith("."):
 993                         continue
 994                 if os.path.isdir(startdir + "/" + x):
 995                         scanlist.append(catdir + "/" + x)
 996         repo_subdir = catdir + os.sep
 997 elif repolevel == 1:
 998         for x in categories:
 999                 if not os.path.isdir(startdir + "/" + x):
1000                         continue
1001                 for y in os.listdir(startdir + "/" + x):
1002                         if y == "CVS" or y.startswith("."):
1003                                 continue
1004                         if os.path.isdir(startdir + "/" + x + "/" + y):
1005                                 scanlist.append(x + "/" + y)
1006         repo_subdir = ""
1007 elif repolevel == 3:
1008         catdir = reposplit[-2]
1009         if catdir not in categories:
1010                 caterror(catdir)
1011         scanlist.append(catdir + "/" + reposplit[-1])
1012         repo_subdir = scanlist[-1] + os.sep
1013 else:
1014         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
1015                 ' from the current working directory'
1016         logging.critical(msg)
1017         sys.exit(1)
1018
1019 repo_subdir_len = len(repo_subdir)
1020 scanlist.sort()
1021
1022 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
1023
1024 def vcs_files_to_cps(vcs_file_iter):
1025         """
1026         Iterate over the given modified file paths returned from the vcs,
1027         and return a frozenset containing category/pn strings for each
1028         modified package.
1029         """
1030
1031         modified_cps = []
1032
1033         if repolevel == 3:
1034                 if reposplit[-2] in categories and \
1035                         next(vcs_file_iter, None) is not None:
1036                         modified_cps.append("/".join(reposplit[-2:]))
1037
1038         elif repolevel == 2:
1039                 category = reposplit[-1]
1040                 if category in categories:
1041                         for filename in vcs_file_iter:
1042                                 f_split = filename.split(os.sep)
1043                                 # ['.', pn, ...]
1044                                 if len(f_split) > 2:
1045                                         modified_cps.append(category + "/" + f_split[1])
1046
1047         else:
1048                 # repolevel == 1
1049                 for filename in vcs_file_iter:
1050                         f_split = filename.split(os.sep)
1051                         # ['.', category, pn, ...]
1052                         if len(f_split) > 3 and f_split[1] in categories:
1053                                 modified_cps.append("/".join(f_split[1:3]))
1054
1055         return frozenset(modified_cps)
1056
1057 def git_supports_gpg_sign():
1058         status, cmd_output = \
1059                 repoman_getstatusoutput("git --version")
1060         cmd_output = cmd_output.split()
1061         if cmd_output:
1062                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1063                 if version is not None:
1064                         version = [int(x) for x in version.groups()]
1065                         if version[0] > 1 or \
1066                                 (version[0] == 1 and version[1] > 7) or \
1067                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1068                                 return True
1069         return False
1070
1071 def dev_keywords(profiles):
1072         """
1073         Create a set of KEYWORDS values that exist in 'dev'
1074         profiles. These are used
1075         to trigger a message notifying the user when they might
1076         want to add the --include-dev option.
1077         """
1078         type_arch_map = {}
1079         for arch, arch_profiles in profiles.items():
1080                 for prof in arch_profiles:
1081                         arch_set = type_arch_map.get(prof.status)
1082                         if arch_set is None:
1083                                 arch_set = set()
1084                                 type_arch_map[prof.status] = arch_set
1085                         arch_set.add(arch)
1086
1087         dev_keywords = type_arch_map.get('dev', set())
1088         dev_keywords.update(['~' + arch for arch in dev_keywords])
1089         return frozenset(dev_keywords)
1090
1091 dev_keywords = dev_keywords(profiles)
1092
1093 stats = {}
1094 fails = {}
1095
1096 for x in qacats:
1097         stats[x] = 0
1098         fails[x] = []
1099
1100 xmllint_capable = False
1101 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1102
1103 def fetch_metadata_dtd():
1104         """
1105         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1106         metadata_dtd_ctime_interval.
1107         @rtype: bool
1108         @return: True if successful, otherwise False
1109         """
1110
1111         must_fetch = True
1112         metadata_dtd_st = None
1113         current_time = int(time.time())
1114         try:
1115                 metadata_dtd_st = os.stat(metadata_dtd)
1116         except EnvironmentError as e:
1117                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1118                         raise
1119                 del e
1120         else:
1121                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1122                 if abs(current_time - metadata_dtd_st.st_ctime) \
1123                         < metadata_dtd_ctime_interval:
1124                         must_fetch = False
1125
1126         if must_fetch:
1127                 print()
1128                 print(green("***") + " the local copy of metadata.dtd " + \
1129                         "needs to be refetched, doing that now")
1130                 print()
1131                 parsed_url = urlparse(metadata_dtd_uri)
1132                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
1133                 fcmd = repoman_settings.get(setting)
1134                 if not fcmd:
1135                         fcmd = repoman_settings.get('FETCHCOMMAND')
1136                         if not fcmd:
1137                                 logging.error("FETCHCOMMAND is unset")
1138                                 return False
1139
1140                 destdir = repoman_settings["DISTDIR"]
1141                 fd, metadata_dtd_tmp = tempfile.mkstemp(
1142                         prefix='metadata.dtd.', dir=destdir)
1143                 os.close(fd)
1144
1145                 try:
1146                         if not portage.getbinpkg.file_get(metadata_dtd_uri,
1147                                 destdir, fcmd=fcmd,
1148                                 filename=os.path.basename(metadata_dtd_tmp)):
1149                                 logging.error("failed to fetch metadata.dtd from '%s'" %
1150                                         metadata_dtd_uri)
1151                                 return False
1152
1153                         try:
1154                                 portage.util.apply_secpass_permissions(metadata_dtd_tmp,
1155                                         gid=portage.data.portage_gid, mode=0o664, mask=0o2)
1156                         except portage.exception.PortageException:
1157                                 pass
1158
1159                         os.rename(metadata_dtd_tmp, metadata_dtd)
1160                 finally:
1161                         try:
1162                                 os.unlink(metadata_dtd_tmp)
1163                         except OSError:
1164                                 pass
1165
1166         return True
1167
1168 if options.mode == "manifest":
1169         pass
1170 elif not find_binary('xmllint'):
1171         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1172         if options.xml_parse or repolevel == 3:
1173                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1174                 sys.exit(1)
1175 else:
1176         if not fetch_metadata_dtd():
1177                 sys.exit(1)
1178         # this can be problematic if xmllint changes their output
1179         xmllint_capable = True
1180
1181 if options.mode == 'commit' and vcs:
1182         utilities.detect_vcs_conflicts(options, vcs)
1183
1184 if options.mode == "manifest":
1185         pass
1186 elif options.pretend:
1187         print(green("\nRepoMan does a once-over of the neighborhood..."))
1188 else:
1189         print(green("\nRepoMan scours the neighborhood..."))
1190
1191 new_ebuilds = set()
1192 modified_ebuilds = set()
1193 modified_changelogs = set()
1194 mychanged = []
1195 mynew = []
1196 myremoved = []
1197
1198 if vcs == "cvs":
1199         mycvstree = cvstree.getentries("./", recursive=1)
1200         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1201         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1202         if options.if_modified == "y":
1203                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1204
1205 elif vcs == "svn":
1206         with repoman_popen("svn status") as f:
1207                 svnstatus = f.readlines()
1208         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1209         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1210         if options.if_modified == "y":
1211                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1212
1213 elif vcs == "git":
1214         with repoman_popen("git diff-index --name-only "
1215                 "--relative --diff-filter=M HEAD") as f:
1216                 mychanged = f.readlines()
1217         mychanged = ["./" + elem[:-1] for elem in mychanged]
1218
1219         with repoman_popen("git diff-index --name-only "
1220                 "--relative --diff-filter=A HEAD") as f:
1221                 mynew = f.readlines()
1222         mynew = ["./" + elem[:-1] for elem in mynew]
1223         if options.if_modified == "y":
1224                 with repoman_popen("git diff-index --name-only "
1225                         "--relative --diff-filter=D HEAD") as f:
1226                         myremoved = f.readlines()
1227                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1228
1229 elif vcs == "bzr":
1230         with repoman_popen("bzr status -S .") as f:
1231                 bzrstatus = f.readlines()
1232         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1233         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1234         if options.if_modified == "y":
1235                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1236
1237 elif vcs == "hg":
1238         with repoman_popen("hg status --no-status --modified .") as f:
1239                 mychanged = f.readlines()
1240         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1241         with repoman_popen("hg status --no-status --added .") as f:
1242                 mynew = f.readlines()
1243         mynew = ["./" + elem.rstrip() for elem in mynew]
1244         if options.if_modified == "y":
1245                 with repoman_popen("hg status --no-status --removed .") as f:
1246                         myremoved = f.readlines()
1247                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1248
1249 if vcs:
1250         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1251         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1252         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1253                 if os.path.basename(x) == "ChangeLog")
1254
1255 def vcs_new_changed(relative_path):
1256         for x in chain(mychanged, mynew):
1257                 if x == relative_path:
1258                         return True
1259         return False
1260
1261 have_pmasked = False
1262 have_dev_keywords = False
1263 dofail = 0
1264
1265 # NOTE: match-all caches are not shared due to potential
1266 # differences between profiles in _get_implicit_iuse.
1267 arch_caches = {}
1268 arch_xmatch_caches = {}
1269 shared_xmatch_caches = {"cp-list":{}}
1270
1271 include_arches = None
1272 if options.include_arches:
1273         include_arches = set()
1274         include_arches.update(*[x.split() for x in options.include_arches])
1275
1276 # Disable the "ebuild.notadded" check when not in commit mode and
1277 # running `svn status` in every package dir will be too expensive.
1278
1279 check_ebuild_notadded = not \
1280         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1281
1282 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1283 thirdpartymirrors = {}
1284 for k, v in repoman_settings.thirdpartymirrors().items():
1285         for v in v:
1286                 if not v.endswith("/"):
1287                         v += "/"
1288                 thirdpartymirrors[v] = k
1289
1290 class _XMLParser(xml.etree.ElementTree.XMLParser):
1291
1292         def __init__(self, data, **kwargs):
1293                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1294                 self._portage_data = data
1295                 if hasattr(self, 'parser'):
1296                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1297                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1298                         self._base_StartDoctypeDeclHandler = \
1299                                 self.parser.StartDoctypeDeclHandler
1300                         self.parser.StartDoctypeDeclHandler = \
1301                                 self._portage_StartDoctypeDeclHandler
1302
1303         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1304                 if self._base_XmlDeclHandler is not None:
1305                         self._base_XmlDeclHandler(version, encoding, standalone)
1306                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1307
1308         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1309                 has_internal_subset):
1310                 if self._base_StartDoctypeDeclHandler is not None:
1311                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1312                                 has_internal_subset)
1313                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1314
1315 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1316         """
1317         Implements doctype() as required to avoid deprecation warnings with
1318         >=python-2.7.
1319         """
1320         def doctype(self, name, pubid, system):
1321                 pass
1322
1323 try:
1324         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1325 except (EnvironmentError, ParseError, PermissionDenied) as e:
1326         err(str(e))
1327 except FileNotFound:
1328         # TODO: Download as we do for metadata.dtd, but add a way to
1329         # disable for non-gentoo repoman users who may not have herds.
1330         herd_base = None
1331
1332 effective_scanlist = scanlist
1333 if options.if_modified == "y":
1334         effective_scanlist = sorted(vcs_files_to_cps(
1335                 chain(mychanged, mynew, myremoved)))
1336
1337 for x in effective_scanlist:
1338         #ebuilds and digests added to cvs respectively.
1339         logging.info("checking package %s" % x)
1340         # save memory by discarding xmatch caches from previous package(s)
1341         arch_xmatch_caches.clear()
1342         eadded = []
1343         catdir, pkgdir = x.split("/")
1344         checkdir = repodir + "/" + x
1345         checkdir_relative = ""
1346         if repolevel < 3:
1347                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1348         if repolevel < 2:
1349                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1350         checkdir_relative = os.path.join(".", checkdir_relative)
1351         generated_manifest = False
1352
1353         if options.mode == "manifest" or \
1354           (options.mode != 'manifest-check' and options.digest == 'y') or \
1355           options.mode in ('commit', 'fix') and not options.pretend:
1356                 auto_assumed = set()
1357                 fetchlist_dict = portage.FetchlistDict(checkdir,
1358                         repoman_settings, portdb)
1359                 if options.mode == 'manifest' and options.force:
1360                         portage._doebuild_manifest_exempt_depend += 1
1361                         try:
1362                                 distdir = repoman_settings['DISTDIR']
1363                                 mf = repoman_settings.repositories.get_repo_for_location(
1364                                         os.path.dirname(os.path.dirname(checkdir)))
1365                                 mf = mf.load_manifest(checkdir, distdir,
1366                                         fetchlist_dict=fetchlist_dict)
1367                                 mf.create(requiredDistfiles=None,
1368                                         assumeDistHashesAlways=True)
1369                                 for distfiles in fetchlist_dict.values():
1370                                         for distfile in distfiles:
1371                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1372                                                         mf.fhashdict['DIST'].pop(distfile, None)
1373                                                 else:
1374                                                         auto_assumed.add(distfile)
1375                                 mf.write()
1376                         finally:
1377                                 portage._doebuild_manifest_exempt_depend -= 1
1378
1379                 repoman_settings["O"] = checkdir
1380                 try:
1381                         generated_manifest = digestgen(
1382                                 mysettings=repoman_settings, myportdb=portdb)
1383                 except portage.exception.PermissionDenied as e:
1384                         generated_manifest = False
1385                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1386                                 level=logging.ERROR, noiselevel=-1)
1387
1388                 if not generated_manifest:
1389                         print("Unable to generate manifest.")
1390                         dofail = 1
1391
1392                 if options.mode == "manifest":
1393                         if not dofail and options.force and auto_assumed and \
1394                                 'assume-digests' in repoman_settings.features:
1395                                 # Show which digests were assumed despite the --force option
1396                                 # being given. This output will already have been shown by
1397                                 # digestgen() if assume-digests is not enabled, so only show
1398                                 # it here if assume-digests is enabled.
1399                                 pkgs = list(fetchlist_dict)
1400                                 pkgs.sort()
1401                                 portage.writemsg_stdout("  digest.assumed" + \
1402                                         portage.output.colorize("WARN",
1403                                         str(len(auto_assumed)).rjust(18)) + "\n")
1404                                 for cpv in pkgs:
1405                                         fetchmap = fetchlist_dict[cpv]
1406                                         pf = portage.catsplit(cpv)[1]
1407                                         for distfile in sorted(fetchmap):
1408                                                 if distfile in auto_assumed:
1409                                                         portage.writemsg_stdout(
1410                                                                 "   %s::%s\n" % (pf, distfile))
1411                         continue
1412                 elif dofail:
1413                         sys.exit(1)
1414
1415         if not generated_manifest:
1416                 repoman_settings['O'] = checkdir
1417                 repoman_settings['PORTAGE_QUIET'] = '1'
1418                 if not portage.digestcheck([], repoman_settings, strict=1):
1419                         stats["manifest.bad"] += 1
1420                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1421                 repoman_settings.pop('PORTAGE_QUIET', None)
1422
1423         if options.mode == 'manifest-check':
1424                 continue
1425
1426         checkdirlist = os.listdir(checkdir)
1427         ebuildlist = []
1428         pkgs = {}
1429         allvalid = True
1430         for y in checkdirlist:
1431                 if (y in no_exec or y.endswith(".ebuild")) and \
1432                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1433                                 stats["file.executable"] += 1
1434                                 fails["file.executable"].append(os.path.join(checkdir, y))
1435                 if y.endswith(".ebuild"):
1436                         pf = y[:-7]
1437                         ebuildlist.append(pf)
1438                         cpv = "%s/%s" % (catdir, pf)
1439                         try:
1440                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1441                         except KeyError:
1442                                 allvalid = False
1443                                 stats["ebuild.syntax"] += 1
1444                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1445                                 continue
1446                         except IOError:
1447                                 allvalid = False
1448                                 stats["ebuild.output"] += 1
1449                                 fails["ebuild.output"].append(os.path.join(x, y))
1450                                 continue
1451                         if not portage.eapi_is_supported(myaux["EAPI"]):
1452                                 allvalid = False
1453                                 stats["EAPI.unsupported"] += 1
1454                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1455                                 continue
1456                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1457                                 root_config=root_config, type_name="ebuild")
1458
1459         slot_keywords = {}
1460
1461         if len(pkgs) != len(ebuildlist):
1462                 # If we can't access all the metadata then it's totally unsafe to
1463                 # commit since there's no way to generate a correct Manifest.
1464                 # Do not try to do any more QA checks on this package since missing
1465                 # metadata leads to false positives for several checks, and false
1466                 # positives confuse users.
1467                 can_force = False
1468                 continue
1469
1470         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1471         ebuildlist = sorted(pkgs.values())
1472         ebuildlist = [pkg.pf for pkg in ebuildlist]
1473
1474         for y in checkdirlist:
1475                 index = repo_config.find_invalid_path_char(y)
1476                 if index != -1:
1477                         y_relative = os.path.join(checkdir_relative, y)
1478                         if vcs is not None and not vcs_new_changed(y_relative):
1479                                 # If the file isn't in the VCS new or changed set, then
1480                                 # assume that it's an irrelevant temporary file (Manifest
1481                                 # entries are not generated for file names containing
1482                                 # prohibited characters). See bug #406877.
1483                                 index = -1
1484                 if index != -1:
1485                         stats["file.name"] += 1
1486                         fails["file.name"].append("%s/%s: char '%s'" % \
1487                                 (checkdir, y, y[index]))
1488
1489                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1490                         continue
1491                 f = None
1492                 try:
1493                         line = 1
1494                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1495                                 encoding=_encodings['fs'], errors='strict'),
1496                                 mode='r', encoding=_encodings['repo.content'])
1497                         for l in f:
1498                                 line += 1
1499                 except UnicodeDecodeError as ue:
1500                         stats["file.UTF8"] += 1
1501                         s = ue.object[:ue.start]
1502                         l2 = s.count("\n")
1503                         line += l2
1504                         if l2 != 0:
1505                                 s = s[s.rfind("\n") + 1:]
1506                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1507                 finally:
1508                         if f is not None:
1509                                 f.close()
1510
1511         if vcs in ("git", "hg") and check_ebuild_notadded:
1512                 if vcs == "git":
1513                         myf = repoman_popen("git ls-files --others %s" % \
1514                                 (portage._shell_quote(checkdir_relative),))
1515                 if vcs == "hg":
1516                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1517                                 (portage._shell_quote(checkdir_relative),))
1518                 for l in myf:
1519                         if l[:-1][-7:] == ".ebuild":
1520                                 stats["ebuild.notadded"] += 1
1521                                 fails["ebuild.notadded"].append(
1522                                         os.path.join(x, os.path.basename(l[:-1])))
1523                 myf.close()
1524
1525         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1526                 try:
1527                         if vcs == "cvs":
1528                                 myf = open(checkdir + "/CVS/Entries", "r")
1529                         if vcs == "svn":
1530                                 myf = repoman_popen("svn status --depth=files --verbose " +
1531                                         portage._shell_quote(checkdir))
1532                         if vcs == "bzr":
1533                                 myf = repoman_popen("bzr ls -v --kind=file " +
1534                                         portage._shell_quote(checkdir))
1535                         myl = myf.readlines()
1536                         myf.close()
1537                         for l in myl:
1538                                 if vcs == "cvs":
1539                                         if l[0] != "/":
1540                                                 continue
1541                                         splitl = l[1:].split("/")
1542                                         if not len(splitl):
1543                                                 continue
1544                                         if splitl[0][-7:] == ".ebuild":
1545                                                 eadded.append(splitl[0][:-7])
1546                                 if vcs == "svn":
1547                                         if l[:1] == "?":
1548                                                 continue
1549                                         if l[:7] == '      >':
1550                                                 # tree conflict, new in subversion 1.6
1551                                                 continue
1552                                         l = l.split()[-1]
1553                                         if l[-7:] == ".ebuild":
1554                                                 eadded.append(os.path.basename(l[:-7]))
1555                                 if vcs == "bzr":
1556                                         if l[1:2] == "?":
1557                                                 continue
1558                                         l = l.split()[-1]
1559                                         if l[-7:] == ".ebuild":
1560                                                 eadded.append(os.path.basename(l[:-7]))
1561                         if vcs == "svn":
1562                                 myf = repoman_popen("svn status " +
1563                                         portage._shell_quote(checkdir))
1564                                 myl = myf.readlines()
1565                                 myf.close()
1566                                 for l in myl:
1567                                         if l[0] == "A":
1568                                                 l = l.rstrip().split(' ')[-1]
1569                                                 if l[-7:] == ".ebuild":
1570                                                         eadded.append(os.path.basename(l[:-7]))
1571                 except IOError:
1572                         if vcs == "cvs":
1573                                 stats["CVS/Entries.IO_error"] += 1
1574                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1575                         else:
1576                                 raise
1577                         continue
1578
1579         mf = repoman_settings.repositories.get_repo_for_location(
1580                 os.path.dirname(os.path.dirname(checkdir)))
1581         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1582         mydigests = mf.getTypeDigests("DIST")
1583
1584         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1585         myfiles_all = []
1586         src_uri_error = False
1587         for mykey in fetchlist_dict:
1588                 try:
1589                         myfiles_all.extend(fetchlist_dict[mykey])
1590                 except portage.exception.InvalidDependString as e:
1591                         src_uri_error = True
1592                         try:
1593                                 portdb.aux_get(mykey, ["SRC_URI"])
1594                         except KeyError:
1595                                 # This will be reported as an "ebuild.syntax" error.
1596                                 pass
1597                         else:
1598                                 stats["SRC_URI.syntax"] += 1
1599                                 fails["SRC_URI.syntax"].append(
1600                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1601         del fetchlist_dict
1602         if not src_uri_error:
1603                 # This test can produce false positives if SRC_URI could not
1604                 # be parsed for one or more ebuilds. There's no point in
1605                 # producing a false error here since the root cause will
1606                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1607                 # or "ebuild.sytax".
1608                 myfiles_all = set(myfiles_all)
1609                 for entry in mydigests:
1610                         if entry not in myfiles_all:
1611                                 stats["digest.unused"] += 1
1612                                 fails["digest.unused"].append(checkdir + "::" + entry)
1613                 for entry in myfiles_all:
1614                         if entry not in mydigests:
1615                                 stats["digest.missing"] += 1
1616                                 fails["digest.missing"].append(checkdir + "::" + entry)
1617         del myfiles_all
1618
1619         if os.path.exists(checkdir + "/files"):
1620                 filesdirlist = os.listdir(checkdir + "/files")
1621
1622                 # recurse through files directory
1623                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1624                 while filesdirlist:
1625                         y = filesdirlist.pop(0)
1626                         relative_path = os.path.join(x, "files", y)
1627                         full_path = os.path.join(repodir, relative_path)
1628                         try:
1629                                 mystat = os.stat(full_path)
1630                         except OSError as oe:
1631                                 if oe.errno == 2:
1632                                         # don't worry about it.  it likely was removed via fix above.
1633                                         continue
1634                                 else:
1635                                         raise oe
1636                         if S_ISDIR(mystat.st_mode):
1637                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1638                                 if y == "CVS" or y == ".svn":
1639                                         continue
1640                                 for z in os.listdir(checkdir + "/files/" + y):
1641                                         if z == "CVS" or z == ".svn":
1642                                                 continue
1643                                         filesdirlist.append(y + "/" + z)
1644                         # Current policy is no files over 20 KiB, these are the checks. File size between
1645                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1646                         elif mystat.st_size > 61440:
1647                                 stats["file.size.fatal"] += 1
1648                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1649                         elif mystat.st_size > 20480:
1650                                 stats["file.size"] += 1
1651                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1652
1653                         index = repo_config.find_invalid_path_char(y)
1654                         if index != -1:
1655                                 y_relative = os.path.join(checkdir_relative, "files", y)
1656                                 if vcs is not None and not vcs_new_changed(y_relative):
1657                                         # If the file isn't in the VCS new or changed set, then
1658                                         # assume that it's an irrelevant temporary file (Manifest
1659                                         # entries are not generated for file names containing
1660                                         # prohibited characters). See bug #406877.
1661                                         index = -1
1662                         if index != -1:
1663                                 stats["file.name"] += 1
1664                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1665                                         (checkdir, y, y[index]))
1666         del mydigests
1667
1668         if check_changelog and "ChangeLog" not in checkdirlist:
1669                 stats["changelog.missing"] += 1
1670                 fails["changelog.missing"].append(x + "/ChangeLog")
1671
1672         musedict = {}
1673         # metadata.xml file check
1674         if "metadata.xml" not in checkdirlist:
1675                 stats["metadata.missing"] += 1
1676                 fails["metadata.missing"].append(x + "/metadata.xml")
1677         # metadata.xml parse check
1678         else:
1679                 metadata_bad = False
1680                 xml_info = {}
1681                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1682
1683                 # read metadata.xml into memory
1684                 try:
1685                         _metadata_xml = xml.etree.ElementTree.parse(
1686                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1687                                 encoding=_encodings['fs'], errors='strict'),
1688                                 parser=xml_parser)
1689                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1690                         metadata_bad = True
1691                         stats["metadata.bad"] += 1
1692                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1693                         del e
1694                 else:
1695                         if not hasattr(xml_parser, 'parser') or \
1696                                 sys.hexversion < 0x2070000 or \
1697                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1698                                 # doctype is not parsed with python 2.6 or 3.1
1699                                 pass
1700                         else:
1701                                 if "XML_DECLARATION" not in xml_info:
1702                                         stats["metadata.bad"] += 1
1703                                         fails["metadata.bad"].append("%s/metadata.xml: "
1704                                                 "xml declaration is missing on first line, "
1705                                                 "should be '%s'" % (x, metadata_xml_declaration))
1706                                 else:
1707                                         xml_version, xml_encoding, xml_standalone = \
1708                                                 xml_info["XML_DECLARATION"]
1709                                         if xml_encoding is None or \
1710                                                 xml_encoding.upper() != metadata_xml_encoding:
1711                                                 stats["metadata.bad"] += 1
1712                                                 if xml_encoding is None:
1713                                                         encoding_problem = "but it is undefined"
1714                                                 else:
1715                                                         encoding_problem = "not '%s'" % xml_encoding
1716                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1717                                                         "xml declaration encoding should be '%s', %s" %
1718                                                         (x, metadata_xml_encoding, encoding_problem))
1719
1720                                 if "DOCTYPE" not in xml_info:
1721                                         metadata_bad = True
1722                                         stats["metadata.bad"] += 1
1723                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1724                                                 "DOCTYPE is missing"))
1725                                 else:
1726                                         doctype_name, doctype_system, doctype_pubid = \
1727                                                 xml_info["DOCTYPE"]
1728                                         if doctype_system != metadata_dtd_uri:
1729                                                 stats["metadata.bad"] += 1
1730                                                 if doctype_system is None:
1731                                                         system_problem = "but it is undefined"
1732                                                 else:
1733                                                         system_problem = "not '%s'" % doctype_system
1734                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1735                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1736                                                         (x, metadata_dtd_uri, system_problem))
1737
1738                                         if doctype_name != metadata_doctype_name:
1739                                                 stats["metadata.bad"] += 1
1740                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1741                                                         "DOCTYPE: name should be '%s', not '%s'" %
1742                                                         (x, metadata_doctype_name, doctype_name))
1743
1744                         # load USE flags from metadata.xml
1745                         try:
1746                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1747                         except portage.exception.ParseError as e:
1748                                 metadata_bad = True
1749                                 stats["metadata.bad"] += 1
1750                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1751                         else:
1752                                 for atom in chain(*musedict.values()):
1753                                         if atom is None:
1754                                                 continue
1755                                         try:
1756                                                 atom = Atom(atom)
1757                                         except InvalidAtom as e:
1758                                                 stats["metadata.bad"] += 1
1759                                                 fails["metadata.bad"].append(
1760                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1761                                         else:
1762                                                 if atom.cp != x:
1763                                                         stats["metadata.bad"] += 1
1764                                                         fails["metadata.bad"].append(
1765                                                                 ("%s/metadata.xml: Atom contains "
1766                                                                 "unexpected cat/pn: %s") % (x, atom))
1767
1768                         # Run other metadata.xml checkers
1769                         try:
1770                                 utilities.check_metadata(_metadata_xml, herd_base)
1771                         except (utilities.UnknownHerdsError, ) as e:
1772                                 metadata_bad = True
1773                                 stats["metadata.bad"] += 1
1774                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1775                                 del e
1776
1777                 #Only carry out if in package directory or check forced
1778                 if xmllint_capable and not metadata_bad:
1779                         # xmlint can produce garbage output even on success, so only dump
1780                         # the ouput when it fails.
1781                         st, out = repoman_getstatusoutput(
1782                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1783                                  (portage._shell_quote(metadata_dtd),
1784                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1785                         if st != os.EX_OK:
1786                                 print(red("!!!") + " metadata.xml is invalid:")
1787                                 for z in out.splitlines():
1788                                         print(red("!!! ") + z)
1789                                 stats["metadata.bad"] += 1
1790                                 fails["metadata.bad"].append(x + "/metadata.xml")
1791
1792                 del metadata_bad
1793         muselist = frozenset(musedict)
1794
1795         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1796         changelog_modified = changelog_path in modified_changelogs
1797
1798         # detect unused local USE-descriptions
1799         used_useflags = set()
1800
1801         for y in ebuildlist:
1802                 relative_path = os.path.join(x, y + ".ebuild")
1803                 full_path = os.path.join(repodir, relative_path)
1804                 ebuild_path = y + ".ebuild"
1805                 if repolevel < 3:
1806                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1807                 if repolevel < 2:
1808                         ebuild_path = os.path.join(catdir, ebuild_path)
1809                 ebuild_path = os.path.join(".", ebuild_path)
1810                 if check_changelog and not changelog_modified \
1811                         and ebuild_path in new_ebuilds:
1812                         stats['changelog.ebuildadded'] += 1
1813                         fails['changelog.ebuildadded'].append(relative_path)
1814
1815                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1816                         #ebuild not added to vcs
1817                         stats["ebuild.notadded"] += 1
1818                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1819                 myesplit = portage.pkgsplit(y)
1820                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1821                             or pv_toolong_re.search(myesplit[1]) \
1822                             or pv_toolong_re.search(myesplit[2]):
1823                         stats["ebuild.invalidname"] += 1
1824                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1825                         continue
1826                 elif myesplit[0] != pkgdir:
1827                         print(pkgdir, myesplit[0])
1828                         stats["ebuild.namenomatch"] += 1
1829                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1830                         continue
1831
1832                 pkg = pkgs[y]
1833
1834                 if pkg.invalid:
1835                         allvalid = False
1836                         for k, msgs in pkg.invalid.items():
1837                                 for msg in msgs:
1838                                         stats[k] += 1
1839                                         fails[k].append("%s: %s" % (relative_path, msg))
1840                         continue
1841
1842                 myaux = pkg._metadata
1843                 eapi = myaux["EAPI"]
1844                 inherited = pkg.inherited
1845                 live_ebuild = live_eclasses.intersection(inherited)
1846
1847                 if repo_config.eapi_is_banned(eapi):
1848                         stats["repo.eapi.banned"] += 1
1849                         fails["repo.eapi.banned"].append(
1850                                 "%s: %s" % (relative_path, eapi))
1851
1852                 elif repo_config.eapi_is_deprecated(eapi):
1853                         stats["repo.eapi.deprecated"] += 1
1854                         fails["repo.eapi.deprecated"].append(
1855                                 "%s: %s" % (relative_path, eapi))
1856
1857                 for k, v in myaux.items():
1858                         if not isinstance(v, basestring):
1859                                 continue
1860                         m = non_ascii_re.search(v)
1861                         if m is not None:
1862                                 stats["variable.invalidchar"] += 1
1863                                 fails["variable.invalidchar"].append(
1864                                         ("%s: %s variable contains non-ASCII " + \
1865                                         "character at position %s") % \
1866                                         (relative_path, k, m.start() + 1))
1867
1868                 if not src_uri_error:
1869                         # Check that URIs don't reference a server from thirdpartymirrors.
1870                         for uri in portage.dep.use_reduce( \
1871                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1872                                 contains_mirror = False
1873                                 for mirror, mirror_alias in thirdpartymirrors.items():
1874                                         if uri.startswith(mirror):
1875                                                 contains_mirror = True
1876                                                 break
1877                                 if not contains_mirror:
1878                                         continue
1879
1880                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1881                                 stats["SRC_URI.mirror"] += 1
1882                                 fails["SRC_URI.mirror"].append(
1883                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1884                                         (relative_path, mirror, new_uri))
1885
1886                 if myaux.get("PROVIDE"):
1887                         stats["virtual.oldstyle"] += 1
1888                         fails["virtual.oldstyle"].append(relative_path)
1889
1890                 for pos, missing_var in enumerate(missingvars):
1891                         if not myaux.get(missing_var):
1892                                 if catdir == "virtual" and \
1893                                         missing_var in ("HOMEPAGE", "LICENSE"):
1894                                         continue
1895                                 if live_ebuild and missing_var == "KEYWORDS":
1896                                         continue
1897                                 myqakey = missingvars[pos] + ".missing"
1898                                 stats[myqakey] += 1
1899                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1900
1901                 if catdir == "virtual":
1902                         for var in ("HOMEPAGE", "LICENSE"):
1903                                 if myaux.get(var):
1904                                         myqakey = var + ".virtual"
1905                                         stats[myqakey] += 1
1906                                         fails[myqakey].append(relative_path)
1907
1908                 # 14 is the length of DESCRIPTION=""
1909                 if len(myaux['DESCRIPTION']) > max_desc_len:
1910                         stats['DESCRIPTION.toolong'] += 1
1911                         fails['DESCRIPTION.toolong'].append(
1912                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1913                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1914
1915                 keywords = myaux["KEYWORDS"].split()
1916                 stable_keywords = []
1917                 for keyword in keywords:
1918                         if not keyword.startswith("~") and \
1919                                 not keyword.startswith("-"):
1920                                 stable_keywords.append(keyword)
1921                 if stable_keywords:
1922                         if ebuild_path in new_ebuilds and catdir != "virtual":
1923                                 stable_keywords.sort()
1924                                 stats["KEYWORDS.stable"] += 1
1925                                 fails["KEYWORDS.stable"].append(
1926                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1927                                                 " ".join(stable_keywords))
1928
1929                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1930                         if not kw.startswith("-"))
1931
1932                 previous_keywords = slot_keywords.get(pkg.slot)
1933                 if previous_keywords is None:
1934                         slot_keywords[pkg.slot] = set()
1935                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1936                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1937                         if dropped_keywords:
1938                                 stats["KEYWORDS.dropped"] += 1
1939                                 fails["KEYWORDS.dropped"].append(
1940                                         relative_path + ": %s" % \
1941                                         " ".join(sorted(dropped_keywords)))
1942
1943                 slot_keywords[pkg.slot].update(ebuild_archs)
1944
1945                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1946                 if "-*" in keywords:
1947                         haskeyword = False
1948                         for kw in keywords:
1949                                 if kw[0] == "~":
1950                                         kw = kw[1:]
1951                                 if kw in kwlist:
1952                                         haskeyword = True
1953                         if not haskeyword:
1954                                 stats["KEYWORDS.stupid"] += 1
1955                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1956
1957                 """
1958                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1959                 not be allowed to be marked stable
1960                 """
1961                 if live_ebuild and repo_config.name == "gentoo":
1962                         bad_stable_keywords = []
1963                         for keyword in keywords:
1964                                 if not keyword.startswith("~") and \
1965                                         not keyword.startswith("-"):
1966                                         bad_stable_keywords.append(keyword)
1967                                 del keyword
1968                         if bad_stable_keywords:
1969                                 stats["LIVEVCS.stable"] += 1
1970                                 fails["LIVEVCS.stable"].append(
1971                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1972                                                 bad_stable_keywords)
1973                         del bad_stable_keywords
1974
1975                         if keywords and not has_global_mask(pkg):
1976                                 stats["LIVEVCS.unmasked"] += 1
1977                                 fails["LIVEVCS.unmasked"].append(relative_path)
1978
1979                 if options.ignore_arches:
1980                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1981                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1982                 else:
1983                         arches = set()
1984                         for keyword in keywords:
1985                                 if keyword[0] == "-":
1986                                         continue
1987                                 elif keyword[0] == "~":
1988                                         arch = keyword[1:]
1989                                         if arch == "*":
1990                                                 for expanded_arch in profiles:
1991                                                         if expanded_arch == "**":
1992                                                                 continue
1993                                                         arches.add((keyword, expanded_arch,
1994                                                                 (expanded_arch, "~" + expanded_arch)))
1995                                         else:
1996                                                 arches.add((keyword, arch, (arch, keyword)))
1997                                 else:
1998                                         if keyword == "*":
1999                                                 for expanded_arch in profiles:
2000                                                         if expanded_arch == "**":
2001                                                                 continue
2002                                                         arches.add((keyword, expanded_arch,
2003                                                                 (expanded_arch,)))
2004                                         else:
2005                                                 arches.add((keyword, keyword, (keyword,)))
2006                         if not arches:
2007                                 # Use an empty profile for checking dependencies of
2008                                 # packages that have empty KEYWORDS.
2009                                 arches.add(('**', '**', ('**',)))
2010
2011                 unknown_pkgs = set()
2012                 baddepsyntax = False
2013                 badlicsyntax = False
2014                 badprovsyntax = False
2015                 catpkg = catdir + "/" + y
2016
2017                 inherited_java_eclass = "java-pkg-2" in inherited or \
2018                         "java-pkg-opt-2" in inherited
2019                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
2020                 operator_tokens = set(["||", "(", ")"])
2021                 type_list, badsyntax = [], []
2022                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
2023                         mydepstr = myaux[mytype]
2024
2025                         buildtime = mytype in Package._buildtime_keys
2026                         runtime = mytype in Package._runtime_keys
2027                         token_class = None
2028                         if mytype.endswith("DEPEND"):
2029                                 token_class = portage.dep.Atom
2030
2031                         try:
2032                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
2033                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
2034                         except portage.exception.InvalidDependString as e:
2035                                 atoms = None
2036                                 badsyntax.append(str(e))
2037
2038                         if atoms and mytype.endswith("DEPEND"):
2039                                 if runtime and \
2040                                         "test?" in mydepstr.split():
2041                                         stats[mytype + '.suspect'] += 1
2042                                         fails[mytype + '.suspect'].append(relative_path + \
2043                                                 ": 'test?' USE conditional in %s" % mytype)
2044
2045                                 for atom in atoms:
2046                                         if atom == "||":
2047                                                 continue
2048
2049                                         # Skip dependency.unknown for blockers, so that we
2050                                         # don't encourage people to remove necessary blockers,
2051                                         # as discussed in bug #382407.
2052                                         if atom.blocker is None and \
2053                                                 not portdb.xmatch("match-all", atom) and \
2054                                                 not atom.cp.startswith("virtual/"):
2055                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2056
2057                                         is_blocker = atom.blocker
2058
2059                                         if catdir != "virtual":
2060                                                 if not is_blocker and \
2061                                                         atom.cp in suspect_virtual:
2062                                                         stats['virtual.suspect'] += 1
2063                                                         fails['virtual.suspect'].append(
2064                                                                 relative_path +
2065                                                                 ": %s: consider using '%s' instead of '%s'" %
2066                                                                 (mytype, suspect_virtual[atom.cp], atom))
2067
2068                                         if buildtime and \
2069                                                 not is_blocker and \
2070                                                 not inherited_java_eclass and \
2071                                                 atom.cp == "virtual/jdk":
2072                                                 stats['java.eclassesnotused'] += 1
2073                                                 fails['java.eclassesnotused'].append(relative_path)
2074                                         elif buildtime and \
2075                                                 not is_blocker and \
2076                                                 not inherited_wxwidgets_eclass and \
2077                                                 atom.cp == "x11-libs/wxGTK":
2078                                                 stats['wxwidgets.eclassnotused'] += 1
2079                                                 fails['wxwidgets.eclassnotused'].append(
2080                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2081                                                         " without inheriting wxwidgets.eclass") % mytype)
2082                                         elif runtime:
2083                                                 if not is_blocker and \
2084                                                         atom.cp in suspect_rdepend:
2085                                                         stats[mytype + '.suspect'] += 1
2086                                                         fails[mytype + '.suspect'].append(
2087                                                                 relative_path + ": '%s'" % atom)
2088
2089                                         if atom.operator == "~" and \
2090                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2091                                                 qacat = 'dependency.badtilde'
2092                                                 stats[qacat] += 1
2093                                                 fails[qacat].append(
2094                                                         (relative_path + ": %s uses the ~ operator"
2095                                                          " with a non-zero revision:" + \
2096                                                          " '%s'") % (mytype, atom))
2097
2098                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2099
2100                 for m, b in zip(type_list, badsyntax):
2101                         if m.endswith("DEPEND"):
2102                                 qacat = "dependency.syntax"
2103                         else:
2104                                 qacat = m + ".syntax"
2105                         stats[qacat] += 1
2106                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2107
2108                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2109                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2110                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2111                 badlicsyntax = badlicsyntax > 0
2112                 badprovsyntax = badprovsyntax > 0
2113
2114                 # uselist checks - global
2115                 myuse = []
2116                 default_use = []
2117                 for myflag in myaux["IUSE"].split():
2118                         flag_name = myflag.lstrip("+-")
2119                         used_useflags.add(flag_name)
2120                         if myflag != flag_name:
2121                                 default_use.append(myflag)
2122                         if flag_name not in uselist:
2123                                 myuse.append(flag_name)
2124
2125                 # uselist checks - metadata
2126                 for mypos in range(len(myuse)-1, -1, -1):
2127                         if myuse[mypos] and (myuse[mypos] in muselist):
2128                                 del myuse[mypos]
2129
2130                 if default_use and not eapi_has_iuse_defaults(eapi):
2131                         for myflag in default_use:
2132                                 stats['EAPI.incompatible'] += 1
2133                                 fails['EAPI.incompatible'].append(
2134                                         (relative_path + ": IUSE defaults" + \
2135                                         " not supported with EAPI='%s':" + \
2136                                         " '%s'") % (eapi, myflag))
2137
2138                 for mypos in range(len(myuse)):
2139                         stats["IUSE.invalid"] += 1
2140                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2141
2142                 # license checks
2143                 if not badlicsyntax:
2144                         # Parse the LICENSE variable, remove USE conditions and
2145                         # flatten it.
2146                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2147                         # Check each entry to ensure that it exists in PORTDIR's
2148                         # license directory.
2149                         for lic in licenses:
2150                                 # Need to check for "||" manually as no portage
2151                                 # function will remove it without removing values.
2152                                 if lic not in liclist and lic != "||":
2153                                         stats["LICENSE.invalid"] += 1
2154                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2155                                 elif lic in liclist_deprecated:
2156                                         stats["LICENSE.deprecated"] += 1
2157                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2158
2159                 #keyword checks
2160                 myuse = myaux["KEYWORDS"].split()
2161                 for mykey in myuse:
2162                         if mykey not in ("-*", "*", "~*"):
2163                                 myskey = mykey
2164                                 if myskey[:1] == "-":
2165                                         myskey = myskey[1:]
2166                                 if myskey[:1] == "~":
2167                                         myskey = myskey[1:]
2168                                 if myskey not in kwlist:
2169                                         stats["KEYWORDS.invalid"] += 1
2170                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2171                                 elif myskey not in profiles:
2172                                         stats["KEYWORDS.invalid"] += 1
2173                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2174
2175                 #restrict checks
2176                 myrestrict = None
2177                 try:
2178                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2179                 except portage.exception.InvalidDependString as e:
2180                         stats["RESTRICT.syntax"] += 1
2181                         fails["RESTRICT.syntax"].append(
2182                                 "%s: RESTRICT: %s" % (relative_path, e))
2183                         del e
2184                 if myrestrict:
2185                         myrestrict = set(myrestrict)
2186                         mybadrestrict = myrestrict.difference(valid_restrict)
2187                         if mybadrestrict:
2188                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2189                                 for mybad in mybadrestrict:
2190                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2191                 #REQUIRED_USE check
2192                 required_use = myaux["REQUIRED_USE"]
2193                 if required_use:
2194                         if not eapi_has_required_use(eapi):
2195                                 stats['EAPI.incompatible'] += 1
2196                                 fails['EAPI.incompatible'].append(
2197                                         relative_path + ": REQUIRED_USE" + \
2198                                         " not supported with EAPI='%s'" % (eapi,))
2199                         try:
2200                                 portage.dep.check_required_use(required_use, (),
2201                                         pkg.iuse.is_valid_flag, eapi=eapi)
2202                         except portage.exception.InvalidDependString as e:
2203                                 stats["REQUIRED_USE.syntax"] += 1
2204                                 fails["REQUIRED_USE.syntax"].append(
2205                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2206                                 del e
2207
2208                 # Syntax Checks
2209                 relative_path = os.path.join(x, y + ".ebuild")
2210                 full_path = os.path.join(repodir, relative_path)
2211                 if not vcs_preserves_mtime:
2212                         if ebuild_path not in new_ebuilds and \
2213                                 ebuild_path not in modified_ebuilds:
2214                                 pkg.mtime = None
2215                 try:
2216                         # All ebuilds should have utf_8 encoding.
2217                         f = io.open(_unicode_encode(full_path,
2218                                 encoding=_encodings['fs'], errors='strict'),
2219                                 mode='r', encoding=_encodings['repo.content'])
2220                         try:
2221                                 for check_name, e in run_checks(f, pkg):
2222                                         stats[check_name] += 1
2223                                         fails[check_name].append(relative_path + ': %s' % e)
2224                         finally:
2225                                 f.close()
2226                 except UnicodeDecodeError:
2227                         # A file.UTF8 failure will have already been recorded above.
2228                         pass
2229
2230                 if options.force:
2231                         # The dep_check() calls are the most expensive QA test. If --force
2232                         # is enabled, there's no point in wasting time on these since the
2233                         # user is intent on forcing the commit anyway.
2234                         continue
2235
2236                 relevant_profiles = []
2237                 for keyword, arch, groups in arches:
2238                         if arch not in profiles:
2239                                 # A missing profile will create an error further down
2240                                 # during the KEYWORDS verification.
2241                                 continue
2242
2243                         if include_arches is not None:
2244                                 if arch not in include_arches:
2245                                         continue
2246
2247                         relevant_profiles.extend((keyword, groups, prof)
2248                                 for prof in profiles[arch])
2249
2250                 def sort_key(item):
2251                         return item[2].sub_path
2252
2253                 relevant_profiles.sort(key=sort_key)
2254
2255                 for keyword, groups, prof in relevant_profiles:
2256
2257                                 if prof.status not in ("stable", "dev") or \
2258                                         prof.status == "dev" and not options.include_dev:
2259                                         continue
2260
2261                                 dep_settings = arch_caches.get(prof.sub_path)
2262                                 if dep_settings is None:
2263                                         dep_settings = portage.config(
2264                                                 config_profile_path=prof.abs_path,
2265                                                 config_incrementals=repoman_incrementals,
2266                                                 config_root=config_root,
2267                                                 local_config=False,
2268                                                 _unmatched_removal=options.unmatched_removal,
2269                                                 env=env, repositories=repoman_settings.repositories)
2270                                         dep_settings.categories = repoman_settings.categories
2271                                         if options.without_mask:
2272                                                 dep_settings._mask_manager_obj = \
2273                                                         copy.deepcopy(dep_settings._mask_manager)
2274                                                 dep_settings._mask_manager._pmaskdict.clear()
2275                                         arch_caches[prof.sub_path] = dep_settings
2276
2277                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2278                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2279                                 if xcache is None:
2280                                         portdb.melt()
2281                                         portdb.freeze()
2282                                         xcache = portdb.xcache
2283                                         xcache.update(shared_xmatch_caches)
2284                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2285
2286                                 trees[root]["porttree"].settings = dep_settings
2287                                 portdb.settings = dep_settings
2288                                 portdb.xcache = xcache
2289
2290                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2291                                 # just in case, prevent config.reset() from nuking these.
2292                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2293
2294                                 # This attribute is used in dbapi._match_use() to apply
2295                                 # use.stable.{mask,force} settings based on the stable
2296                                 # status of the parent package. This is required in order
2297                                 # for USE deps of unstable packages to be resolved correctly,
2298                                 # since otherwise use.stable.{mask,force} settings of
2299                                 # dependencies may conflict (see bug #456342).
2300                                 dep_settings._parent_stable = dep_settings._isStable(pkg)
2301
2302                                 # Handle package.use*.{force,mask) calculation, for use
2303                                 # in dep_check.
2304                                 dep_settings.useforce = dep_settings._use_manager.getUseForce(
2305                                         pkg, stable=dep_settings._parent_stable)
2306                                 dep_settings.usemask = dep_settings._use_manager.getUseMask(
2307                                         pkg, stable=dep_settings._parent_stable)
2308
2309                                 if not baddepsyntax:
2310                                         ismasked = not ebuild_archs or \
2311                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2312                                         if ismasked:
2313                                                 if not have_pmasked:
2314                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2315                                                                 pkg.cpv, pkg._metadata))
2316                                                 if options.ignore_masked:
2317                                                         continue
2318                                                 #we are testing deps for a masked package; give it some lee-way
2319                                                 suffix = "masked"
2320                                                 matchmode = "minimum-all"
2321                                         else:
2322                                                 suffix = ""
2323                                                 matchmode = "minimum-visible"
2324
2325                                         if not have_dev_keywords:
2326                                                 have_dev_keywords = \
2327                                                         bool(dev_keywords.intersection(keywords))
2328
2329                                         if prof.status == "dev":
2330                                                 suffix = suffix + "indev"
2331
2332                                         for mytype in Package._dep_keys:
2333
2334                                                 mykey = "dependency.bad" + suffix
2335                                                 myvalue = myaux[mytype]
2336                                                 if not myvalue:
2337                                                         continue
2338
2339                                                 success, atoms = portage.dep_check(myvalue, portdb,
2340                                                         dep_settings, use="all", mode=matchmode,
2341                                                         trees=trees)
2342
2343                                                 if success:
2344                                                         if atoms:
2345
2346                                                                 # Don't bother with dependency.unknown for
2347                                                                 # cases in which *DEPEND.bad is triggered.
2348                                                                 for atom in atoms:
2349                                                                         # dep_check returns all blockers and they
2350                                                                         # aren't counted for *DEPEND.bad, so we
2351                                                                         # ignore them here.
2352                                                                         if not atom.blocker:
2353                                                                                 unknown_pkgs.discard(
2354                                                                                         (mytype, atom.unevaluated_atom))
2355
2356                                                                 if not prof.sub_path:
2357                                                                         # old-style virtuals currently aren't
2358                                                                         # resolvable with empty profile, since
2359                                                                         # 'virtuals' mappings are unavailable
2360                                                                         # (it would be expensive to search
2361                                                                         # for PROVIDE in all ebuilds)
2362                                                                         atoms = [atom for atom in atoms if not \
2363                                                                                 (atom.cp.startswith('virtual/') and \
2364                                                                                 not portdb.cp_list(atom.cp))]
2365
2366                                                                 #we have some unsolvable deps
2367                                                                 #remove ! deps, which always show up as unsatisfiable
2368                                                                 atoms = [str(atom.unevaluated_atom) \
2369                                                                         for atom in atoms if not atom.blocker]
2370
2371                                                                 #if we emptied out our list, continue:
2372                                                                 if not atoms:
2373                                                                         continue
2374                                                                 stats[mykey] += 1
2375                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2376                                                                         (relative_path, mytype, keyword,
2377                                                                         prof, repr(atoms)))
2378                                                 else:
2379                                                         stats[mykey] += 1
2380                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2381                                                                 (relative_path, mytype, keyword,
2382                                                                 prof, repr(atoms)))
2383
2384                 if not baddepsyntax and unknown_pkgs:
2385                         type_map = {}
2386                         for mytype, atom in unknown_pkgs:
2387                                 type_map.setdefault(mytype, set()).add(atom)
2388                         for mytype, atoms in type_map.items():
2389                                 stats["dependency.unknown"] += 1
2390                                 fails["dependency.unknown"].append("%s: %s: %s" %
2391                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2392
2393         # check if there are unused local USE-descriptions in metadata.xml
2394         # (unless there are any invalids, to avoid noise)
2395         if allvalid:
2396                 for myflag in muselist.difference(used_useflags):
2397                         stats["metadata.warning"] += 1
2398                         fails["metadata.warning"].append(
2399                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2400                                 (x, myflag))
2401
2402 if options.if_modified == "y" and len(effective_scanlist) < 1:
2403         logging.warn("--if-modified is enabled, but no modified packages were found!")
2404
2405 if options.mode == "manifest":
2406         sys.exit(dofail)
2407
2408 # dofail will be set to 1 if we have failed in at least one non-warning category
2409 dofail = 0
2410 # dowarn will be set to 1 if we tripped any warnings
2411 dowarn = 0
2412 # dofull will be set if we should print a "repoman full" informational message
2413 dofull = options.mode != 'full'
2414
2415 for x in qacats:
2416         if not stats[x]:
2417                 continue
2418         dowarn = 1
2419         if x not in qawarnings:
2420                 dofail = 1
2421
2422 if dofail or \
2423         (dowarn and not (options.quiet or options.mode == "scan")):
2424         dofull = 0
2425
2426 # Save QA output so that it can be conveniently displayed
2427 # in $EDITOR while the user creates a commit message.
2428 # Otherwise, the user would not be able to see this output
2429 # once the editor has taken over the screen.
2430 qa_output = io.StringIO()
2431 style_file = ConsoleStyleFile(sys.stdout)
2432 if options.mode == 'commit' and \
2433         (not commitmessage or not commitmessage.strip()):
2434         style_file.write_listener = qa_output
2435 console_writer = StyleWriter(file=style_file, maxcol=9999)
2436 console_writer.style_listener = style_file.new_styles
2437
2438 f = formatter.AbstractFormatter(console_writer)
2439
2440 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2441
2442 style_file.flush()
2443 del console_writer, f, style_file
2444 qa_output = qa_output.getvalue()
2445 qa_output = qa_output.splitlines(True)
2446
2447 suggest_ignore_masked = False
2448 suggest_include_dev = False
2449
2450 if have_pmasked and not (options.without_mask or options.ignore_masked):
2451         suggest_ignore_masked = True
2452 if have_dev_keywords and not options.include_dev:
2453         suggest_include_dev = True
2454
2455 if suggest_ignore_masked or suggest_include_dev:
2456         print()
2457         if suggest_ignore_masked:
2458                 print(bold("Note: use --without-mask to check " + \
2459                         "KEYWORDS on dependencies of masked packages"))
2460
2461         if suggest_include_dev:
2462                 print(bold("Note: use --include-dev (-d) to check " + \
2463                         "dependencies for 'dev' profiles"))
2464         print()
2465
2466 if options.mode != 'commit':
2467         if dofull:
2468                 print(bold("Note: type \"repoman full\" for a complete listing."))
2469         if dowarn and not dofail:
2470                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2471         elif not dofail:
2472                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2473         elif dofail:
2474                 print(bad("Please fix these important QA issues first."))
2475                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2476                 sys.exit(1)
2477 else:
2478         if dofail and can_force and options.force and not options.pretend:
2479                 print(green("RepoMan sez:") + \
2480                         " \"You want to commit even with these QA issues?\n" + \
2481                         "              I'll take it this time, but I'm not happy.\"\n")
2482         elif dofail:
2483                 if options.force and not can_force:
2484                         print(bad("The --force option has been disabled due to extraordinary issues."))
2485                 print(bad("Please fix these important QA issues first."))
2486                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2487                 sys.exit(1)
2488
2489         if options.pretend:
2490                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2491
2492         myunadded = []
2493         if vcs == "cvs":
2494                 try:
2495                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2496                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2497                 except SystemExit as e:
2498                         raise  # TODO propagate this
2499                 except:
2500                         err("Error retrieving CVS tree; exiting.")
2501         if vcs == "svn":
2502                 try:
2503                         with repoman_popen("svn status --no-ignore") as f:
2504                                 svnstatus = f.readlines()
2505                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2506                 except SystemExit as e:
2507                         raise  # TODO propagate this
2508                 except:
2509                         err("Error retrieving SVN info; exiting.")
2510         if vcs == "git":
2511                 # get list of files not under version control or missing
2512                 myf = repoman_popen("git ls-files --others")
2513                 myunadded = ["./" + elem[:-1] for elem in myf]
2514                 myf.close()
2515         if vcs == "bzr":
2516                 try:
2517                         with repoman_popen("bzr status -S .") as f:
2518                                 bzrstatus = f.readlines()
2519                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2520                 except SystemExit as e:
2521                         raise  # TODO propagate this
2522                 except:
2523                         err("Error retrieving bzr info; exiting.")
2524         if vcs == "hg":
2525                 with repoman_popen("hg status --no-status --unknown .") as f:
2526                         myunadded = f.readlines()
2527                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2528
2529                 # Mercurial doesn't handle manually deleted files as removed from
2530                 # the repository, so the user need to remove them before commit,
2531                 # using "hg remove [FILES]"
2532                 with repoman_popen("hg status --no-status --deleted .") as f:
2533                         mydeleted = f.readlines()
2534                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2535
2536
2537         myautoadd = []
2538         if myunadded:
2539                 for x in range(len(myunadded)-1, -1, -1):
2540                         xs = myunadded[x].split("/")
2541                         if xs[-1] == "files":
2542                                 print("!!! files dir is not added! Please correct this.")
2543                                 sys.exit(-1)
2544                         elif xs[-1] == "Manifest":
2545                                 # It's a manifest... auto add
2546                                 myautoadd += [myunadded[x]]
2547                                 del myunadded[x]
2548
2549         if myunadded:
2550                 print(red("!!! The following files are in your local tree but are not added to the master"))
2551                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2552                 for x in myunadded:
2553                         print("   ", x)
2554                 print()
2555                 print()
2556                 sys.exit(1)
2557
2558         if vcs == "hg" and mydeleted:
2559                 print(red("!!! The following files are removed manually from your local tree but are not"))
2560                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2561                 for x in mydeleted:
2562                         print("   ", x)
2563                 print()
2564                 print()
2565                 sys.exit(1)
2566
2567         if vcs == "cvs":
2568                 mycvstree = cvstree.getentries("./", recursive=1)
2569                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2570                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2571                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2572                 bin_blob_pattern = re.compile("^-kb$")
2573                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2574                         recursive=1, basedir="./"))
2575
2576         if vcs == "svn":
2577                 with repoman_popen("svn status") as f:
2578                         svnstatus = f.readlines()
2579                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2580                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2581                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2582
2583                 # Subversion expands keywords specified in svn:keywords properties.
2584                 with repoman_popen("svn propget -R svn:keywords") as f:
2585                         props = f.readlines()
2586                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2587                         for prop in props if " - " in prop)
2588
2589         elif vcs == "git":
2590                 with repoman_popen("git diff-index --name-only "
2591                         "--relative --diff-filter=M HEAD") as f:
2592                         mychanged = f.readlines()
2593                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2594
2595                 with repoman_popen("git diff-index --name-only "
2596                         "--relative --diff-filter=A HEAD") as f:
2597                         mynew = f.readlines()
2598                 mynew = ["./" + elem[:-1] for elem in mynew]
2599
2600                 with repoman_popen("git diff-index --name-only "
2601                         "--relative --diff-filter=D HEAD") as f:
2602                         myremoved = f.readlines()
2603                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2604
2605         if vcs == "bzr":
2606                 with repoman_popen("bzr status -S .") as f:
2607                         bzrstatus = f.readlines()
2608                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2609                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2610                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2611                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2612                 # Bazaar expands nothing.
2613
2614         if vcs == "hg":
2615                 with repoman_popen("hg status --no-status --modified .") as f:
2616                         mychanged = f.readlines()
2617                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2618
2619                 with repoman_popen("hg status --no-status --added .") as f:
2620                         mynew = f.readlines()
2621                 mynew = ["./" + elem.rstrip() for elem in mynew]
2622
2623                 with repoman_popen("hg status --no-status --removed .") as f:
2624                         myremoved = f.readlines()
2625                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2626
2627         if vcs:
2628                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2629                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2630                         print()
2631                         print("(Didn't find any changed files...)")
2632                         print()
2633                         sys.exit(1)
2634
2635         # Manifests need to be regenerated after all other commits, so don't commit
2636         # them now even if they have changed.
2637         mymanifests = set()
2638         myupdates = set()
2639         for f in mychanged + mynew:
2640                 if "Manifest" == os.path.basename(f):
2641                         mymanifests.add(f)
2642                 else:
2643                         myupdates.add(f)
2644         myupdates.difference_update(myremoved)
2645         myupdates = list(myupdates)
2646         mymanifests = list(mymanifests)
2647         myheaders = []
2648         mydirty = []
2649
2650         commitmessage = options.commitmsg
2651         if options.commitmsgfile:
2652                 try:
2653                         f = io.open(_unicode_encode(options.commitmsgfile,
2654                         encoding=_encodings['fs'], errors='strict'),
2655                         mode='r', encoding=_encodings['content'], errors='replace')
2656                         commitmessage = f.read()
2657                         f.close()
2658                         del f
2659                 except (IOError, OSError) as e:
2660                         if e.errno == errno.ENOENT:
2661                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2662                         else:
2663                                 raise
2664                 # We've read the content so the file is no longer needed.
2665                 commitmessagefile = None
2666         if not commitmessage or not commitmessage.strip():
2667                 try:
2668                         editor = os.environ.get("EDITOR")
2669                         if editor and utilities.editor_is_executable(editor):
2670                                 commitmessage = utilities.get_commit_message_with_editor(
2671                                         editor, message=qa_output)
2672                         else:
2673                                 commitmessage = utilities.get_commit_message_with_stdin()
2674                 except KeyboardInterrupt:
2675                         exithandler()
2676                 if not commitmessage or not commitmessage.strip():
2677                         print("* no commit message?  aborting commit.")
2678                         sys.exit(1)
2679         commitmessage = commitmessage.rstrip()
2680         changelog_msg = commitmessage
2681         portage_version = getattr(portage, "VERSION", None)
2682         gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
2683         dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
2684         if portage_version is None:
2685                 sys.stderr.write("Failed to insert portage version in message!\n")
2686                 sys.stderr.flush()
2687                 portage_version = "Unknown"
2688
2689         report_options = []
2690         if options.force:
2691                 report_options.append("--force")
2692         if options.ignore_arches:
2693                 report_options.append("--ignore-arches")
2694         if include_arches is not None:
2695                 report_options.append("--include-arches=\"%s\"" %
2696                         " ".join(sorted(include_arches)))
2697
2698         if vcs == "git":
2699                 # Use new footer only for git (see bug #438364).
2700                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2701                 if report_options:
2702                         commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
2703                 if sign_manifests:
2704                         commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
2705                 if dco_sob:
2706                         commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
2707         else:
2708                 unameout = platform.system() + " "
2709                 if platform.system() in ["Darwin", "SunOS"]:
2710                         unameout += platform.processor()
2711                 else:
2712                         unameout += platform.machine()
2713                 commit_footer = "\n\n"
2714                 if dco_sob:
2715                         commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
2716                 commit_footer += "(Portage version: %s/%s/%s" % \
2717                         (portage_version, vcs, unameout)
2718                 if report_options:
2719                         commit_footer += ", RepoMan options: " + " ".join(report_options)
2720                 if sign_manifests:
2721                         commit_footer += ", signed Manifest commit with key %s" % \
2722                                 (gpg_key, )
2723                 else:
2724                         commit_footer += ", unsigned Manifest commit"
2725                 commit_footer += ")"
2726
2727         commitmessage += commit_footer
2728
2729         if options.echangelog in ('y', 'force'):
2730                 logging.info("checking for unmodified ChangeLog files")
2731                 committer_name = utilities.get_committer_name(env=repoman_settings)
2732                 for x in sorted(vcs_files_to_cps(
2733                         chain(myupdates, mymanifests, myremoved))):
2734                         catdir, pkgdir = x.split("/")
2735                         checkdir = repodir + "/" + x
2736                         checkdir_relative = ""
2737                         if repolevel < 3:
2738                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2739                         if repolevel < 2:
2740                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2741                         checkdir_relative = os.path.join(".", checkdir_relative)
2742
2743                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2744                         changelog_modified = changelog_path in modified_changelogs
2745                         if changelog_modified and options.echangelog != 'force':
2746                                 continue
2747
2748                         # get changes for this package
2749                         cdrlen = len(checkdir_relative)
2750                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2751                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2752                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2753
2754                         # Skip ChangeLog generation if only the Manifest was modified,
2755                         # as discussed in bug #398009.
2756                         nontrivial_cl_files = set()
2757                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2758                         nontrivial_cl_files.difference_update(['Manifest'])
2759                         if not nontrivial_cl_files and options.echangelog != 'force':
2760                                 continue
2761
2762                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2763                                 committer_name, changelog_msg,
2764                                 os.path.join(repodir, 'skel.ChangeLog'),
2765                                 catdir, pkgdir,
2766                                 new=clnew, removed=clremoved, changed=clchanged,
2767                                 pretend=options.pretend)
2768                         if new_changelog is None:
2769                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2770                                         level=logging.ERROR, noiselevel=-1)
2771                                 sys.exit(1)
2772
2773                         # if the ChangeLog was just created, add it to vcs
2774                         if new_changelog:
2775                                 myautoadd.append(changelog_path)
2776                                 # myautoadd is appended to myupdates below
2777                         else:
2778                                 myupdates.append(changelog_path)
2779
2780                         if options.ask and not options.pretend:
2781                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2782                                 repoman_settings["O"] = checkdir
2783                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2784
2785         if myautoadd:
2786                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2787                 add_cmd = [vcs, "add"]
2788                 add_cmd += myautoadd
2789                 if options.pretend:
2790                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2791                                 noiselevel=-1)
2792                 else:
2793
2794                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2795                                 not os.path.isabs(add_cmd[0]):
2796                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2797                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2798                                 fullname = find_binary(add_cmd[0])
2799                                 if fullname is None:
2800                                         raise portage.exception.CommandNotFound(add_cmd[0])
2801                                 add_cmd[0] = fullname
2802
2803                         add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2804                         retcode = subprocess.call(add_cmd)
2805                         if retcode != os.EX_OK:
2806                                 logging.error(
2807                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2808                                 sys.exit(retcode)
2809
2810                 myupdates += myautoadd
2811
2812         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2813
2814         if vcs not in ('cvs', 'svn'):
2815                 # With git, bzr and hg, there's never any keyword expansion, so
2816                 # there's no need to regenerate manifests and all files will be
2817                 # committed in one big commit at the end.
2818                 print()
2819         elif not repo_config.thin_manifest:
2820                 if vcs == 'cvs':
2821                         headerstring = "'\$(Header|Id).*\$'"
2822                 elif vcs == "svn":
2823                         svn_keywords = dict((k.lower(), k) for k in [
2824                                         "Rev",
2825                                         "Revision",
2826                                         "LastChangedRevision",
2827                                         "Date",
2828                                         "LastChangedDate",
2829                                         "Author",
2830                                         "LastChangedBy",
2831                                         "URL",
2832                                         "HeadURL",
2833                                         "Id",
2834                                         "Header",
2835                         ])
2836
2837                 for myfile in myupdates:
2838
2839                         # for CVS, no_expansion contains files that are excluded from expansion
2840                         if vcs == "cvs":
2841                                 if myfile in no_expansion:
2842                                         continue
2843
2844                         # for SVN, expansion contains files that are included in expansion
2845                         elif vcs == "svn":
2846                                 if myfile not in expansion:
2847                                         continue
2848
2849                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2850                                 enabled_keywords = []
2851                                 for k in expansion[myfile]:
2852                                         keyword = svn_keywords.get(k.lower())
2853                                         if keyword is not None:
2854                                                 enabled_keywords.append(keyword)
2855
2856                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2857
2858                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2859                                 portage._shell_quote(myfile))
2860                         if myout[0] == 0:
2861                                 myheaders.append(myfile)
2862
2863                 print("%s have headers that will change." % green(str(len(myheaders))))
2864                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2865
2866         logging.info("myupdates: %s", myupdates)
2867         logging.info("myheaders: %s", myheaders)
2868
2869         if options.ask and userquery('Commit changes?', True) != 'Yes':
2870                 print("* aborting commit.")
2871                 sys.exit(128 + signal.SIGINT)
2872
2873         # Handle the case where committed files have keywords which
2874         # will change and need a priming commit before the Manifest
2875         # can be committed.
2876         if (myupdates or myremoved) and myheaders:
2877                 myfiles = myupdates + myremoved
2878                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2879                 mymsg = os.fdopen(fd, "wb")
2880                 mymsg.write(_unicode_encode(commitmessage))
2881                 mymsg.close()
2882
2883                 print()
2884                 print(green("Using commit message:"))
2885                 print(green("------------------------------------------------------------------------------"))
2886                 print(commitmessage)
2887                 print(green("------------------------------------------------------------------------------"))
2888                 print()
2889
2890                 # Having a leading ./ prefix on file paths can trigger a bug in
2891                 # the cvs server when committing files to multiple directories,
2892                 # so strip the prefix.
2893                 myfiles = [f.lstrip("./") for f in myfiles]
2894
2895                 commit_cmd = [vcs]
2896                 commit_cmd.extend(vcs_global_opts)
2897                 commit_cmd.append("commit")
2898                 commit_cmd.extend(vcs_local_opts)
2899                 commit_cmd.extend(["-F", commitmessagefile])
2900                 commit_cmd.extend(myfiles)
2901
2902                 try:
2903                         if options.pretend:
2904                                 print("(%s)" % (" ".join(commit_cmd),))
2905                         else:
2906                                 retval = spawn(commit_cmd, env=commit_env)
2907                                 if retval != os.EX_OK:
2908                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2909                                                 "error code: %s\n") % (vcs, retval),
2910                                                 level=logging.ERROR, noiselevel=-1)
2911                                         sys.exit(retval)
2912                 finally:
2913                         try:
2914                                 os.unlink(commitmessagefile)
2915                         except OSError:
2916                                 pass
2917
2918         # Setup the GPG commands
2919         def gpgsign(filename):
2920                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2921                 if gpgcmd is None:
2922                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2923                                 " Is make.globals missing?")
2924                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2925                         "PORTAGE_GPG_KEY" not in repoman_settings:
2926                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2927                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2928                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2929                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2930                                         os.path.expanduser("~/.gnupg")
2931                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2932                                         % repoman_settings["PORTAGE_GPG_DIR"])
2933                         else:
2934                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2935                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2936                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2937                                 raise portage.exception.InvalidLocation(
2938                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2939                                         repoman_settings["PORTAGE_GPG_DIR"])
2940                 gpgvars = {"FILE": filename}
2941                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2942                         v = repoman_settings.get(k)
2943                         if v is not None:
2944                                 gpgvars[k] = v
2945                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2946                 if options.pretend:
2947                         print("(" + gpgcmd + ")")
2948                 else:
2949                         # Encode unicode manually for bug #310789.
2950                         gpgcmd = portage.util.shlex_split(gpgcmd)
2951
2952                         if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
2953                                 not os.path.isabs(gpgcmd[0]):
2954                                 # Python 3.1 _execvp throws TypeError for non-absolute executable
2955                                 # path passed as bytes (see http://bugs.python.org/issue8513).
2956                                 fullname = find_binary(gpgcmd[0])
2957                                 if fullname is None:
2958                                         raise portage.exception.CommandNotFound(gpgcmd[0])
2959                                 gpgcmd[0] = fullname
2960
2961                         gpgcmd = [_unicode_encode(arg,
2962                                 encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2963                         rValue = subprocess.call(gpgcmd)
2964                         if rValue == os.EX_OK:
2965                                 os.rename(filename + ".asc", filename)
2966                         else:
2967                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2968
2969         def need_signature(filename):
2970                 try:
2971                         with open(_unicode_encode(filename,
2972                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2973                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2974                 except IOError as e:
2975                         if e.errno in (errno.ENOENT, errno.ESTALE):
2976                                 return False
2977                         raise
2978
2979         # When files are removed and re-added, the cvs server will put /Attic/
2980         # inside the $Header path. This code detects the problem and corrects it
2981         # so that the Manifest will generate correctly. See bug #169500.
2982         # Use binary mode in order to avoid potential character encoding issues.
2983         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2984         attic_str = b'/Attic/'
2985         attic_replace = b'/'
2986         for x in myheaders:
2987                 f = open(_unicode_encode(x,
2988                         encoding=_encodings['fs'], errors='strict'),
2989                         mode='rb')
2990                 mylines = f.readlines()
2991                 f.close()
2992                 modified = False
2993                 for i, line in enumerate(mylines):
2994                         if cvs_header_re.match(line) is not None and \
2995                                 attic_str in line:
2996                                 mylines[i] = line.replace(attic_str, attic_replace)
2997                                 modified = True
2998                 if modified:
2999                         portage.util.write_atomic(x, b''.join(mylines),
3000                                 mode='wb')
3001
3002         if repolevel == 1:
3003                 print(green("RepoMan sez:"), "\"You're rather crazy... "
3004                         "doing the entire repository.\"\n")
3005
3006         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
3007
3008                 for x in sorted(vcs_files_to_cps(
3009                         chain(myupdates, myremoved, mymanifests))):
3010                         repoman_settings["O"] = os.path.join(repodir, x)
3011                         digestgen(mysettings=repoman_settings, myportdb=portdb)
3012
3013         signed = False
3014         if sign_manifests:
3015                 signed = True
3016                 try:
3017                         for x in sorted(vcs_files_to_cps(
3018                                 chain(myupdates, myremoved, mymanifests))):
3019                                 repoman_settings["O"] = os.path.join(repodir, x)
3020                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
3021                                 if not need_signature(manifest_path):
3022                                         continue
3023                                 gpgsign(manifest_path)
3024                 except portage.exception.PortageException as e:
3025                         portage.writemsg("!!! %s\n" % str(e))
3026                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
3027                         signed = False
3028
3029         if vcs == 'git':
3030                 # It's not safe to use the git commit -a option since there might
3031                 # be some modified files elsewhere in the working tree that the
3032                 # user doesn't want to commit. Therefore, call git update-index
3033                 # in order to ensure that the index is updated with the latest
3034                 # versions of all new and modified files in the relevant portion
3035                 # of the working tree.
3036                 myfiles = mymanifests + myupdates
3037                 myfiles.sort()
3038                 update_index_cmd = ["git", "update-index"]
3039                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
3040                 if options.pretend:
3041                         print("(%s)" % (" ".join(update_index_cmd),))
3042                 else:
3043                         retval = spawn(update_index_cmd, env=os.environ)
3044                         if retval != os.EX_OK:
3045                                 writemsg_level(("!!! Exiting on %s (shell) " + \
3046                                         "error code: %s\n") % (vcs, retval),
3047                                         level=logging.ERROR, noiselevel=-1)
3048                                 sys.exit(retval)
3049
3050         if True:
3051                 myfiles = mymanifests[:]
3052                 # If there are no header (SVN/CVS keywords) changes in
3053                 # the files, this Manifest commit must include the
3054                 # other (yet uncommitted) files.
3055                 if not myheaders:
3056                         myfiles += myupdates
3057                         myfiles += myremoved
3058                 myfiles.sort()
3059
3060                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3061                 mymsg = os.fdopen(fd, "wb")
3062                 mymsg.write(_unicode_encode(commitmessage))
3063                 mymsg.close()
3064
3065                 commit_cmd = []
3066                 if options.pretend and vcs is None:
3067                         # substitute a bogus value for pretend output
3068                         commit_cmd.append("cvs")
3069                 else:
3070                         commit_cmd.append(vcs)
3071                 commit_cmd.extend(vcs_global_opts)
3072                 commit_cmd.append("commit")
3073                 commit_cmd.extend(vcs_local_opts)
3074                 if vcs == "hg":
3075                         commit_cmd.extend(["--logfile", commitmessagefile])
3076                         commit_cmd.extend(myfiles)
3077                 else:
3078                         commit_cmd.extend(["-F", commitmessagefile])
3079                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3080
3081                 try:
3082                         if options.pretend:
3083                                 print("(%s)" % (" ".join(commit_cmd),))
3084                         else:
3085                                 retval = spawn(commit_cmd, env=commit_env)
3086                                 if retval != os.EX_OK:
3087                                         if repo_config.sign_commit and vcs == 'git' and \
3088                                                 not git_supports_gpg_sign():
3089                                                 # Inform user that newer git is needed (bug #403323).
3090                                                 logging.error(
3091                                                         "Git >=1.7.9 is required for signed commits!")
3092
3093                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3094                                                 "error code: %s\n") % (vcs, retval),
3095                                                 level=logging.ERROR, noiselevel=-1)
3096                                         sys.exit(retval)
3097                 finally:
3098                         try:
3099                                 os.unlink(commitmessagefile)
3100                         except OSError:
3101                                 pass
3102
3103         print()
3104         if vcs:
3105                 print("Commit complete.")
3106         else:
3107                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3108         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3109 sys.exit(0)