bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import io
  17 import logging
  18 import optparse
  19 import re
  20 import signal
  21 import stat
  22 import subprocess
  23 import sys
  24 import tempfile
  25 import textwrap
  26 import time
  27 import platform
  28
  29 try:
  30         from urllib.request import urlopen as urllib_request_urlopen
  31 except ImportError:
  32         from urllib import urlopen as urllib_request_urlopen
  33
  34 from itertools import chain
  35 from stat import S_ISDIR
  36
  37 from os import path as osp
  38 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  39 sys.path.insert(0, pym_path)
  40 import portage
  41 portage._internal_caller = True
  42 portage._disable_legacy_globals()
  43
  44 try:
  45         import xml.etree.ElementTree
  46         from xml.parsers.expat import ExpatError
  47 except (SystemExit, KeyboardInterrupt):
  48         raise
  49 except (ImportError, SystemError, RuntimeError, Exception):
  50         # broken or missing xml support
  51         # http://bugs.python.org/issue14988
  52         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  53         from portage.output import EOutput
  54         out = EOutput()
  55         for line in msg:
  56                 out.eerror(line)
  57         sys.exit(1)
  58
  59 from portage import os
  60 from portage import _encodings
  61 from portage import _unicode_encode
  62 from repoman.checks import run_checks
  63 from repoman import utilities
  64 from repoman.herdbase import make_herd_base
  65 from _emerge.Package import Package
  66 from _emerge.RootConfig import RootConfig
  67 from _emerge.userquery import userquery
  68 import portage.checksum
  69 import portage.const
  70 from portage import cvstree, normalize_path
  71 from portage import util
  72 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  73         ParseError, PermissionDenied)
  74 from portage.dep import Atom
  75 from portage.process import find_binary, spawn
  76 from portage.output import bold, create_color_func, \
  77         green, nocolor, red
  78 from portage.output import ConsoleStyleFile, StyleWriter
  79 from portage.util import writemsg_level
  80 from portage.package.ebuild.digestgen import digestgen
  81 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  82
  83 if sys.hexversion >= 0x3000000:
  84         basestring = str
  85
  86 util.initialize_logger()
  87
  88 # 14 is the length of DESCRIPTION=""
  89 max_desc_len = 100
  90 allowed_filename_chars="a-zA-Z0-9._-+:"
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  93 bad = create_color_func("BAD")
  94
  95 # A sane umask is needed for files that portage creates.
  96 os.umask(0o22)
  97 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  98 # behave incrementally.
  99 repoman_incrementals = tuple(x for x in \
 100         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 101 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 102 repoman_settings = portage.config(config_root=config_root, local_config=False)
 103
 104 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 105         repoman_settings.get('TERM') == 'dumb' or \
 106         not sys.stdout.isatty():
 107         nocolor()
 108
 109 def warn(txt):
 110         print("repoman: " + txt)
 111
 112 def err(txt):
 113         warn(txt)
 114         sys.exit(1)
 115
 116 def exithandler(signum=None, frame=None):
 117         logging.fatal("Interrupted; exiting...")
 118         if signum is None:
 119                 sys.exit(1)
 120         else:
 121                 sys.exit(128 + signum)
 122
 123 signal.signal(signal.SIGINT, exithandler)
 124
 125 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 126         """Repoman needs its own HelpFormatter for now, because the default ones
 127         murder the help text."""
 128
 129         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 130                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 131
 132         def format_description(self, description):
 133                 return description
 134
 135 class RepomanOptionParser(optparse.OptionParser):
 136         """Add the on_tail function, ruby has it, optionParser should too
 137         """
 138
 139         def __init__(self, *args, **kwargs):
 140                 optparse.OptionParser.__init__(self, *args, **kwargs)
 141                 self.tail = ""
 142
 143         def on_tail(self, description):
 144                 self.tail += description
 145
 146         def format_help(self, formatter=None):
 147                 result = optparse.OptionParser.format_help(self, formatter)
 148                 result += self.tail
 149                 return result
 150
 151
 152 def ParseArgs(argv, qahelp):
 153         """This function uses a customized optionParser to parse command line arguments for repoman
 154         Args:
 155           argv - a sequence of command line arguments
 156                 qahelp - a dict of qa warning to help message
 157         Returns:
 158           (opts, args), just like a call to parser.parse_args()
 159         """
 160
 161         if argv and isinstance(argv[0], bytes):
 162                 argv = [portage._unicode_decode(x) for x in argv]
 163
 164         modes = {
 165                 'commit' : 'Run a scan then commit changes',
 166                 'ci' : 'Run a scan then commit changes',
 167                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 168                 'full' : 'Scan directory tree and print all issues (not a summary)',
 169                 'help' : 'Show this screen',
 170                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 171                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 172                 'scan' : 'Scan directory tree for QA issues'
 173         }
 174
 175         mode_keys = list(modes)
 176         mode_keys.sort()
 177
 178         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 179         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 180         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 181         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 182         parser.description += "\nmodes: " + " | ".join(map(green, mode_keys))
 183
 184         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 185                 help='Request a confirmation before commiting')
 186
 187         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 188                 help='specify a commit message on the command line')
 189
 190         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 191                 help='specify a path to a file that contains a commit message')
 192
 193         parser.add_option('--digest',
 194                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 195                 help='Automatically update Manifest digests for modified files')
 196
 197         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 198                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 199
 200         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 201                 help='do not print unnecessary messages')
 202
 203         parser.add_option(
 204                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 205                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 206                 'regardless of modification if \'force\' is specified)')
 207
 208         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 209                 help='Commit with QA violations')
 210
 211         parser.add_option('--vcs', dest='vcs',
 212                 help='Force using specific VCS instead of autodetection')
 213
 214         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 215                 help='be very verbose in output', default=0)
 216
 217         parser.add_option('-V', '--version', dest='version', action='store_true',
 218                 help='show version info')
 219
 220         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 221                 default=False, help='forces the metadata.xml parse check to be carried out')
 222
 223         parser.add_option(
 224                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 225                 metavar="<y|n>",
 226                 help='only check packages that have uncommitted modifications')
 227
 228         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 229                 default=False, help='ignore arch-specific failures (where arch != host)')
 230
 231         parser.add_option("--ignore-default-opts",
 232                 action="store_true",
 233                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 234
 235         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 236                 default=False, help='ignore masked packages (not allowed with commit mode)')
 237
 238         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 239                 default=False, help='include dev profiles in dependency checks')
 240
 241         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 242                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 243
 244         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 245                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 246
 247         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 248                 help='specify which mode repoman will run in (default=full)')
 249
 250         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 251
 252         for k in mode_keys:
 253                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 254
 255         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 256
 257         sorted_qa = list(qahelp)
 258         sorted_qa.sort()
 259         for k in sorted_qa:
 260                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 261
 262         opts, args = parser.parse_args(argv[1:])
 263
 264         if not opts.ignore_default_opts:
 265                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 266                 if default_opts:
 267                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 268
 269         if opts.mode == 'help':
 270                 parser.print_help(short=False)
 271
 272         for arg in args:
 273                 if arg in modes:
 274                         if not opts.mode:
 275                                 opts.mode = arg
 276                                 break
 277                 else:
 278                         parser.error("invalid mode: %s" % arg)
 279
 280         if not opts.mode:
 281                 opts.mode = 'full'
 282
 283         if opts.mode == 'ci':
 284                 opts.mode = 'commit'  # backwards compat shortcut
 285
 286         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 287                 if opts.ignore_masked:
 288                         parser.error('Commit mode and --ignore-masked are not compatible')
 289                 if opts.without_mask:
 290                         parser.error('Commit mode and --without-mask are not compatible')
 291
 292         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 293         for val in range(opts.verbosity):
 294                 logger = logging.getLogger()
 295                 logger.setLevel(logger.getEffectiveLevel() - 10)
 296
 297         for val in range(opts.quiet):
 298                 logger = logging.getLogger()
 299                 logger.setLevel(logger.getEffectiveLevel() + 10)
 300
 301         return (opts, args)
 302
 303 qahelp = {
 304         "CVS/Entries.IO_error": "Attempting to commit, and an IO error was encountered access the Entries file",
 305         "ebuild.invalidname": "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 306         "ebuild.namenomatch": "Ebuild files that do not have the same name as their parent directory",
 307         "changelog.ebuildadded": "An ebuild was added but the ChangeLog was not modified",
 308         "changelog.missing": "Missing ChangeLog files",
 309         "ebuild.notadded": "Ebuilds that exist but have not been added to cvs",
 310         "ebuild.patches": "PATCHES variable should be a bash array to ensure white space safety",
 311         "changelog.notadded": "ChangeLogs that exist but have not been added to cvs",
 312         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 313         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 314         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 315         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 316         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 317         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 318         "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 319         "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 320         "file.size": "Files in the files directory must be under 20 KiB",
 321         "file.size.fatal": "Files in the files directory must be under 60 KiB",
 322         "file.name": "File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 323         "file.UTF8": "File is not UTF8 compliant",
 324         "inherit.deprecated": "Ebuild inherits a deprecated eclass",
 325         "inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 326         "inherit.unused": "Ebuild inherits an eclass but does not use it",
 327         "java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
 328         "wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 329         "KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 330         "KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
 331         "KEYWORDS.stable": "Ebuilds that have been added directly with stable KEYWORDS",
 332         "KEYWORDS.stupid": "Ebuilds that use KEYWORDS=-* instead of package.mask",
 333         "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable",
 334         "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable",
 335         "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable",
 336         "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len,
 337         "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 338         "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI",
 339         "EAPI.incompatible": "Ebuilds that use features that are only available with a different EAPI",
 340         "EAPI.unsupported": "Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 341         "SLOT.invalid": "Ebuilds that have a missing or invalid SLOT variable value",
 342         "HOMEPAGE.missing": "Ebuilds that have a missing or empty HOMEPAGE variable",
 343         "HOMEPAGE.virtual": "Virtuals that have a non-empty HOMEPAGE variable",
 344         "PDEPEND.suspect": "PDEPEND contains a package that usually only belongs in DEPEND.",
 345         "LICENSE.syntax": "Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 346         "PROVIDE.syntax": "Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 347         "PROPERTIES.syntax": "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 348         "RESTRICT.syntax": "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 349         "REQUIRED_USE.syntax": "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 350         "SRC_URI.syntax": "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 351         "SRC_URI.mirror": "A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 352         "ebuild.syntax": "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 353         "ebuild.output": "A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 354         "ebuild.nesteddie": "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 355         "variable.invalidchar": "A variable contains an invalid character that is not part of the ASCII character set",
 356         "variable.readonly": "Assigning a readonly variable",
 357         "variable.usedwithhelpers": "Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 358         "LIVEVCS.stable": "This ebuild is a live checkout from a VCS but has stable keywords.",
 359         "LIVEVCS.unmasked": "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 360         "IUSE.invalid": "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 361         "IUSE.missing": "This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 362         "LICENSE.invalid": "This ebuild is listing a license that doesnt exist in portages license/ dir.",
 363         "LICENSE.deprecated": "This ebuild is listing a deprecated license.",
 364         "KEYWORDS.invalid": "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 365         "RDEPEND.implicit": "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 366         "RDEPEND.suspect": "RDEPEND contains a package that usually only belongs in DEPEND.",
 367         "RESTRICT.invalid": "This ebuild contains invalid RESTRICT values.",
 368         "digest.assumed": "Existing digest must be assumed correct (Package level only)",
 369         "digest.missing": "Some files listed in SRC_URI aren't referenced in the Manifest",
 370         "digest.unused": "Some files listed in the Manifest aren't referenced in SRC_URI",
 371         "ebuild.majorsyn": "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 372         "ebuild.minorsyn": "This ebuild has a minor syntax error that contravenes gentoo coding style",
 373         "ebuild.badheader": "This ebuild has a malformed header",
 374         "manifest.bad": "Manifest has missing or incorrect digests",
 375         "metadata.missing": "Missing metadata.xml files",
 376         "metadata.bad": "Bad metadata.xml files",
 377         "metadata.warning": "Warnings in metadata.xml files",
 378         "portage.internal": "The ebuild uses an internal Portage function or variable",
 379         "virtual.oldstyle": "The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 380         "virtual.suspect": "Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 381         "usage.obsolete": "The ebuild makes use of an obsolete construct",
 382         "upstream.workaround": "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 383 }
 384
 385 qacats = list(qahelp)
 386 qacats.sort()
 387
 388 qawarnings = set((
 389 "changelog.missing",
 390 "changelog.notadded",
 391 "dependency.unknown",
 392 "digest.assumed",
 393 "digest.unused",
 394 "ebuild.notadded",
 395 "ebuild.nesteddie",
 396 "dependency.badmasked",
 397 "dependency.badindev",
 398 "dependency.badmaskedindev",
 399 "dependency.badtilde",
 400 "DESCRIPTION.toolong",
 401 "EAPI.deprecated",
 402 "HOMEPAGE.virtual",
 403 "LICENSE.deprecated",
 404 "LICENSE.virtual",
 405 "KEYWORDS.dropped",
 406 "KEYWORDS.stupid",
 407 "KEYWORDS.missing",
 408 "PDEPEND.suspect",
 409 "RDEPEND.implicit",
 410 "RDEPEND.suspect",
 411 "virtual.suspect",
 412 "RESTRICT.invalid",
 413 "ebuild.minorsyn",
 414 "ebuild.badheader",
 415 "ebuild.patches",
 416 "file.size",
 417 "inherit.unused",
 418 "inherit.deprecated",
 419 "java.eclassesnotused",
 420 "wxwidgets.eclassnotused",
 421 "metadata.warning",
 422 "portage.internal",
 423 "usage.obsolete",
 424 "upstream.workaround",
 425 "LIVEVCS.stable",
 426 "LIVEVCS.unmasked",
 427 ))
 428
 429 if portage.const._ENABLE_INHERIT_CHECK:
 430         # This is experimental, so it's non-fatal.
 431         qawarnings.add("inherit.missing")
 432
 433 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 434
 435 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 436 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 437 allvars.update(Package.metadata_keys)
 438 allvars = sorted(allvars)
 439 commitmessage = None
 440 for x in missingvars:
 441         x += ".missing"
 442         if x not in qacats:
 443                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 444                 qacats.append(x)
 445                 qawarnings.add(x)
 446
 447 valid_restrict = frozenset(["binchecks", "bindist",
 448         "fetch", "installsources", "mirror",
 449         "primaryuri", "splitdebug", "strip", "test", "userpriv"])
 450
 451 live_eclasses = frozenset([
 452         "bzr",
 453         "cvs",
 454         "darcs",
 455         "git",
 456         "git-2",
 457         "mercurial",
 458         "subversion",
 459         "tla",
 460 ])
 461
 462 suspect_rdepend = frozenset([
 463         "app-arch/cabextract",
 464         "app-arch/rpm2targz",
 465         "app-doc/doxygen",
 466         "dev-lang/nasm",
 467         "dev-lang/swig",
 468         "dev-lang/yasm",
 469         "dev-perl/extutils-pkgconfig",
 470         "dev-util/byacc",
 471         "dev-util/cmake",
 472         "dev-util/ftjam",
 473         "dev-util/gperf",
 474         "dev-util/gtk-doc",
 475         "dev-util/gtk-doc-am",
 476         "dev-util/intltool",
 477         "dev-util/jam",
 478         "dev-util/pkg-config-lite",
 479         "dev-util/pkgconf",
 480         "dev-util/pkgconfig",
 481         "dev-util/pkgconfig-openbsd",
 482         "dev-util/scons",
 483         "dev-util/unifdef",
 484         "dev-util/yacc",
 485         "media-gfx/ebdftopcf",
 486         "sys-apps/help2man",
 487         "sys-devel/autoconf",
 488         "sys-devel/automake",
 489         "sys-devel/bin86",
 490         "sys-devel/bison",
 491         "sys-devel/dev86",
 492         "sys-devel/flex",
 493         "sys-devel/m4",
 494         "sys-devel/pmake",
 495         "virtual/linux-sources",
 496         "virtual/pkgconfig",
 497         "x11-misc/bdftopcf",
 498         "x11-misc/imake",
 499 ])
 500
 501 suspect_virtual = {
 502         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 503         "dev-util/pkgconf":"virtual/pkgconfig",
 504         "dev-util/pkgconfig":"virtual/pkgconfig",
 505         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 506         "dev-libs/libusb":"virtual/libusb",
 507         "dev-libs/libusbx":"virtual/libusb",
 508         "dev-libs/libusb-compat":"virtual/libusb",
 509 }
 510
 511 metadata_xml_encoding = 'UTF-8'
 512 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 513         (metadata_xml_encoding,)
 514 metadata_doctype_name = 'pkgmetadata'
 515 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 516 # force refetch if the local copy creation time is older than this
 517 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 518
 519 # file.executable
 520 no_exec = frozenset(["Manifest", "ChangeLog", "metadata.xml"])
 521
 522 options, arguments = ParseArgs(sys.argv, qahelp)
 523
 524 if options.version:
 525         print("Portage", portage.VERSION)
 526         sys.exit(0)
 527
 528 # Set this to False when an extraordinary issue (generally
 529 # something other than a QA issue) makes it impossible to
 530 # commit (like if Manifest generation fails).
 531 can_force = True
 532
 533 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 534 if portdir is None:
 535         sys.exit(1)
 536
 537 myreporoot = os.path.basename(portdir_overlay)
 538 myreporoot += mydir[len(portdir_overlay):]
 539
 540 if options.vcs:
 541         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 542                 vcs = options.vcs
 543         else:
 544                 vcs = None
 545 else:
 546         vcses = utilities.FindVCS()
 547         if len(vcses) > 1:
 548                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 549                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 550                 sys.exit(1)
 551         elif vcses:
 552                 vcs = vcses[0]
 553         else:
 554                 vcs = None
 555
 556 if options.if_modified == "y" and vcs is None:
 557         logging.info("Not in a version controlled repository; "
 558                 "disabling --if-modified.")
 559         options.if_modified = "n"
 560
 561 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 562 vcs_preserves_mtime = vcs in ('cvs',)
 563
 564 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 565 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 566 if vcs_global_opts is None:
 567         if vcs in ('cvs', 'svn'):
 568                 vcs_global_opts = "-q"
 569         else:
 570                 vcs_global_opts = ""
 571 vcs_global_opts = vcs_global_opts.split()
 572
 573 if options.mode == 'commit' and not options.pretend and not vcs:
 574         logging.info("Not in a version controlled repository; enabling pretend mode.")
 575         options.pretend = True
 576
 577 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 578 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 579         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 580         portage._shell_quote(portdir_overlay))
 581 # We have to call the config constructor again so
 582 # that config.repositories is initialized correctly.
 583 repoman_settings = portage.config(config_root=config_root, local_config=False,
 584         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 585
 586 root = repoman_settings['EROOT']
 587 trees = {
 588         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 589 }
 590 portdb = trees[root]['porttree'].dbapi
 591
 592 # Constrain dependency resolution to the master(s)
 593 # that are specified in layout.conf.
 594 repodir = os.path.realpath(portdir_overlay)
 595 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 596 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 597 portdir = portdb.porttrees[0]
 598 commit_env = os.environ.copy()
 599
 600 if repo_config.allow_provide_virtual:
 601         qawarnings.add("virtual.oldstyle")
 602
 603 if repo_config.sign_commit:
 604         if vcs == 'git':
 605                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 606                 # the commit arguments. If key_id is unspecified, then it must be
 607                 # configured by `git config user.signingkey key_id`.
 608                 vcs_local_opts.append("--gpg-sign")
 609                 if repoman_settings.get("PORTAGE_GPG_DIR"):
 610                         # Pass GNUPGHOME to git for bug #462362.
 611                         commit_env["GNUPGHOME"] = repoman_settings["PORTAGE_GPG_DIR"]
 612
 613 # In order to disable manifest signatures, repos may set
 614 # "sign-manifests = false" in metadata/layout.conf. This
 615 # can be used to prevent merge conflicts like those that
 616 # thin-manifests is designed to prevent.
 617 sign_manifests = "sign" in repoman_settings.features and \
 618         repo_config.sign_manifest
 619
 620 if repo_config.sign_manifest and repo_config.name == "gentoo" and \
 621         options.mode in ("commit",) and not sign_manifests:
 622         msg = ("The '%s' repository has manifest signatures enabled, "
 623         "but FEATURES=sign is currently disabled. In order to avoid this "
 624         "warning, enable FEATURES=sign in make.conf. Alternatively, "
 625         "repositories can disable manifest signatures by setting "
 626         "'sign-manifests = false' in metadata/layout.conf.") % \
 627         (repo_config.name,)
 628         for line in textwrap.wrap(msg, 60):
 629                 logging.warn(line)
 630
 631 if sign_manifests and options.mode in ("commit",) and \
 632         repoman_settings.get("PORTAGE_GPG_KEY") and \
 633         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 634         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 635         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 636                 repoman_settings["PORTAGE_GPG_KEY"])
 637         sys.exit(1)
 638
 639 manifest_hashes = repo_config.manifest_hashes
 640 if manifest_hashes is None:
 641         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 642
 643 if options.mode in ("commit", "fix", "manifest"):
 644         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 645                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 646                 "metadata/layout.conf does not contain the '%s' hash which "
 647                 "is required by this portage version. You will have to "
 648                 "upgrade portage if you want to generate valid manifests for "
 649                 "this repository.") % \
 650                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 651                 for line in textwrap.wrap(msg, 70):
 652                         logging.error(line)
 653                 sys.exit(1)
 654
 655         unsupported_hashes = manifest_hashes.difference(
 656                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 657         if unsupported_hashes:
 658                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 659                 "metadata/layout.conf contains one or more hash types '%s' "
 660                 "which are not supported by this portage version. You will "
 661                 "have to upgrade portage if you want to generate valid "
 662                 "manifests for this repository.") % \
 663                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 664                 for line in textwrap.wrap(msg, 70):
 665                         logging.error(line)
 666                 sys.exit(1)
 667
 668 if options.echangelog is None and repo_config.update_changelog:
 669         options.echangelog = 'y'
 670
 671 if vcs is None:
 672         options.echangelog = 'n'
 673
 674 # The --echangelog option causes automatic ChangeLog generation,
 675 # which invalidates changelog.ebuildadded and changelog.missing
 676 # checks.
 677 # Note: Some don't use ChangeLogs in distributed SCMs.
 678 # It will be generated on server side from scm log,
 679 # before package moves to the rsync server.
 680 # This is needed because they try to avoid merge collisions.
 681 # Gentoo's Council decided to always use the ChangeLog file.
 682 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 683 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 684
 685 if 'digest' in repoman_settings.features and options.digest != 'n':
 686         options.digest = 'y'
 687
 688 logging.debug("vcs: %s" % (vcs,))
 689 logging.debug("repo config: %s" % (repo_config,))
 690 logging.debug("options: %s" % (options,))
 691
 692 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 693 # profile-specific config constructor calls.
 694 env = os.environ.copy()
 695 env['PORTDIR'] = portdir
 696 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 697
 698 logging.info('Setting paths:')
 699 logging.info('PORTDIR = "' + portdir + '"')
 700 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 701
 702 # It's confusing if these warnings are displayed without the user
 703 # being told which profile they come from, so disable them.
 704 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 705
 706 categories = []
 707 for path in repo_config.eclass_db.porttrees:
 708         categories.extend(portage.util.grabfile(
 709                 os.path.join(path, 'profiles', 'categories')))
 710 repoman_settings.categories = frozenset(
 711         portage.util.stack_lists([categories], incremental=1))
 712 categories = repoman_settings.categories
 713
 714 portdb.settings = repoman_settings
 715 root_config = RootConfig(repoman_settings, trees[root], None)
 716 # We really only need to cache the metadata that's necessary for visibility
 717 # filtering. Anything else can be discarded to reduce memory consumption.
 718 portdb._aux_cache_keys.clear()
 719 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 720
 721 reposplit = myreporoot.split(os.path.sep)
 722 repolevel = len(reposplit)
 723
 724 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 725 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 726 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 727 if options.mode == 'commit' and repolevel not in [1, 2, 3]:
 728         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 729         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 730         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 731         print(red("***"))
 732         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 733
 734 # Make startdir relative to the canonical repodir, so that we can pass
 735 # it to digestgen and it won't have to be canonicalized again.
 736 if repolevel == 1:
 737         startdir = repodir
 738 else:
 739         startdir = normalize_path(mydir)
 740         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
 741
 742 def caterror(mycat):
 743         err(mycat + " is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add " + catdir + " to " + repodir + "/profiles/categories\nif it is a new category.")
 744
 745 def repoman_getstatusoutput(cmd):
 746         """
 747         Implements an interface similar to getstatusoutput(), but with
 748         customized unicode handling (see bug #310789) and without the shell.
 749         """
 750         args = portage.util.shlex_split(cmd)
 751         encoding = _encodings['fs']
 752         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 753                 # Python 3.1 does not support bytes in Popen args.
 754                 args = [_unicode_encode(x,
 755                         encoding=encoding, errors='strict') for x in args]
 756         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 757                 stderr=subprocess.STDOUT)
 758         output = portage._unicode_decode(proc.communicate()[0],
 759                 encoding=encoding, errors='strict')
 760         if output and output[-1] == "\n":
 761                 # getstatusoutput strips one newline
 762                 output = output[:-1]
 763         return (proc.wait(), output)
 764
 765 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 766         """
 767         Implements an interface similar to os.popen(), but with customized
 768         unicode handling (see bug #310789) and without the shell.
 769         """
 770
 771         __slots__ = ('_proc', '_stdout')
 772
 773         def __init__(self, cmd):
 774                 args = portage.util.shlex_split(cmd)
 775                 encoding = _encodings['fs']
 776                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 777                         # Python 3.1 does not support bytes in Popen args.
 778                         args = [_unicode_encode(x,
 779                                 encoding=encoding, errors='strict') for x in args]
 780                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 781                 object.__setattr__(self, '_proc', proc)
 782                 object.__setattr__(self, '_stdout',
 783                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 784
 785         def _get_target(self):
 786                 return object.__getattribute__(self, '_stdout')
 787
 788         __enter__ = _get_target
 789
 790         def __exit__(self, exc_type, exc_value, traceback):
 791                 proc = object.__getattribute__(self, '_proc')
 792                 proc.wait()
 793                 proc.stdout.close()
 794
 795 class ProfileDesc(object):
 796         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 797         def __init__(self, arch, status, sub_path, tree_path):
 798                 self.arch = arch
 799                 self.status = status
 800                 if sub_path:
 801                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 802                 self.sub_path = sub_path
 803                 self.tree_path = tree_path
 804                 if tree_path:
 805                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 806                 else:
 807                         self.abs_path = tree_path
 808
 809         def __str__(self):
 810                 if self.sub_path:
 811                         return self.sub_path
 812                 return 'empty profile'
 813
 814 profile_list = []
 815 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 816
 817 # get lists of valid keywords, licenses, and use
 818 kwlist = set()
 819 liclist = set()
 820 uselist = set()
 821 global_pmasklines = []
 822
 823 for path in portdb.porttrees:
 824         try:
 825                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 826         except OSError:
 827                 pass
 828         kwlist.update(portage.grabfile(os.path.join(path,
 829                 "profiles", "arch.list")))
 830
 831         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 832         for x in use_desc:
 833                 x = x.split()
 834                 if x:
 835                         uselist.add(x[0])
 836
 837         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 838         try:
 839                 expand_list = os.listdir(expand_desc_dir)
 840         except OSError:
 841                 pass
 842         else:
 843                 for fn in expand_list:
 844                         if not fn[-5:] == '.desc':
 845                                 continue
 846                         use_prefix = fn[:-5].lower() + '_'
 847                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 848                                 x = x.split()
 849                                 if x:
 850                                         uselist.add(use_prefix + x[0])
 851
 852         global_pmasklines.append(portage.util.grabfile_package(
 853                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 854
 855         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 856         try:
 857                 desc_file = io.open(_unicode_encode(desc_path,
 858                         encoding=_encodings['fs'], errors='strict'),
 859                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 860         except EnvironmentError:
 861                 pass
 862         else:
 863                 for i, x in enumerate(desc_file):
 864                         if x[0] == "#":
 865                                 continue
 866                         arch = x.split()
 867                         if len(arch) == 0:
 868                                 continue
 869                         if len(arch) != 3:
 870                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 871                                         desc_path + " line %d" % (i + 1, ))
 872                         elif arch[0] not in kwlist:
 873                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 874                                         desc_path + " line %d" % (i + 1, ))
 875                         elif arch[2] not in valid_profile_types:
 876                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 877                                         desc_path + " line %d" % (i + 1, ))
 878                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 879                         if not os.path.isdir(profile_desc.abs_path):
 880                                 logging.error(
 881                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 882                                         arch[2], arch[1], arch[0], desc_path, i + 1)
 883                                 continue
 884                         if os.path.exists(
 885                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 886                                 continue
 887                         profile_list.append(profile_desc)
 888                 desc_file.close()
 889
 890 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 891 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 892
 893 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 894 global_pmaskdict = {}
 895 for x in global_pmasklines:
 896         global_pmaskdict.setdefault(x.cp, []).append(x)
 897 del global_pmasklines
 898
 899 def has_global_mask(pkg):
 900         mask_atoms = global_pmaskdict.get(pkg.cp)
 901         if mask_atoms:
 902                 pkg_list = [pkg]
 903                 for x in mask_atoms:
 904                         if portage.dep.match_from_list(x, pkg_list):
 905                                 return x
 906         return None
 907
 908 # Ensure that profile sub_path attributes are unique. Process in reverse order
 909 # so that profiles with duplicate sub_path from overlays will override
 910 # profiles with the same sub_path from parent repos.
 911 profiles = {}
 912 profile_list.reverse()
 913 profile_sub_paths = set()
 914 for prof in profile_list:
 915         if prof.sub_path in profile_sub_paths:
 916                 continue
 917         profile_sub_paths.add(prof.sub_path)
 918         profiles.setdefault(prof.arch, []).append(prof)
 919
 920 # Use an empty profile for checking dependencies of
 921 # packages that have empty KEYWORDS.
 922 prof = ProfileDesc('**', 'stable', '', '')
 923 profiles.setdefault(prof.arch, []).append(prof)
 924
 925 for x in repoman_settings.archlist():
 926         if x[0] == "~":
 927                 continue
 928         if x not in profiles:
 929                 print(red("\"" + x + "\" doesn't have a valid profile listed in profiles.desc."))
 930                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 931                 print(red("up with the " + x + " team."))
 932                 print()
 933
 934 liclist_deprecated = set()
 935 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 936         liclist_deprecated.update(
 937                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 938
 939 if not liclist:
 940         logging.fatal("Couldn't find licenses?")
 941         sys.exit(1)
 942
 943 if not kwlist:
 944         logging.fatal("Couldn't read KEYWORDS from arch.list")
 945         sys.exit(1)
 946
 947 if not uselist:
 948         logging.fatal("Couldn't find use.desc?")
 949         sys.exit(1)
 950
 951 scanlist = []
 952 if repolevel == 2:
 953         # we are inside a category directory
 954         catdir = reposplit[-1]
 955         if catdir not in categories:
 956                 caterror(catdir)
 957         mydirlist = os.listdir(startdir)
 958         for x in mydirlist:
 959                 if x == "CVS" or x.startswith("."):
 960                         continue
 961                 if os.path.isdir(startdir + "/" + x):
 962                         scanlist.append(catdir + "/" + x)
 963         repo_subdir = catdir + os.sep
 964 elif repolevel == 1:
 965         for x in categories:
 966                 if not os.path.isdir(startdir + "/" + x):
 967                         continue
 968                 for y in os.listdir(startdir + "/" + x):
 969                         if y == "CVS" or y.startswith("."):
 970                                 continue
 971                         if os.path.isdir(startdir + "/" + x + "/" + y):
 972                                 scanlist.append(x + "/" + y)
 973         repo_subdir = ""
 974 elif repolevel == 3:
 975         catdir = reposplit[-2]
 976         if catdir not in categories:
 977                 caterror(catdir)
 978         scanlist.append(catdir + "/" + reposplit[-1])
 979         repo_subdir = scanlist[-1] + os.sep
 980 else:
 981         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 982                 ' from the current working directory'
 983         logging.critical(msg)
 984         sys.exit(1)
 985
 986 repo_subdir_len = len(repo_subdir)
 987 scanlist.sort()
 988
 989 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 990
 991 def vcs_files_to_cps(vcs_file_iter):
 992         """
 993         Iterate over the given modified file paths returned from the vcs,
 994         and return a frozenset containing category/pn strings for each
 995         modified package.
 996         """
 997
 998         modified_cps = []
 999
1000         if repolevel == 3:
1001                 if reposplit[-2] in categories and \
1002                         next(vcs_file_iter, None) is not None:
1003                         modified_cps.append("/".join(reposplit[-2:]))
1004
1005         elif repolevel == 2:
1006                 category = reposplit[-1]
1007                 if category in categories:
1008                         for filename in vcs_file_iter:
1009                                 f_split = filename.split(os.sep)
1010                                 # ['.', pn, ...]
1011                                 if len(f_split) > 2:
1012                                         modified_cps.append(category + "/" + f_split[1])
1013
1014         else:
1015                 # repolevel == 1
1016                 for filename in vcs_file_iter:
1017                         f_split = filename.split(os.sep)
1018                         # ['.', category, pn, ...]
1019                         if len(f_split) > 3 and f_split[1] in categories:
1020                                 modified_cps.append("/".join(f_split[1:3]))
1021
1022         return frozenset(modified_cps)
1023
1024 def git_supports_gpg_sign():
1025         status, cmd_output = \
1026                 repoman_getstatusoutput("git --version")
1027         cmd_output = cmd_output.split()
1028         if cmd_output:
1029                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1030                 if version is not None:
1031                         version = [int(x) for x in version.groups()]
1032                         if version[0] > 1 or \
1033                                 (version[0] == 1 and version[1] > 7) or \
1034                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1035                                 return True
1036         return False
1037
1038 def dev_keywords(profiles):
1039         """
1040         Create a set of KEYWORDS values that exist in 'dev'
1041         profiles. These are used
1042         to trigger a message notifying the user when they might
1043         want to add the --include-dev option.
1044         """
1045         type_arch_map = {}
1046         for arch, arch_profiles in profiles.items():
1047                 for prof in arch_profiles:
1048                         arch_set = type_arch_map.get(prof.status)
1049                         if arch_set is None:
1050                                 arch_set = set()
1051                                 type_arch_map[prof.status] = arch_set
1052                         arch_set.add(arch)
1053
1054         dev_keywords = type_arch_map.get('dev', set())
1055         dev_keywords.update(['~' + arch for arch in dev_keywords])
1056         return frozenset(dev_keywords)
1057
1058 dev_keywords = dev_keywords(profiles)
1059
1060 stats = {}
1061 fails = {}
1062
1063 for x in qacats:
1064         stats[x] = 0
1065         fails[x] = []
1066
1067 xmllint_capable = False
1068 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1069
1070 def parsedate(s):
1071         """Parse a RFC 822 date and time string.
1072         This is required for python3 compatibility, since the
1073         rfc822.parsedate() function is not available."""
1074
1075         s_split = []
1076         for x in s.upper().split():
1077                 for y in x.split(','):
1078                         if y:
1079                                 s_split.append(y)
1080
1081         if len(s_split) != 6:
1082                 return None
1083
1084         # %a, %d %b %Y %H:%M:%S %Z
1085         a, d, b, Y, H_M_S, Z = s_split
1086
1087         # Convert month to integer, since strptime %w is locale-dependent.
1088         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1089                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1090         m = month_map.get(b)
1091         if m is None:
1092                 return None
1093         m = "%02d" % m
1094
1095         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1096
1097 def fetch_metadata_dtd():
1098         """
1099         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1100         metadata_dtd_ctime_interval.
1101         @rtype: bool
1102         @return: True if successful, otherwise False
1103         """
1104
1105         must_fetch = True
1106         metadata_dtd_st = None
1107         current_time = int(time.time())
1108         try:
1109                 metadata_dtd_st = os.stat(metadata_dtd)
1110         except EnvironmentError as e:
1111                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1112                         raise
1113                 del e
1114         else:
1115                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1116                 if abs(current_time - metadata_dtd_st.st_ctime) \
1117                         < metadata_dtd_ctime_interval:
1118                         must_fetch = False
1119
1120         if must_fetch:
1121                 print()
1122                 print(green("***") + " the local copy of metadata.dtd " + \
1123                         "needs to be refetched, doing that now")
1124                 print()
1125                 try:
1126                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1127                         msg_info = url_f.info()
1128                         last_modified = msg_info.get('last-modified')
1129                         if last_modified is not None:
1130                                 last_modified = parsedate(last_modified)
1131                                 if last_modified is not None:
1132                                         last_modified = calendar.timegm(last_modified)
1133
1134                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1135                         try:
1136                                 local_f = open(metadata_dtd_tmp, mode='wb')
1137                                 local_f.write(url_f.read())
1138                                 local_f.close()
1139                                 if last_modified is not None:
1140                                         try:
1141                                                 os.utime(metadata_dtd_tmp,
1142                                                         (int(last_modified), int(last_modified)))
1143                                         except OSError:
1144                                                 # This fails on some odd non-unix-like filesystems.
1145                                                 # We don't really need the mtime to be preserved
1146                                                 # anyway here (currently we use ctime to trigger
1147                                                 # fetch), so just ignore it.
1148                                                 pass
1149                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1150                         finally:
1151                                 try:
1152                                         os.unlink(metadata_dtd_tmp)
1153                                 except OSError:
1154                                         pass
1155
1156                         url_f.close()
1157
1158                 except EnvironmentError as e:
1159                         print()
1160                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1161                         print(red("!!!")+" exception '%s' though." % (e,))
1162                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1163                         return False
1164
1165         return True
1166
1167 if options.mode == "manifest":
1168         pass
1169 elif not find_binary('xmllint'):
1170         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1171         if options.xml_parse or repolevel == 3:
1172                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1173                 sys.exit(1)
1174 else:
1175         if not fetch_metadata_dtd():
1176                 sys.exit(1)
1177         # this can be problematic if xmllint changes their output
1178         xmllint_capable = True
1179
1180 if options.mode == 'commit' and vcs:
1181         utilities.detect_vcs_conflicts(options, vcs)
1182
1183 if options.mode == "manifest":
1184         pass
1185 elif options.pretend:
1186         print(green("\nRepoMan does a once-over of the neighborhood..."))
1187 else:
1188         print(green("\nRepoMan scours the neighborhood..."))
1189
1190 new_ebuilds = set()
1191 modified_ebuilds = set()
1192 modified_changelogs = set()
1193 mychanged = []
1194 mynew = []
1195 myremoved = []
1196
1197 if vcs == "cvs":
1198         mycvstree = cvstree.getentries("./", recursive=1)
1199         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1200         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1201         if options.if_modified == "y":
1202                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1203
1204 elif vcs == "svn":
1205         with repoman_popen("svn status") as f:
1206                 svnstatus = f.readlines()
1207         mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR"]
1208         mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
1209         if options.if_modified == "y":
1210                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1211
1212 elif vcs == "git":
1213         with repoman_popen("git diff-index --name-only "
1214                 "--relative --diff-filter=M HEAD") as f:
1215                 mychanged = f.readlines()
1216         mychanged = ["./" + elem[:-1] for elem in mychanged]
1217
1218         with repoman_popen("git diff-index --name-only "
1219                 "--relative --diff-filter=A HEAD") as f:
1220                 mynew = f.readlines()
1221         mynew = ["./" + elem[:-1] for elem in mynew]
1222         if options.if_modified == "y":
1223                 with repoman_popen("git diff-index --name-only "
1224                         "--relative --diff-filter=D HEAD") as f:
1225                         myremoved = f.readlines()
1226                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1227
1228 elif vcs == "bzr":
1229         with repoman_popen("bzr status -S .") as f:
1230                 bzrstatus = f.readlines()
1231         mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
1232         mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "NK" or elem[0:1] == "R")]
1233         if options.if_modified == "y":
1234                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
1235
1236 elif vcs == "hg":
1237         with repoman_popen("hg status --no-status --modified .") as f:
1238                 mychanged = f.readlines()
1239         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1240         with repoman_popen("hg status --no-status --added .") as f:
1241                 mynew = f.readlines()
1242         mynew = ["./" + elem.rstrip() for elem in mynew]
1243         if options.if_modified == "y":
1244                 with repoman_popen("hg status --no-status --removed .") as f:
1245                         myremoved = f.readlines()
1246                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1247
1248 if vcs:
1249         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1250         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1251         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1252                 if os.path.basename(x) == "ChangeLog")
1253
1254 def vcs_new_changed(relative_path):
1255         for x in chain(mychanged, mynew):
1256                 if x == relative_path:
1257                         return True
1258         return False
1259
1260 have_pmasked = False
1261 have_dev_keywords = False
1262 dofail = 0
1263
1264 # NOTE: match-all caches are not shared due to potential
1265 # differences between profiles in _get_implicit_iuse.
1266 arch_caches = {}
1267 arch_xmatch_caches = {}
1268 shared_xmatch_caches = {"cp-list":{}}
1269
1270 # Disable the "ebuild.notadded" check when not in commit mode and
1271 # running `svn status` in every package dir will be too expensive.
1272
1273 check_ebuild_notadded = not \
1274         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1275
1276 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1277 thirdpartymirrors = {}
1278 for k, v in repoman_settings.thirdpartymirrors().items():
1279         for v in v:
1280                 if not v.endswith("/"):
1281                         v += "/"
1282                 thirdpartymirrors[v] = k
1283
1284 class _XMLParser(xml.etree.ElementTree.XMLParser):
1285
1286         def __init__(self, data, **kwargs):
1287                 xml.etree.ElementTree.XMLParser.__init__(self, **kwargs)
1288                 self._portage_data = data
1289                 if hasattr(self, 'parser'):
1290                         self._base_XmlDeclHandler = self.parser.XmlDeclHandler
1291                         self.parser.XmlDeclHandler = self._portage_XmlDeclHandler
1292                         self._base_StartDoctypeDeclHandler = \
1293                                 self.parser.StartDoctypeDeclHandler
1294                         self.parser.StartDoctypeDeclHandler = \
1295                                 self._portage_StartDoctypeDeclHandler
1296
1297         def _portage_XmlDeclHandler(self, version, encoding, standalone):
1298                 if self._base_XmlDeclHandler is not None:
1299                         self._base_XmlDeclHandler(version, encoding, standalone)
1300                 self._portage_data["XML_DECLARATION"] = (version, encoding, standalone)
1301
1302         def _portage_StartDoctypeDeclHandler(self, doctypeName, systemId, publicId,
1303                 has_internal_subset):
1304                 if self._base_StartDoctypeDeclHandler is not None:
1305                         self._base_StartDoctypeDeclHandler(doctypeName, systemId, publicId,
1306                                 has_internal_subset)
1307                 self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId)
1308
1309 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1310         """
1311         Implements doctype() as required to avoid deprecation warnings with
1312         >=python-2.7.
1313         """
1314         def doctype(self, name, pubid, system):
1315                 pass
1316
1317 try:
1318         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1319 except (EnvironmentError, ParseError, PermissionDenied) as e:
1320         err(str(e))
1321 except FileNotFound:
1322         # TODO: Download as we do for metadata.dtd, but add a way to
1323         # disable for non-gentoo repoman users who may not have herds.
1324         herd_base = None
1325
1326 effective_scanlist = scanlist
1327 if options.if_modified == "y":
1328         effective_scanlist = sorted(vcs_files_to_cps(
1329                 chain(mychanged, mynew, myremoved)))
1330
1331 for x in effective_scanlist:
1332         #ebuilds and digests added to cvs respectively.
1333         logging.info("checking package %s" % x)
1334         # save memory by discarding xmatch caches from previous package(s)
1335         arch_xmatch_caches.clear()
1336         eadded = []
1337         catdir, pkgdir = x.split("/")
1338         checkdir = repodir + "/" + x
1339         checkdir_relative = ""
1340         if repolevel < 3:
1341                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1342         if repolevel < 2:
1343                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1344         checkdir_relative = os.path.join(".", checkdir_relative)
1345         generated_manifest = False
1346
1347         if options.mode == "manifest" or \
1348           (options.mode != 'manifest-check' and options.digest == 'y') or \
1349           options.mode in ('commit', 'fix') and not options.pretend:
1350                 auto_assumed = set()
1351                 fetchlist_dict = portage.FetchlistDict(checkdir,
1352                         repoman_settings, portdb)
1353                 if options.mode == 'manifest' and options.force:
1354                         portage._doebuild_manifest_exempt_depend += 1
1355                         try:
1356                                 distdir = repoman_settings['DISTDIR']
1357                                 mf = repoman_settings.repositories.get_repo_for_location(
1358                                         os.path.dirname(os.path.dirname(checkdir)))
1359                                 mf = mf.load_manifest(checkdir, distdir,
1360                                         fetchlist_dict=fetchlist_dict)
1361                                 mf.create(requiredDistfiles=None,
1362                                         assumeDistHashesAlways=True)
1363                                 for distfiles in fetchlist_dict.values():
1364                                         for distfile in distfiles:
1365                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1366                                                         mf.fhashdict['DIST'].pop(distfile, None)
1367                                                 else:
1368                                                         auto_assumed.add(distfile)
1369                                 mf.write()
1370                         finally:
1371                                 portage._doebuild_manifest_exempt_depend -= 1
1372
1373                 repoman_settings["O"] = checkdir
1374                 try:
1375                         generated_manifest = digestgen(
1376                                 mysettings=repoman_settings, myportdb=portdb)
1377                 except portage.exception.PermissionDenied as e:
1378                         generated_manifest = False
1379                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1380                                 level=logging.ERROR, noiselevel=-1)
1381
1382                 if not generated_manifest:
1383                         print("Unable to generate manifest.")
1384                         dofail = 1
1385
1386                 if options.mode == "manifest":
1387                         if not dofail and options.force and auto_assumed and \
1388                                 'assume-digests' in repoman_settings.features:
1389                                 # Show which digests were assumed despite the --force option
1390                                 # being given. This output will already have been shown by
1391                                 # digestgen() if assume-digests is not enabled, so only show
1392                                 # it here if assume-digests is enabled.
1393                                 pkgs = list(fetchlist_dict)
1394                                 pkgs.sort()
1395                                 portage.writemsg_stdout("  digest.assumed" + \
1396                                         portage.output.colorize("WARN",
1397                                         str(len(auto_assumed)).rjust(18)) + "\n")
1398                                 for cpv in pkgs:
1399                                         fetchmap = fetchlist_dict[cpv]
1400                                         pf = portage.catsplit(cpv)[1]
1401                                         for distfile in sorted(fetchmap):
1402                                                 if distfile in auto_assumed:
1403                                                         portage.writemsg_stdout(
1404                                                                 "   %s::%s\n" % (pf, distfile))
1405                         continue
1406                 elif dofail:
1407                         sys.exit(1)
1408
1409         if not generated_manifest:
1410                 repoman_settings['O'] = checkdir
1411                 repoman_settings['PORTAGE_QUIET'] = '1'
1412                 if not portage.digestcheck([], repoman_settings, strict=1):
1413                         stats["manifest.bad"] += 1
1414                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1415                 repoman_settings.pop('PORTAGE_QUIET', None)
1416
1417         if options.mode == 'manifest-check':
1418                 continue
1419
1420         checkdirlist = os.listdir(checkdir)
1421         ebuildlist = []
1422         pkgs = {}
1423         allvalid = True
1424         for y in checkdirlist:
1425                 if (y in no_exec or y.endswith(".ebuild")) and \
1426                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1427                                 stats["file.executable"] += 1
1428                                 fails["file.executable"].append(os.path.join(checkdir, y))
1429                 if y.endswith(".ebuild"):
1430                         pf = y[:-7]
1431                         ebuildlist.append(pf)
1432                         cpv = "%s/%s" % (catdir, pf)
1433                         try:
1434                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1435                         except KeyError:
1436                                 allvalid = False
1437                                 stats["ebuild.syntax"] += 1
1438                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1439                                 continue
1440                         except IOError:
1441                                 allvalid = False
1442                                 stats["ebuild.output"] += 1
1443                                 fails["ebuild.output"].append(os.path.join(x, y))
1444                                 continue
1445                         if not portage.eapi_is_supported(myaux["EAPI"]):
1446                                 allvalid = False
1447                                 stats["EAPI.unsupported"] += 1
1448                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1449                                 continue
1450                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1451                                 root_config=root_config, type_name="ebuild")
1452
1453         slot_keywords = {}
1454
1455         if len(pkgs) != len(ebuildlist):
1456                 # If we can't access all the metadata then it's totally unsafe to
1457                 # commit since there's no way to generate a correct Manifest.
1458                 # Do not try to do any more QA checks on this package since missing
1459                 # metadata leads to false positives for several checks, and false
1460                 # positives confuse users.
1461                 can_force = False
1462                 continue
1463
1464         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1465         ebuildlist = sorted(pkgs.values())
1466         ebuildlist = [pkg.pf for pkg in ebuildlist]
1467
1468         for y in checkdirlist:
1469                 index = repo_config.find_invalid_path_char(y)
1470                 if index != -1:
1471                         y_relative = os.path.join(checkdir_relative, y)
1472                         if vcs is not None and not vcs_new_changed(y_relative):
1473                                 # If the file isn't in the VCS new or changed set, then
1474                                 # assume that it's an irrelevant temporary file (Manifest
1475                                 # entries are not generated for file names containing
1476                                 # prohibited characters). See bug #406877.
1477                                 index = -1
1478                 if index != -1:
1479                         stats["file.name"] += 1
1480                         fails["file.name"].append("%s/%s: char '%s'" % \
1481                                 (checkdir, y, y[index]))
1482
1483                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1484                         continue
1485                 f = None
1486                 try:
1487                         line = 1
1488                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1489                                 encoding=_encodings['fs'], errors='strict'),
1490                                 mode='r', encoding=_encodings['repo.content'])
1491                         for l in f:
1492                                 line += 1
1493                 except UnicodeDecodeError as ue:
1494                         stats["file.UTF8"] += 1
1495                         s = ue.object[:ue.start]
1496                         l2 = s.count("\n")
1497                         line += l2
1498                         if l2 != 0:
1499                                 s = s[s.rfind("\n") + 1:]
1500                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1501                 finally:
1502                         if f is not None:
1503                                 f.close()
1504
1505         if vcs in ("git", "hg") and check_ebuild_notadded:
1506                 if vcs == "git":
1507                         myf = repoman_popen("git ls-files --others %s" % \
1508                                 (portage._shell_quote(checkdir_relative),))
1509                 if vcs == "hg":
1510                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1511                                 (portage._shell_quote(checkdir_relative),))
1512                 for l in myf:
1513                         if l[:-1][-7:] == ".ebuild":
1514                                 stats["ebuild.notadded"] += 1
1515                                 fails["ebuild.notadded"].append(
1516                                         os.path.join(x, os.path.basename(l[:-1])))
1517                 myf.close()
1518
1519         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1520                 try:
1521                         if vcs == "cvs":
1522                                 myf = open(checkdir + "/CVS/Entries", "r")
1523                         if vcs == "svn":
1524                                 myf = repoman_popen("svn status --depth=files --verbose " +
1525                                         portage._shell_quote(checkdir))
1526                         if vcs == "bzr":
1527                                 myf = repoman_popen("bzr ls -v --kind=file " +
1528                                         portage._shell_quote(checkdir))
1529                         myl = myf.readlines()
1530                         myf.close()
1531                         for l in myl:
1532                                 if vcs == "cvs":
1533                                         if l[0] != "/":
1534                                                 continue
1535                                         splitl = l[1:].split("/")
1536                                         if not len(splitl):
1537                                                 continue
1538                                         if splitl[0][-7:] == ".ebuild":
1539                                                 eadded.append(splitl[0][:-7])
1540                                 if vcs == "svn":
1541                                         if l[:1] == "?":
1542                                                 continue
1543                                         if l[:7] == '      >':
1544                                                 # tree conflict, new in subversion 1.6
1545                                                 continue
1546                                         l = l.split()[-1]
1547                                         if l[-7:] == ".ebuild":
1548                                                 eadded.append(os.path.basename(l[:-7]))
1549                                 if vcs == "bzr":
1550                                         if l[1:2] == "?":
1551                                                 continue
1552                                         l = l.split()[-1]
1553                                         if l[-7:] == ".ebuild":
1554                                                 eadded.append(os.path.basename(l[:-7]))
1555                         if vcs == "svn":
1556                                 myf = repoman_popen("svn status " +
1557                                         portage._shell_quote(checkdir))
1558                                 myl = myf.readlines()
1559                                 myf.close()
1560                                 for l in myl:
1561                                         if l[0] == "A":
1562                                                 l = l.rstrip().split(' ')[-1]
1563                                                 if l[-7:] == ".ebuild":
1564                                                         eadded.append(os.path.basename(l[:-7]))
1565                 except IOError:
1566                         if vcs == "cvs":
1567                                 stats["CVS/Entries.IO_error"] += 1
1568                                 fails["CVS/Entries.IO_error"].append(checkdir + "/CVS/Entries")
1569                         else:
1570                                 raise
1571                         continue
1572
1573         mf = repoman_settings.repositories.get_repo_for_location(
1574                 os.path.dirname(os.path.dirname(checkdir)))
1575         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1576         mydigests = mf.getTypeDigests("DIST")
1577
1578         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1579         myfiles_all = []
1580         src_uri_error = False
1581         for mykey in fetchlist_dict:
1582                 try:
1583                         myfiles_all.extend(fetchlist_dict[mykey])
1584                 except portage.exception.InvalidDependString as e:
1585                         src_uri_error = True
1586                         try:
1587                                 portdb.aux_get(mykey, ["SRC_URI"])
1588                         except KeyError:
1589                                 # This will be reported as an "ebuild.syntax" error.
1590                                 pass
1591                         else:
1592                                 stats["SRC_URI.syntax"] += 1
1593                                 fails["SRC_URI.syntax"].append(
1594                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1595         del fetchlist_dict
1596         if not src_uri_error:
1597                 # This test can produce false positives if SRC_URI could not
1598                 # be parsed for one or more ebuilds. There's no point in
1599                 # producing a false error here since the root cause will
1600                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1601                 # or "ebuild.sytax".
1602                 myfiles_all = set(myfiles_all)
1603                 for entry in mydigests:
1604                         if entry not in myfiles_all:
1605                                 stats["digest.unused"] += 1
1606                                 fails["digest.unused"].append(checkdir + "::" + entry)
1607                 for entry in myfiles_all:
1608                         if entry not in mydigests:
1609                                 stats["digest.missing"] += 1
1610                                 fails["digest.missing"].append(checkdir + "::" + entry)
1611         del myfiles_all
1612
1613         if os.path.exists(checkdir + "/files"):
1614                 filesdirlist = os.listdir(checkdir + "/files")
1615
1616                 # recurse through files directory
1617                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1618                 while filesdirlist:
1619                         y = filesdirlist.pop(0)
1620                         relative_path = os.path.join(x, "files", y)
1621                         full_path = os.path.join(repodir, relative_path)
1622                         try:
1623                                 mystat = os.stat(full_path)
1624                         except OSError as oe:
1625                                 if oe.errno == 2:
1626                                         # don't worry about it.  it likely was removed via fix above.
1627                                         continue
1628                                 else:
1629                                         raise oe
1630                         if S_ISDIR(mystat.st_mode):
1631                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1632                                 if y == "CVS" or y == ".svn":
1633                                         continue
1634                                 for z in os.listdir(checkdir + "/files/" + y):
1635                                         if z == "CVS" or z == ".svn":
1636                                                 continue
1637                                         filesdirlist.append(y + "/" + z)
1638                         # Current policy is no files over 20 KiB, these are the checks. File size between
1639                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1640                         elif mystat.st_size > 61440:
1641                                 stats["file.size.fatal"] += 1
1642                                 fails["file.size.fatal"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1643                         elif mystat.st_size > 20480:
1644                                 stats["file.size"] += 1
1645                                 fails["file.size"].append("(" + str(mystat.st_size//1024) + " KiB) " + x + "/files/" + y)
1646
1647                         index = repo_config.find_invalid_path_char(y)
1648                         if index != -1:
1649                                 y_relative = os.path.join(checkdir_relative, "files", y)
1650                                 if vcs is not None and not vcs_new_changed(y_relative):
1651                                         # If the file isn't in the VCS new or changed set, then
1652                                         # assume that it's an irrelevant temporary file (Manifest
1653                                         # entries are not generated for file names containing
1654                                         # prohibited characters). See bug #406877.
1655                                         index = -1
1656                         if index != -1:
1657                                 stats["file.name"] += 1
1658                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1659                                         (checkdir, y, y[index]))
1660         del mydigests
1661
1662         if check_changelog and "ChangeLog" not in checkdirlist:
1663                 stats["changelog.missing"] += 1
1664                 fails["changelog.missing"].append(x + "/ChangeLog")
1665
1666         musedict = {}
1667         # metadata.xml file check
1668         if "metadata.xml" not in checkdirlist:
1669                 stats["metadata.missing"] += 1
1670                 fails["metadata.missing"].append(x + "/metadata.xml")
1671         # metadata.xml parse check
1672         else:
1673                 metadata_bad = False
1674                 xml_info = {}
1675                 xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder())
1676
1677                 # read metadata.xml into memory
1678                 try:
1679                         _metadata_xml = xml.etree.ElementTree.parse(
1680                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1681                                 encoding=_encodings['fs'], errors='strict'),
1682                                 parser=xml_parser)
1683                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1684                         metadata_bad = True
1685                         stats["metadata.bad"] += 1
1686                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1687                         del e
1688                 else:
1689                         if not hasattr(xml_parser, 'parser') or \
1690                                 sys.hexversion < 0x2070000 or \
1691                                 (sys.hexversion > 0x3000000 and sys.hexversion < 0x3020000):
1692                                 # doctype is not parsed with python 2.6 or 3.1
1693                                 pass
1694                         else:
1695                                 if "XML_DECLARATION" not in xml_info:
1696                                         stats["metadata.bad"] += 1
1697                                         fails["metadata.bad"].append("%s/metadata.xml: "
1698                                                 "xml declaration is missing on first line, "
1699                                                 "should be '%s'" % (x, metadata_xml_declaration))
1700                                 else:
1701                                         xml_version, xml_encoding, xml_standalone = \
1702                                                 xml_info["XML_DECLARATION"]
1703                                         if xml_encoding is None or \
1704                                                 xml_encoding.upper() != metadata_xml_encoding:
1705                                                 stats["metadata.bad"] += 1
1706                                                 if xml_encoding is None:
1707                                                         encoding_problem = "but it is undefined"
1708                                                 else:
1709                                                         encoding_problem = "not '%s'" % xml_encoding
1710                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1711                                                         "xml declaration encoding should be '%s', %s" %
1712                                                         (x, metadata_xml_encoding, encoding_problem))
1713
1714                                 if "DOCTYPE" not in xml_info:
1715                                         metadata_bad = True
1716                                         stats["metadata.bad"] += 1
1717                                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x,
1718                                                 "DOCTYPE is missing"))
1719                                 else:
1720                                         doctype_name, doctype_system, doctype_pubid = \
1721                                                 xml_info["DOCTYPE"]
1722                                         if doctype_system != metadata_dtd_uri:
1723                                                 stats["metadata.bad"] += 1
1724                                                 if doctype_system is None:
1725                                                         system_problem = "but it is undefined"
1726                                                 else:
1727                                                         system_problem = "not '%s'" % doctype_system
1728                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1729                                                         "DOCTYPE: SYSTEM should refer to '%s', %s" %
1730                                                         (x, metadata_dtd_uri, system_problem))
1731
1732                                         if doctype_name != metadata_doctype_name:
1733                                                 stats["metadata.bad"] += 1
1734                                                 fails["metadata.bad"].append("%s/metadata.xml: "
1735                                                         "DOCTYPE: name should be '%s', not '%s'" %
1736                                                         (x, metadata_doctype_name, doctype_name))
1737
1738                         # load USE flags from metadata.xml
1739                         try:
1740                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1741                         except portage.exception.ParseError as e:
1742                                 metadata_bad = True
1743                                 stats["metadata.bad"] += 1
1744                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1745                         else:
1746                                 for atom in chain(*musedict.values()):
1747                                         if atom is None:
1748                                                 continue
1749                                         try:
1750                                                 atom = Atom(atom)
1751                                         except InvalidAtom as e:
1752                                                 stats["metadata.bad"] += 1
1753                                                 fails["metadata.bad"].append(
1754                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1755                                         else:
1756                                                 if atom.cp != x:
1757                                                         stats["metadata.bad"] += 1
1758                                                         fails["metadata.bad"].append(
1759                                                                 ("%s/metadata.xml: Atom contains "
1760                                                                 "unexpected cat/pn: %s") % (x, atom))
1761
1762                         # Run other metadata.xml checkers
1763                         try:
1764                                 utilities.check_metadata(_metadata_xml, herd_base)
1765                         except (utilities.UnknownHerdsError, ) as e:
1766                                 metadata_bad = True
1767                                 stats["metadata.bad"] += 1
1768                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1769                                 del e
1770
1771                 #Only carry out if in package directory or check forced
1772                 if xmllint_capable and not metadata_bad:
1773                         # xmlint can produce garbage output even on success, so only dump
1774                         # the ouput when it fails.
1775                         st, out = repoman_getstatusoutput(
1776                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1777                                  (portage._shell_quote(metadata_dtd),
1778                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1779                         if st != os.EX_OK:
1780                                 print(red("!!!") + " metadata.xml is invalid:")
1781                                 for z in out.splitlines():
1782                                         print(red("!!! ") + z)
1783                                 stats["metadata.bad"] += 1
1784                                 fails["metadata.bad"].append(x + "/metadata.xml")
1785
1786                 del metadata_bad
1787         muselist = frozenset(musedict)
1788
1789         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1790         changelog_modified = changelog_path in modified_changelogs
1791
1792         # detect unused local USE-descriptions
1793         used_useflags = set()
1794
1795         for y in ebuildlist:
1796                 relative_path = os.path.join(x, y + ".ebuild")
1797                 full_path = os.path.join(repodir, relative_path)
1798                 ebuild_path = y + ".ebuild"
1799                 if repolevel < 3:
1800                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1801                 if repolevel < 2:
1802                         ebuild_path = os.path.join(catdir, ebuild_path)
1803                 ebuild_path = os.path.join(".", ebuild_path)
1804                 if check_changelog and not changelog_modified \
1805                         and ebuild_path in new_ebuilds:
1806                         stats['changelog.ebuildadded'] += 1
1807                         fails['changelog.ebuildadded'].append(relative_path)
1808
1809                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1810                         #ebuild not added to vcs
1811                         stats["ebuild.notadded"] += 1
1812                         fails["ebuild.notadded"].append(x + "/" + y + ".ebuild")
1813                 myesplit = portage.pkgsplit(y)
1814                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1815                             or pv_toolong_re.search(myesplit[1]) \
1816                             or pv_toolong_re.search(myesplit[2]):
1817                         stats["ebuild.invalidname"] += 1
1818                         fails["ebuild.invalidname"].append(x + "/" + y + ".ebuild")
1819                         continue
1820                 elif myesplit[0] != pkgdir:
1821                         print(pkgdir, myesplit[0])
1822                         stats["ebuild.namenomatch"] += 1
1823                         fails["ebuild.namenomatch"].append(x + "/" + y + ".ebuild")
1824                         continue
1825
1826                 pkg = pkgs[y]
1827
1828                 if pkg.invalid:
1829                         allvalid = False
1830                         for k, msgs in pkg.invalid.items():
1831                                 for msg in msgs:
1832                                         stats[k] += 1
1833                                         fails[k].append("%s: %s" % (relative_path, msg))
1834                         continue
1835
1836                 myaux = pkg._metadata
1837                 eapi = myaux["EAPI"]
1838                 inherited = pkg.inherited
1839                 live_ebuild = live_eclasses.intersection(inherited)
1840
1841                 for k, v in myaux.items():
1842                         if not isinstance(v, basestring):
1843                                 continue
1844                         m = non_ascii_re.search(v)
1845                         if m is not None:
1846                                 stats["variable.invalidchar"] += 1
1847                                 fails["variable.invalidchar"].append(
1848                                         ("%s: %s variable contains non-ASCII " + \
1849                                         "character at position %s") % \
1850                                         (relative_path, k, m.start() + 1))
1851
1852                 if not src_uri_error:
1853                         # Check that URIs don't reference a server from thirdpartymirrors.
1854                         for uri in portage.dep.use_reduce( \
1855                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1856                                 contains_mirror = False
1857                                 for mirror, mirror_alias in thirdpartymirrors.items():
1858                                         if uri.startswith(mirror):
1859                                                 contains_mirror = True
1860                                                 break
1861                                 if not contains_mirror:
1862                                         continue
1863
1864                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1865                                 stats["SRC_URI.mirror"] += 1
1866                                 fails["SRC_URI.mirror"].append(
1867                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1868                                         (relative_path, mirror, new_uri))
1869
1870                 if myaux.get("PROVIDE"):
1871                         stats["virtual.oldstyle"] += 1
1872                         fails["virtual.oldstyle"].append(relative_path)
1873
1874                 for pos, missing_var in enumerate(missingvars):
1875                         if not myaux.get(missing_var):
1876                                 if catdir == "virtual" and \
1877                                         missing_var in ("HOMEPAGE", "LICENSE"):
1878                                         continue
1879                                 if live_ebuild and missing_var == "KEYWORDS":
1880                                         continue
1881                                 myqakey = missingvars[pos] + ".missing"
1882                                 stats[myqakey] += 1
1883                                 fails[myqakey].append(x + "/" + y + ".ebuild")
1884
1885                 if catdir == "virtual":
1886                         for var in ("HOMEPAGE", "LICENSE"):
1887                                 if myaux.get(var):
1888                                         myqakey = var + ".virtual"
1889                                         stats[myqakey] += 1
1890                                         fails[myqakey].append(relative_path)
1891
1892                 # 14 is the length of DESCRIPTION=""
1893                 if len(myaux['DESCRIPTION']) > max_desc_len:
1894                         stats['DESCRIPTION.toolong'] += 1
1895                         fails['DESCRIPTION.toolong'].append(
1896                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1897                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1898
1899                 keywords = myaux["KEYWORDS"].split()
1900                 stable_keywords = []
1901                 for keyword in keywords:
1902                         if not keyword.startswith("~") and \
1903                                 not keyword.startswith("-"):
1904                                 stable_keywords.append(keyword)
1905                 if stable_keywords:
1906                         if ebuild_path in new_ebuilds and catdir != "virtual":
1907                                 stable_keywords.sort()
1908                                 stats["KEYWORDS.stable"] += 1
1909                                 fails["KEYWORDS.stable"].append(
1910                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1911                                                 " ".join(stable_keywords))
1912
1913                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1914                         if not kw.startswith("-"))
1915
1916                 previous_keywords = slot_keywords.get(pkg.slot)
1917                 if previous_keywords is None:
1918                         slot_keywords[pkg.slot] = set()
1919                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1920                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1921                         if dropped_keywords:
1922                                 stats["KEYWORDS.dropped"] += 1
1923                                 fails["KEYWORDS.dropped"].append(
1924                                         relative_path + ": %s" % \
1925                                         " ".join(sorted(dropped_keywords)))
1926
1927                 slot_keywords[pkg.slot].update(ebuild_archs)
1928
1929                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1930                 if "-*" in keywords:
1931                         haskeyword = False
1932                         for kw in keywords:
1933                                 if kw[0] == "~":
1934                                         kw = kw[1:]
1935                                 if kw in kwlist:
1936                                         haskeyword = True
1937                         if not haskeyword:
1938                                 stats["KEYWORDS.stupid"] += 1
1939                                 fails["KEYWORDS.stupid"].append(x + "/" + y + ".ebuild")
1940
1941                 """
1942                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1943                 not be allowed to be marked stable
1944                 """
1945                 if live_ebuild and repo_config.name == "gentoo":
1946                         bad_stable_keywords = []
1947                         for keyword in keywords:
1948                                 if not keyword.startswith("~") and \
1949                                         not keyword.startswith("-"):
1950                                         bad_stable_keywords.append(keyword)
1951                                 del keyword
1952                         if bad_stable_keywords:
1953                                 stats["LIVEVCS.stable"] += 1
1954                                 fails["LIVEVCS.stable"].append(
1955                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1956                                                 bad_stable_keywords)
1957                         del bad_stable_keywords
1958
1959                         if keywords and not has_global_mask(pkg):
1960                                 stats["LIVEVCS.unmasked"] += 1
1961                                 fails["LIVEVCS.unmasked"].append(relative_path)
1962
1963                 if options.ignore_arches:
1964                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1965                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1966                 else:
1967                         arches = set()
1968                         for keyword in keywords:
1969                                 if keyword[0] == "-":
1970                                         continue
1971                                 elif keyword[0] == "~":
1972                                         arch = keyword[1:]
1973                                         if arch == "*":
1974                                                 for expanded_arch in profiles:
1975                                                         if expanded_arch == "**":
1976                                                                 continue
1977                                                         arches.add((keyword, expanded_arch,
1978                                                                 (expanded_arch, "~" + expanded_arch)))
1979                                         else:
1980                                                 arches.add((keyword, arch, (arch, keyword)))
1981                                 else:
1982                                         if keyword == "*":
1983                                                 for expanded_arch in profiles:
1984                                                         if expanded_arch == "**":
1985                                                                 continue
1986                                                         arches.add((keyword, expanded_arch,
1987                                                                 (expanded_arch,)))
1988                                         else:
1989                                                 arches.add((keyword, keyword, (keyword,)))
1990                         if not arches:
1991                                 # Use an empty profile for checking dependencies of
1992                                 # packages that have empty KEYWORDS.
1993                                 arches.add(('**', '**', ('**',)))
1994
1995                 unknown_pkgs = set()
1996                 baddepsyntax = False
1997                 badlicsyntax = False
1998                 badprovsyntax = False
1999                 catpkg = catdir + "/" + y
2000
2001                 inherited_java_eclass = "java-pkg-2" in inherited or \
2002                         "java-pkg-opt-2" in inherited
2003                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
2004                 operator_tokens = set(["||", "(", ")"])
2005                 type_list, badsyntax = [], []
2006                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
2007                         mydepstr = myaux[mytype]
2008
2009                         buildtime = mytype in Package._buildtime_keys
2010                         runtime = mytype in Package._runtime_keys
2011                         token_class = None
2012                         if mytype.endswith("DEPEND"):
2013                                 token_class = portage.dep.Atom
2014
2015                         try:
2016                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
2017                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
2018                         except portage.exception.InvalidDependString as e:
2019                                 atoms = None
2020                                 badsyntax.append(str(e))
2021
2022                         if atoms and mytype.endswith("DEPEND"):
2023                                 if runtime and \
2024                                         "test?" in mydepstr.split():
2025                                         stats[mytype + '.suspect'] += 1
2026                                         fails[mytype + '.suspect'].append(relative_path + \
2027                                                 ": 'test?' USE conditional in %s" % mytype)
2028
2029                                 for atom in atoms:
2030                                         if atom == "||":
2031                                                 continue
2032
2033                                         # Skip dependency.unknown for blockers, so that we
2034                                         # don't encourage people to remove necessary blockers,
2035                                         # as discussed in bug #382407.
2036                                         if atom.blocker is None and \
2037                                                 not portdb.xmatch("match-all", atom) and \
2038                                                 not atom.cp.startswith("virtual/"):
2039                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
2040
2041                                         is_blocker = atom.blocker
2042
2043                                         if catdir != "virtual":
2044                                                 if not is_blocker and \
2045                                                         atom.cp in suspect_virtual:
2046                                                         stats['virtual.suspect'] += 1
2047                                                         fails['virtual.suspect'].append(
2048                                                                 relative_path +
2049                                                                 ": %s: consider using '%s' instead of '%s'" %
2050                                                                 (mytype, suspect_virtual[atom.cp], atom))
2051
2052                                         if buildtime and \
2053                                                 not is_blocker and \
2054                                                 not inherited_java_eclass and \
2055                                                 atom.cp == "virtual/jdk":
2056                                                 stats['java.eclassesnotused'] += 1
2057                                                 fails['java.eclassesnotused'].append(relative_path)
2058                                         elif buildtime and \
2059                                                 not is_blocker and \
2060                                                 not inherited_wxwidgets_eclass and \
2061                                                 atom.cp == "x11-libs/wxGTK":
2062                                                 stats['wxwidgets.eclassnotused'] += 1
2063                                                 fails['wxwidgets.eclassnotused'].append(
2064                                                         (relative_path + ": %ss on x11-libs/wxGTK"
2065                                                         " without inheriting wxwidgets.eclass") % mytype)
2066                                         elif runtime:
2067                                                 if not is_blocker and \
2068                                                         atom.cp in suspect_rdepend:
2069                                                         stats[mytype + '.suspect'] += 1
2070                                                         fails[mytype + '.suspect'].append(
2071                                                                 relative_path + ": '%s'" % atom)
2072
2073                                         if atom.operator == "~" and \
2074                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
2075                                                 qacat = 'dependency.badtilde'
2076                                                 stats[qacat] += 1
2077                                                 fails[qacat].append(
2078                                                         (relative_path + ": %s uses the ~ operator"
2079                                                          " with a non-zero revision:" + \
2080                                                          " '%s'") % (mytype, atom))
2081
2082                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
2083
2084                 for m, b in zip(type_list, badsyntax):
2085                         if m.endswith("DEPEND"):
2086                                 qacat = "dependency.syntax"
2087                         else:
2088                                 qacat = m + ".syntax"
2089                         stats[qacat] += 1
2090                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
2091
2092                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
2093                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2094                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2095                 badlicsyntax = badlicsyntax > 0
2096                 badprovsyntax = badprovsyntax > 0
2097
2098                 # uselist checks - global
2099                 myuse = []
2100                 default_use = []
2101                 for myflag in myaux["IUSE"].split():
2102                         flag_name = myflag.lstrip("+-")
2103                         used_useflags.add(flag_name)
2104                         if myflag != flag_name:
2105                                 default_use.append(myflag)
2106                         if flag_name not in uselist:
2107                                 myuse.append(flag_name)
2108
2109                 # uselist checks - metadata
2110                 for mypos in range(len(myuse)-1, -1, -1):
2111                         if myuse[mypos] and (myuse[mypos] in muselist):
2112                                 del myuse[mypos]
2113
2114                 if default_use and not eapi_has_iuse_defaults(eapi):
2115                         for myflag in default_use:
2116                                 stats['EAPI.incompatible'] += 1
2117                                 fails['EAPI.incompatible'].append(
2118                                         (relative_path + ": IUSE defaults" + \
2119                                         " not supported with EAPI='%s':" + \
2120                                         " '%s'") % (eapi, myflag))
2121
2122                 for mypos in range(len(myuse)):
2123                         stats["IUSE.invalid"] += 1
2124                         fails["IUSE.invalid"].append(x + "/" + y + ".ebuild: %s" % myuse[mypos])
2125
2126                 # license checks
2127                 if not badlicsyntax:
2128                         # Parse the LICENSE variable, remove USE conditions and
2129                         # flatten it.
2130                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2131                         # Check each entry to ensure that it exists in PORTDIR's
2132                         # license directory.
2133                         for lic in licenses:
2134                                 # Need to check for "||" manually as no portage
2135                                 # function will remove it without removing values.
2136                                 if lic not in liclist and lic != "||":
2137                                         stats["LICENSE.invalid"] += 1
2138                                         fails["LICENSE.invalid"].append(x + "/" + y + ".ebuild: %s" % lic)
2139                                 elif lic in liclist_deprecated:
2140                                         stats["LICENSE.deprecated"] += 1
2141                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2142
2143                 #keyword checks
2144                 myuse = myaux["KEYWORDS"].split()
2145                 for mykey in myuse:
2146                         if mykey not in ("-*", "*", "~*"):
2147                                 myskey = mykey
2148                                 if myskey[:1] == "-":
2149                                         myskey = myskey[1:]
2150                                 if myskey[:1] == "~":
2151                                         myskey = myskey[1:]
2152                                 if myskey not in kwlist:
2153                                         stats["KEYWORDS.invalid"] += 1
2154                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s" % mykey)
2155                                 elif myskey not in profiles:
2156                                         stats["KEYWORDS.invalid"] += 1
2157                                         fails["KEYWORDS.invalid"].append(x + "/" + y + ".ebuild: %s (profile invalid)" % mykey)
2158
2159                 #restrict checks
2160                 myrestrict = None
2161                 try:
2162                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2163                 except portage.exception.InvalidDependString as e:
2164                         stats["RESTRICT.syntax"] += 1
2165                         fails["RESTRICT.syntax"].append(
2166                                 "%s: RESTRICT: %s" % (relative_path, e))
2167                         del e
2168                 if myrestrict:
2169                         myrestrict = set(myrestrict)
2170                         mybadrestrict = myrestrict.difference(valid_restrict)
2171                         if mybadrestrict:
2172                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2173                                 for mybad in mybadrestrict:
2174                                         fails["RESTRICT.invalid"].append(x + "/" + y + ".ebuild: %s" % mybad)
2175                 #REQUIRED_USE check
2176                 required_use = myaux["REQUIRED_USE"]
2177                 if required_use:
2178                         if not eapi_has_required_use(eapi):
2179                                 stats['EAPI.incompatible'] += 1
2180                                 fails['EAPI.incompatible'].append(
2181                                         relative_path + ": REQUIRED_USE" + \
2182                                         " not supported with EAPI='%s'" % (eapi,))
2183                         try:
2184                                 portage.dep.check_required_use(required_use, (),
2185                                         pkg.iuse.is_valid_flag, eapi=eapi)
2186                         except portage.exception.InvalidDependString as e:
2187                                 stats["REQUIRED_USE.syntax"] += 1
2188                                 fails["REQUIRED_USE.syntax"].append(
2189                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2190                                 del e
2191
2192                 # Syntax Checks
2193                 relative_path = os.path.join(x, y + ".ebuild")
2194                 full_path = os.path.join(repodir, relative_path)
2195                 if not vcs_preserves_mtime:
2196                         if ebuild_path not in new_ebuilds and \
2197                                 ebuild_path not in modified_ebuilds:
2198                                 pkg.mtime = None
2199                 try:
2200                         # All ebuilds should have utf_8 encoding.
2201                         f = io.open(_unicode_encode(full_path,
2202                                 encoding=_encodings['fs'], errors='strict'),
2203                                 mode='r', encoding=_encodings['repo.content'])
2204                         try:
2205                                 for check_name, e in run_checks(f, pkg):
2206                                         stats[check_name] += 1
2207                                         fails[check_name].append(relative_path + ': %s' % e)
2208                         finally:
2209                                 f.close()
2210                 except UnicodeDecodeError:
2211                         # A file.UTF8 failure will have already been recorded above.
2212                         pass
2213
2214                 if options.force:
2215                         # The dep_check() calls are the most expensive QA test. If --force
2216                         # is enabled, there's no point in wasting time on these since the
2217                         # user is intent on forcing the commit anyway.
2218                         continue
2219
2220                 relevant_profiles = []
2221                 for keyword, arch, groups in arches:
2222                         if arch not in profiles:
2223                                 # A missing profile will create an error further down
2224                                 # during the KEYWORDS verification.
2225                                 continue
2226                         relevant_profiles.extend((keyword, groups, prof)
2227                                 for prof in profiles[arch])
2228
2229                 def sort_key(item):
2230                         return item[2].sub_path
2231
2232                 relevant_profiles.sort(key=sort_key)
2233
2234                 for keyword, groups, prof in relevant_profiles:
2235
2236                                 if prof.status not in ("stable", "dev") or \
2237                                         prof.status == "dev" and not options.include_dev:
2238                                         continue
2239
2240                                 dep_settings = arch_caches.get(prof.sub_path)
2241                                 if dep_settings is None:
2242                                         dep_settings = portage.config(
2243                                                 config_profile_path=prof.abs_path,
2244                                                 config_incrementals=repoman_incrementals,
2245                                                 config_root=config_root,
2246                                                 local_config=False,
2247                                                 _unmatched_removal=options.unmatched_removal,
2248                                                 env=env)
2249                                         dep_settings.categories = repoman_settings.categories
2250                                         if options.without_mask:
2251                                                 dep_settings._mask_manager_obj = \
2252                                                         copy.deepcopy(dep_settings._mask_manager)
2253                                                 dep_settings._mask_manager._pmaskdict.clear()
2254                                         arch_caches[prof.sub_path] = dep_settings
2255
2256                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2257                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2258                                 if xcache is None:
2259                                         portdb.melt()
2260                                         portdb.freeze()
2261                                         xcache = portdb.xcache
2262                                         xcache.update(shared_xmatch_caches)
2263                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2264
2265                                 trees[root]["porttree"].settings = dep_settings
2266                                 portdb.settings = dep_settings
2267                                 portdb.xcache = xcache
2268                                 # for package.use.mask support inside dep_check
2269                                 dep_settings.setcpv(pkg)
2270                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2271                                 # just in case, prevent config.reset() from nuking these.
2272                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2273
2274                                 # This attribute is used in dbapi._match_use() to apply
2275                                 # use.stable.{mask,force} settings based on the stable
2276                                 # status of the parent package. This is required in order
2277                                 # for USE deps of unstable packages to be resolved correctly,
2278                                 # since otherwise use.stable.{mask,force} settings of
2279                                 # dependencies may conflict (see bug #456342).
2280                                 dep_settings._parent_stable = dep_settings._isStable(pkg)
2281
2282                                 if not baddepsyntax:
2283                                         ismasked = not ebuild_archs or \
2284                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2285                                         if ismasked:
2286                                                 if not have_pmasked:
2287                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2288                                                                 pkg.cpv, pkg._metadata))
2289                                                 if options.ignore_masked:
2290                                                         continue
2291                                                 #we are testing deps for a masked package; give it some lee-way
2292                                                 suffix = "masked"
2293                                                 matchmode = "minimum-all"
2294                                         else:
2295                                                 suffix = ""
2296                                                 matchmode = "minimum-visible"
2297
2298                                         if not have_dev_keywords:
2299                                                 have_dev_keywords = \
2300                                                         bool(dev_keywords.intersection(keywords))
2301
2302                                         if prof.status == "dev":
2303                                                 suffix = suffix + "indev"
2304
2305                                         for mytype in Package._dep_keys:
2306
2307                                                 mykey = "dependency.bad" + suffix
2308                                                 myvalue = myaux[mytype]
2309                                                 if not myvalue:
2310                                                         continue
2311
2312                                                 success, atoms = portage.dep_check(myvalue, portdb,
2313                                                         dep_settings, use="all", mode=matchmode,
2314                                                         trees=trees)
2315
2316                                                 if success:
2317                                                         if atoms:
2318
2319                                                                 # Don't bother with dependency.unknown for
2320                                                                 # cases in which *DEPEND.bad is triggered.
2321                                                                 for atom in atoms:
2322                                                                         # dep_check returns all blockers and they
2323                                                                         # aren't counted for *DEPEND.bad, so we
2324                                                                         # ignore them here.
2325                                                                         if not atom.blocker:
2326                                                                                 unknown_pkgs.discard(
2327                                                                                         (mytype, atom.unevaluated_atom))
2328
2329                                                                 if not prof.sub_path:
2330                                                                         # old-style virtuals currently aren't
2331                                                                         # resolvable with empty profile, since
2332                                                                         # 'virtuals' mappings are unavailable
2333                                                                         # (it would be expensive to search
2334                                                                         # for PROVIDE in all ebuilds)
2335                                                                         atoms = [atom for atom in atoms if not \
2336                                                                                 (atom.cp.startswith('virtual/') and \
2337                                                                                 not portdb.cp_list(atom.cp))]
2338
2339                                                                 #we have some unsolvable deps
2340                                                                 #remove ! deps, which always show up as unsatisfiable
2341                                                                 atoms = [str(atom.unevaluated_atom) \
2342                                                                         for atom in atoms if not atom.blocker]
2343
2344                                                                 #if we emptied out our list, continue:
2345                                                                 if not atoms:
2346                                                                         continue
2347                                                                 stats[mykey] += 1
2348                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2349                                                                         (relative_path, mytype, keyword,
2350                                                                         prof, repr(atoms)))
2351                                                 else:
2352                                                         stats[mykey] += 1
2353                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2354                                                                 (relative_path, mytype, keyword,
2355                                                                 prof, repr(atoms)))
2356
2357                 if not baddepsyntax and unknown_pkgs:
2358                         type_map = {}
2359                         for mytype, atom in unknown_pkgs:
2360                                 type_map.setdefault(mytype, set()).add(atom)
2361                         for mytype, atoms in type_map.items():
2362                                 stats["dependency.unknown"] += 1
2363                                 fails["dependency.unknown"].append("%s: %s: %s" %
2364                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2365
2366         # check if there are unused local USE-descriptions in metadata.xml
2367         # (unless there are any invalids, to avoid noise)
2368         if allvalid:
2369                 for myflag in muselist.difference(used_useflags):
2370                         stats["metadata.warning"] += 1
2371                         fails["metadata.warning"].append(
2372                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2373                                 (x, myflag))
2374
2375 if options.if_modified == "y" and len(effective_scanlist) < 1:
2376         logging.warn("--if-modified is enabled, but no modified packages were found!")
2377
2378 if options.mode == "manifest":
2379         sys.exit(dofail)
2380
2381 # dofail will be set to 1 if we have failed in at least one non-warning category
2382 dofail = 0
2383 # dowarn will be set to 1 if we tripped any warnings
2384 dowarn = 0
2385 # dofull will be set if we should print a "repoman full" informational message
2386 dofull = options.mode != 'full'
2387
2388 for x in qacats:
2389         if not stats[x]:
2390                 continue
2391         dowarn = 1
2392         if x not in qawarnings:
2393                 dofail = 1
2394
2395 if dofail or \
2396         (dowarn and not (options.quiet or options.mode == "scan")):
2397         dofull = 0
2398
2399 # Save QA output so that it can be conveniently displayed
2400 # in $EDITOR while the user creates a commit message.
2401 # Otherwise, the user would not be able to see this output
2402 # once the editor has taken over the screen.
2403 qa_output = io.StringIO()
2404 style_file = ConsoleStyleFile(sys.stdout)
2405 if options.mode == 'commit' and \
2406         (not commitmessage or not commitmessage.strip()):
2407         style_file.write_listener = qa_output
2408 console_writer = StyleWriter(file=style_file, maxcol=9999)
2409 console_writer.style_listener = style_file.new_styles
2410
2411 f = formatter.AbstractFormatter(console_writer)
2412
2413 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2414
2415 style_file.flush()
2416 del console_writer, f, style_file
2417 qa_output = qa_output.getvalue()
2418 qa_output = qa_output.splitlines(True)
2419
2420 suggest_ignore_masked = False
2421 suggest_include_dev = False
2422
2423 if have_pmasked and not (options.without_mask or options.ignore_masked):
2424         suggest_ignore_masked = True
2425 if have_dev_keywords and not options.include_dev:
2426         suggest_include_dev = True
2427
2428 if suggest_ignore_masked or suggest_include_dev:
2429         print()
2430         if suggest_ignore_masked:
2431                 print(bold("Note: use --without-mask to check " + \
2432                         "KEYWORDS on dependencies of masked packages"))
2433
2434         if suggest_include_dev:
2435                 print(bold("Note: use --include-dev (-d) to check " + \
2436                         "dependencies for 'dev' profiles"))
2437         print()
2438
2439 if options.mode != 'commit':
2440         if dofull:
2441                 print(bold("Note: type \"repoman full\" for a complete listing."))
2442         if dowarn and not dofail:
2443                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2444         elif not dofail:
2445                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2446         elif dofail:
2447                 print(bad("Please fix these important QA issues first."))
2448                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2449                 sys.exit(1)
2450 else:
2451         if dofail and can_force and options.force and not options.pretend:
2452                 print(green("RepoMan sez:") + \
2453                         " \"You want to commit even with these QA issues?\n" + \
2454                         "              I'll take it this time, but I'm not happy.\"\n")
2455         elif dofail:
2456                 if options.force and not can_force:
2457                         print(bad("The --force option has been disabled due to extraordinary issues."))
2458                 print(bad("Please fix these important QA issues first."))
2459                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2460                 sys.exit(1)
2461
2462         if options.pretend:
2463                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2464
2465         myunadded = []
2466         if vcs == "cvs":
2467                 try:
2468                         myvcstree = portage.cvstree.getentries("./", recursive=1)
2469                         myunadded = portage.cvstree.findunadded(myvcstree, recursive=1, basedir="./")
2470                 except SystemExit as e:
2471                         raise  # TODO propagate this
2472                 except:
2473                         err("Error retrieving CVS tree; exiting.")
2474         if vcs == "svn":
2475                 try:
2476                         with repoman_popen("svn status --no-ignore") as f:
2477                                 svnstatus = f.readlines()
2478                         myunadded = ["./" + elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I")]
2479                 except SystemExit as e:
2480                         raise  # TODO propagate this
2481                 except:
2482                         err("Error retrieving SVN info; exiting.")
2483         if vcs == "git":
2484                 # get list of files not under version control or missing
2485                 myf = repoman_popen("git ls-files --others")
2486                 myunadded = ["./" + elem[:-1] for elem in myf]
2487                 myf.close()
2488         if vcs == "bzr":
2489                 try:
2490                         with repoman_popen("bzr status -S .") as f:
2491                                 bzrstatus = f.readlines()
2492                         myunadded = ["./" + elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D"]
2493                 except SystemExit as e:
2494                         raise  # TODO propagate this
2495                 except:
2496                         err("Error retrieving bzr info; exiting.")
2497         if vcs == "hg":
2498                 with repoman_popen("hg status --no-status --unknown .") as f:
2499                         myunadded = f.readlines()
2500                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2501
2502                 # Mercurial doesn't handle manually deleted files as removed from
2503                 # the repository, so the user need to remove them before commit,
2504                 # using "hg remove [FILES]"
2505                 with repoman_popen("hg status --no-status --deleted .") as f:
2506                         mydeleted = f.readlines()
2507                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2508
2509
2510         myautoadd = []
2511         if myunadded:
2512                 for x in range(len(myunadded)-1, -1, -1):
2513                         xs = myunadded[x].split("/")
2514                         if xs[-1] == "files":
2515                                 print("!!! files dir is not added! Please correct this.")
2516                                 sys.exit(-1)
2517                         elif xs[-1] == "Manifest":
2518                                 # It's a manifest... auto add
2519                                 myautoadd += [myunadded[x]]
2520                                 del myunadded[x]
2521
2522         if myunadded:
2523                 print(red("!!! The following files are in your local tree but are not added to the master"))
2524                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2525                 for x in myunadded:
2526                         print("   ", x)
2527                 print()
2528                 print()
2529                 sys.exit(1)
2530
2531         if vcs == "hg" and mydeleted:
2532                 print(red("!!! The following files are removed manually from your local tree but are not"))
2533                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2534                 for x in mydeleted:
2535                         print("   ", x)
2536                 print()
2537                 print()
2538                 sys.exit(1)
2539
2540         if vcs == "cvs":
2541                 mycvstree = cvstree.getentries("./", recursive=1)
2542                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2543                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2544                 myremoved = portage.cvstree.findremoved(mycvstree, recursive=1, basedir="./")
2545                 bin_blob_pattern = re.compile("^-kb$")
2546                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2547                         recursive=1, basedir="./"))
2548
2549         if vcs == "svn":
2550                 with repoman_popen("svn status") as f:
2551                         svnstatus = f.readlines()
2552                 mychanged = ["./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2553                 mynew     = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2554                 myremoved = ["./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2555
2556                 # Subversion expands keywords specified in svn:keywords properties.
2557                 with repoman_popen("svn propget -R svn:keywords") as f:
2558                         props = f.readlines()
2559                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2560                         for prop in props if " - " in prop)
2561
2562         elif vcs == "git":
2563                 with repoman_popen("git diff-index --name-only "
2564                         "--relative --diff-filter=M HEAD") as f:
2565                         mychanged = f.readlines()
2566                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2567
2568                 with repoman_popen("git diff-index --name-only "
2569                         "--relative --diff-filter=A HEAD") as f:
2570                         mynew = f.readlines()
2571                 mynew = ["./" + elem[:-1] for elem in mynew]
2572
2573                 with repoman_popen("git diff-index --name-only "
2574                         "--relative --diff-filter=D HEAD") as f:
2575                         myremoved = f.readlines()
2576                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2577
2578         if vcs == "bzr":
2579                 with repoman_popen("bzr status -S .") as f:
2580                         bzrstatus = f.readlines()
2581                 mychanged = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M"]
2582                 mynew     = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
2583                 myremoved = ["./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-")]
2584                 myremoved = ["./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
2585                 # Bazaar expands nothing.
2586
2587         if vcs == "hg":
2588                 with repoman_popen("hg status --no-status --modified .") as f:
2589                         mychanged = f.readlines()
2590                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2591
2592                 with repoman_popen("hg status --no-status --added .") as f:
2593                         mynew = f.readlines()
2594                 mynew = ["./" + elem.rstrip() for elem in mynew]
2595
2596                 with repoman_popen("hg status --no-status --removed .") as f:
2597                         myremoved = f.readlines()
2598                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2599
2600         if vcs:
2601                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2602                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2603                         print()
2604                         print("(Didn't find any changed files...)")
2605                         print()
2606                         sys.exit(1)
2607
2608         # Manifests need to be regenerated after all other commits, so don't commit
2609         # them now even if they have changed.
2610         mymanifests = set()
2611         myupdates = set()
2612         for f in mychanged + mynew:
2613                 if "Manifest" == os.path.basename(f):
2614                         mymanifests.add(f)
2615                 else:
2616                         myupdates.add(f)
2617         myupdates.difference_update(myremoved)
2618         myupdates = list(myupdates)
2619         mymanifests = list(mymanifests)
2620         myheaders = []
2621         mydirty = []
2622
2623         commitmessage = options.commitmsg
2624         if options.commitmsgfile:
2625                 try:
2626                         f = io.open(_unicode_encode(options.commitmsgfile,
2627                         encoding=_encodings['fs'], errors='strict'),
2628                         mode='r', encoding=_encodings['content'], errors='replace')
2629                         commitmessage = f.read()
2630                         f.close()
2631                         del f
2632                 except (IOError, OSError) as e:
2633                         if e.errno == errno.ENOENT:
2634                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2635                         else:
2636                                 raise
2637                 # We've read the content so the file is no longer needed.
2638                 commitmessagefile = None
2639         if not commitmessage or not commitmessage.strip():
2640                 try:
2641                         editor = os.environ.get("EDITOR")
2642                         if editor and utilities.editor_is_executable(editor):
2643                                 commitmessage = utilities.get_commit_message_with_editor(
2644                                         editor, message=qa_output)
2645                         else:
2646                                 commitmessage = utilities.get_commit_message_with_stdin()
2647                 except KeyboardInterrupt:
2648                         exithandler()
2649                 if not commitmessage or not commitmessage.strip():
2650                         print("* no commit message?  aborting commit.")
2651                         sys.exit(1)
2652         commitmessage = commitmessage.rstrip()
2653         changelog_msg = commitmessage
2654         portage_version = getattr(portage, "VERSION", None)
2655         if portage_version is None:
2656                 sys.stderr.write("Failed to insert portage version in message!\n")
2657                 sys.stderr.flush()
2658                 portage_version = "Unknown"
2659
2660         if vcs == "git":
2661                 # Use new footer only for git (see bug #438364).
2662                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2663                 if options.force:
2664                         commit_footer += "\nRepoMan-Options: --force"
2665                 if sign_manifests:
2666                         commit_footer += "\nManifest-Sign-Key: %s" % \
2667                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2668         else:
2669                 unameout = platform.system() + " "
2670                 if platform.system() in ["Darwin", "SunOS"]:
2671                         unameout += platform.processor()
2672                 else:
2673                         unameout += platform.machine()
2674                 commit_footer = "\n\n(Portage version: %s/%s/%s" % \
2675                         (portage_version, vcs, unameout)
2676                 if options.force:
2677                         commit_footer += ", RepoMan options: --force"
2678                 if sign_manifests:
2679                         commit_footer += ", signed Manifest commit with key %s" % \
2680                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2681                 else:
2682                         commit_footer += ", unsigned Manifest commit"
2683                 commit_footer += ")"
2684
2685         commitmessage += commit_footer
2686
2687         if options.echangelog in ('y', 'force'):
2688                 logging.info("checking for unmodified ChangeLog files")
2689                 committer_name = utilities.get_committer_name(env=repoman_settings)
2690                 for x in sorted(vcs_files_to_cps(
2691                         chain(myupdates, mymanifests, myremoved))):
2692                         catdir, pkgdir = x.split("/")
2693                         checkdir = repodir + "/" + x
2694                         checkdir_relative = ""
2695                         if repolevel < 3:
2696                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2697                         if repolevel < 2:
2698                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2699                         checkdir_relative = os.path.join(".", checkdir_relative)
2700
2701                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2702                         changelog_modified = changelog_path in modified_changelogs
2703                         if changelog_modified and options.echangelog != 'force':
2704                                 continue
2705
2706                         # get changes for this package
2707                         cdrlen = len(checkdir_relative)
2708                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2709                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2710                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2711
2712                         # Skip ChangeLog generation if only the Manifest was modified,
2713                         # as discussed in bug #398009.
2714                         nontrivial_cl_files = set()
2715                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2716                         nontrivial_cl_files.difference_update(['Manifest'])
2717                         if not nontrivial_cl_files and options.echangelog != 'force':
2718                                 continue
2719
2720                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2721                                 committer_name, changelog_msg,
2722                                 os.path.join(repodir, 'skel.ChangeLog'),
2723                                 catdir, pkgdir,
2724                                 new=clnew, removed=clremoved, changed=clchanged,
2725                                 pretend=options.pretend)
2726                         if new_changelog is None:
2727                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2728                                         level=logging.ERROR, noiselevel=-1)
2729                                 sys.exit(1)
2730
2731                         # if the ChangeLog was just created, add it to vcs
2732                         if new_changelog:
2733                                 myautoadd.append(changelog_path)
2734                                 # myautoadd is appended to myupdates below
2735                         else:
2736                                 myupdates.append(changelog_path)
2737
2738                         if options.ask and not options.pretend:
2739                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2740                                 repoman_settings["O"] = checkdir
2741                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2742
2743         if myautoadd:
2744                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2745                 add_cmd = [vcs, "add"]
2746                 add_cmd += myautoadd
2747                 if options.pretend:
2748                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2749                                 noiselevel=-1)
2750                 else:
2751                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2752                                 # Python 3.1 produces the following TypeError if raw bytes are
2753                                 # passed to subprocess.call():
2754                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2755                                 #     errread, errwrite)
2756                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2757                                 #     raise child_exception
2758                                 # TypeError: expected an object with the buffer interface
2759                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2760                         retcode = subprocess.call(add_cmd)
2761                         if retcode != os.EX_OK:
2762                                 logging.error(
2763                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2764                                 sys.exit(retcode)
2765
2766                 myupdates += myautoadd
2767
2768         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2769
2770         if vcs not in ('cvs', 'svn'):
2771                 # With git, bzr and hg, there's never any keyword expansion, so
2772                 # there's no need to regenerate manifests and all files will be
2773                 # committed in one big commit at the end.
2774                 print()
2775         elif not repo_config.thin_manifest:
2776                 if vcs == 'cvs':
2777                         headerstring = "'\$(Header|Id).*\$'"
2778                 elif vcs == "svn":
2779                         svn_keywords = dict((k.lower(), k) for k in [
2780                                         "Rev",
2781                                         "Revision",
2782                                         "LastChangedRevision",
2783                                         "Date",
2784                                         "LastChangedDate",
2785                                         "Author",
2786                                         "LastChangedBy",
2787                                         "URL",
2788                                         "HeadURL",
2789                                         "Id",
2790                                         "Header",
2791                         ])
2792
2793                 for myfile in myupdates:
2794
2795                         # for CVS, no_expansion contains files that are excluded from expansion
2796                         if vcs == "cvs":
2797                                 if myfile in no_expansion:
2798                                         continue
2799
2800                         # for SVN, expansion contains files that are included in expansion
2801                         elif vcs == "svn":
2802                                 if myfile not in expansion:
2803                                         continue
2804
2805                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2806                                 enabled_keywords = []
2807                                 for k in expansion[myfile]:
2808                                         keyword = svn_keywords.get(k.lower())
2809                                         if keyword is not None:
2810                                                 enabled_keywords.append(keyword)
2811
2812                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2813
2814                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2815                                 portage._shell_quote(myfile))
2816                         if myout[0] == 0:
2817                                 myheaders.append(myfile)
2818
2819                 print("%s have headers that will change." % green(str(len(myheaders))))
2820                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2821
2822         logging.info("myupdates: %s", myupdates)
2823         logging.info("myheaders: %s", myheaders)
2824
2825         if options.ask and userquery('Commit changes?', True) != 'Yes':
2826                 print("* aborting commit.")
2827                 sys.exit(128 + signal.SIGINT)
2828
2829         # Handle the case where committed files have keywords which
2830         # will change and need a priming commit before the Manifest
2831         # can be committed.
2832         if (myupdates or myremoved) and myheaders:
2833                 myfiles = myupdates + myremoved
2834                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2835                 mymsg = os.fdopen(fd, "wb")
2836                 mymsg.write(_unicode_encode(commitmessage))
2837                 mymsg.close()
2838
2839                 print()
2840                 print(green("Using commit message:"))
2841                 print(green("------------------------------------------------------------------------------"))
2842                 print(commitmessage)
2843                 print(green("------------------------------------------------------------------------------"))
2844                 print()
2845
2846                 # Having a leading ./ prefix on file paths can trigger a bug in
2847                 # the cvs server when committing files to multiple directories,
2848                 # so strip the prefix.
2849                 myfiles = [f.lstrip("./") for f in myfiles]
2850
2851                 commit_cmd = [vcs]
2852                 commit_cmd.extend(vcs_global_opts)
2853                 commit_cmd.append("commit")
2854                 commit_cmd.extend(vcs_local_opts)
2855                 commit_cmd.extend(["-F", commitmessagefile])
2856                 commit_cmd.extend(myfiles)
2857
2858                 try:
2859                         if options.pretend:
2860                                 print("(%s)" % (" ".join(commit_cmd),))
2861                         else:
2862                                 retval = spawn(commit_cmd, env=commit_env)
2863                                 if retval != os.EX_OK:
2864                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2865                                                 "error code: %s\n") % (vcs, retval),
2866                                                 level=logging.ERROR, noiselevel=-1)
2867                                         sys.exit(retval)
2868                 finally:
2869                         try:
2870                                 os.unlink(commitmessagefile)
2871                         except OSError:
2872                                 pass
2873
2874         # Setup the GPG commands
2875         def gpgsign(filename):
2876                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2877                 if gpgcmd is None:
2878                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2879                                 " Is make.globals missing?")
2880                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2881                         "PORTAGE_GPG_KEY" not in repoman_settings:
2882                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2883                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2884                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2885                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2886                                         os.path.expanduser("~/.gnupg")
2887                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2888                                         % repoman_settings["PORTAGE_GPG_DIR"])
2889                         else:
2890                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2891                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2892                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2893                                 raise portage.exception.InvalidLocation(
2894                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2895                                         repoman_settings["PORTAGE_GPG_DIR"])
2896                 gpgvars = {"FILE": filename}
2897                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2898                         v = repoman_settings.get(k)
2899                         if v is not None:
2900                                 gpgvars[k] = v
2901                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2902                 if options.pretend:
2903                         print("(" + gpgcmd + ")")
2904                 else:
2905                         # Encode unicode manually for bug #310789.
2906                         gpgcmd = portage.util.shlex_split(gpgcmd)
2907                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2908                                 # Python 3.1 does not support bytes in Popen args.
2909                                 gpgcmd = [_unicode_encode(arg,
2910                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2911                         rValue = subprocess.call(gpgcmd)
2912                         if rValue == os.EX_OK:
2913                                 os.rename(filename + ".asc", filename)
2914                         else:
2915                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2916
2917         def need_signature(filename):
2918                 try:
2919                         with open(_unicode_encode(filename,
2920                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2921                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2922                 except IOError as e:
2923                         if e.errno in (errno.ENOENT, errno.ESTALE):
2924                                 return False
2925                         raise
2926
2927         # When files are removed and re-added, the cvs server will put /Attic/
2928         # inside the $Header path. This code detects the problem and corrects it
2929         # so that the Manifest will generate correctly. See bug #169500.
2930         # Use binary mode in order to avoid potential character encoding issues.
2931         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2932         attic_str = b'/Attic/'
2933         attic_replace = b'/'
2934         for x in myheaders:
2935                 f = open(_unicode_encode(x,
2936                         encoding=_encodings['fs'], errors='strict'),
2937                         mode='rb')
2938                 mylines = f.readlines()
2939                 f.close()
2940                 modified = False
2941                 for i, line in enumerate(mylines):
2942                         if cvs_header_re.match(line) is not None and \
2943                                 attic_str in line:
2944                                 mylines[i] = line.replace(attic_str, attic_replace)
2945                                 modified = True
2946                 if modified:
2947                         portage.util.write_atomic(x, b''.join(mylines),
2948                                 mode='wb')
2949
2950         if repolevel == 1:
2951                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2952                         "doing the entire repository.\"\n")
2953
2954         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2955
2956                 for x in sorted(vcs_files_to_cps(
2957                         chain(myupdates, myremoved, mymanifests))):
2958                         repoman_settings["O"] = os.path.join(repodir, x)
2959                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2960
2961         signed = False
2962         if sign_manifests:
2963                 signed = True
2964                 try:
2965                         for x in sorted(vcs_files_to_cps(
2966                                 chain(myupdates, myremoved, mymanifests))):
2967                                 repoman_settings["O"] = os.path.join(repodir, x)
2968                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2969                                 if not need_signature(manifest_path):
2970                                         continue
2971                                 gpgsign(manifest_path)
2972                 except portage.exception.PortageException as e:
2973                         portage.writemsg("!!! %s\n" % str(e))
2974                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2975                         signed = False
2976
2977         if vcs == 'git':
2978                 # It's not safe to use the git commit -a option since there might
2979                 # be some modified files elsewhere in the working tree that the
2980                 # user doesn't want to commit. Therefore, call git update-index
2981                 # in order to ensure that the index is updated with the latest
2982                 # versions of all new and modified files in the relevant portion
2983                 # of the working tree.
2984                 myfiles = mymanifests + myupdates
2985                 myfiles.sort()
2986                 update_index_cmd = ["git", "update-index"]
2987                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2988                 if options.pretend:
2989                         print("(%s)" % (" ".join(update_index_cmd),))
2990                 else:
2991                         retval = spawn(update_index_cmd, env=os.environ)
2992                         if retval != os.EX_OK:
2993                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2994                                         "error code: %s\n") % (vcs, retval),
2995                                         level=logging.ERROR, noiselevel=-1)
2996                                 sys.exit(retval)
2997
2998         if True:
2999                 myfiles = mymanifests[:]
3000                 # If there are no header (SVN/CVS keywords) changes in
3001                 # the files, this Manifest commit must include the
3002                 # other (yet uncommitted) files.
3003                 if not myheaders:
3004                         myfiles += myupdates
3005                         myfiles += myremoved
3006                 myfiles.sort()
3007
3008                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
3009                 mymsg = os.fdopen(fd, "wb")
3010                 mymsg.write(_unicode_encode(commitmessage))
3011                 mymsg.close()
3012
3013                 commit_cmd = []
3014                 if options.pretend and vcs is None:
3015                         # substitute a bogus value for pretend output
3016                         commit_cmd.append("cvs")
3017                 else:
3018                         commit_cmd.append(vcs)
3019                 commit_cmd.extend(vcs_global_opts)
3020                 commit_cmd.append("commit")
3021                 commit_cmd.extend(vcs_local_opts)
3022                 if vcs == "hg":
3023                         commit_cmd.extend(["--logfile", commitmessagefile])
3024                         commit_cmd.extend(myfiles)
3025                 else:
3026                         commit_cmd.extend(["-F", commitmessagefile])
3027                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
3028
3029                 try:
3030                         if options.pretend:
3031                                 print("(%s)" % (" ".join(commit_cmd),))
3032                         else:
3033                                 retval = spawn(commit_cmd, env=commit_env)
3034                                 if retval != os.EX_OK:
3035                                         if repo_config.sign_commit and vcs == 'git' and \
3036                                                 not git_supports_gpg_sign():
3037                                                 # Inform user that newer git is needed (bug #403323).
3038                                                 logging.error(
3039                                                         "Git >=1.7.9 is required for signed commits!")
3040
3041                                         writemsg_level(("!!! Exiting on %s (shell) " + \
3042                                                 "error code: %s\n") % (vcs, retval),
3043                                                 level=logging.ERROR, noiselevel=-1)
3044                                         sys.exit(retval)
3045                 finally:
3046                         try:
3047                                 os.unlink(commitmessagefile)
3048                         except OSError:
3049                                 pass
3050
3051         print()
3052         if vcs:
3053                 print("Commit complete.")
3054         else:
3055                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
3056         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
3057 sys.exit(0)