bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2012 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import io
  17 import logging
  18 import optparse
  19 import re
  20 import signal
  21 import stat
  22 import subprocess
  23 import sys
  24 import tempfile
  25 import textwrap
  26 import time
  27 import platform
  28
  29 try:
  30         from urllib.request import urlopen as urllib_request_urlopen
  31 except ImportError:
  32         from urllib import urlopen as urllib_request_urlopen
  33
  34 from itertools import chain
  35 from stat import S_ISDIR
  36
  37 from os import path as osp
  38 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  39 sys.path.insert(0, pym_path)
  40 import portage
  41 portage._disable_legacy_globals()
  42 portage.dep._internal_warnings = True
  43
  44 try:
  45         import xml.etree.ElementTree
  46         from xml.parsers.expat import ExpatError
  47 except (SystemExit, KeyboardInterrupt):
  48         raise
  49 except (ImportError, SystemError, RuntimeError, Exception):
  50         # broken or missing xml support
  51         # http://bugs.python.org/issue14988
  52         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  53         from portage.output import EOutput
  54         out = EOutput()
  55         for line in msg:
  56                 out.eerror(line)
  57         sys.exit(1)
  58
  59 from portage import os
  60 from portage import _encodings
  61 from portage import _unicode_encode
  62 from repoman.checks import run_checks
  63 from repoman import utilities
  64 from repoman.herdbase import make_herd_base
  65 from _emerge.Package import Package
  66 from _emerge.RootConfig import RootConfig
  67 from _emerge.userquery import userquery
  68 import portage.checksum
  69 import portage.const
  70 from portage import cvstree, normalize_path
  71 from portage import util
  72 from portage.exception import (FileNotFound, MissingParameter,
  73         ParseError, PermissionDenied)
  74 from portage.process import find_binary, spawn
  75 from portage.output import bold, create_color_func, \
  76         green, nocolor, red
  77 from portage.output import ConsoleStyleFile, StyleWriter
  78 from portage.util import writemsg_level
  79 from portage.package.ebuild.digestgen import digestgen
  80 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  81
  82 if sys.hexversion >= 0x3000000:
  83         basestring = str
  84
  85 util.initialize_logger()
  86
  87 # 14 is the length of DESCRIPTION=""
  88 max_desc_len = 100
  89 allowed_filename_chars="a-zA-Z0-9._-+:"
  90 pv_toolong_re = re.compile(r'[0-9]{19,}')
  91 bad = create_color_func("BAD")
  92
  93 # A sane umask is needed for files that portage creates.
  94 os.umask(0o22)
  95 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  96 # behave incrementally.
  97 repoman_incrementals = tuple(x for x in \
  98         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
  99 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 100 repoman_settings = portage.config(config_root=config_root, local_config=False)
 101
 102 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 103         repoman_settings.get('TERM') == 'dumb' or \
 104         not sys.stdout.isatty():
 105         nocolor()
 106
 107 def warn(txt):
 108         print("repoman: " + txt)
 109
 110 def err(txt):
 111         warn(txt)
 112         sys.exit(1)
 113
 114 def exithandler(signum=None, frame=None):
 115         logging.fatal("Interrupted; exiting...")
 116         if signum is None:
 117                 sys.exit(1)
 118         else:
 119                 sys.exit(128 + signum)
 120
 121 signal.signal(signal.SIGINT,exithandler)
 122
 123 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 124         """Repoman needs it's own HelpFormatter for now, because the default ones
 125         murder the help text."""
 126
 127         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 128                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 129
 130         def format_description(self, description):
 131                 return description
 132
 133 class RepomanOptionParser(optparse.OptionParser):
 134         """Add the on_tail function, ruby has it, optionParser should too
 135         """
 136
 137         def __init__(self, *args, **kwargs):
 138                 optparse.OptionParser.__init__(self, *args, **kwargs)
 139                 self.tail = ""
 140
 141         def on_tail(self, description):
 142                 self.tail += description
 143
 144         def format_help(self, formatter=None):
 145                 result = optparse.OptionParser.format_help(self, formatter)
 146                 result += self.tail
 147                 return result
 148
 149
 150 def ParseArgs(argv, qahelp):
 151         """This function uses a customized optionParser to parse command line arguments for repoman
 152         Args:
 153           argv - a sequence of command line arguments
 154                 qahelp - a dict of qa warning to help message
 155         Returns:
 156           (opts, args), just like a call to parser.parse_args()
 157         """
 158
 159         if argv and isinstance(argv[0], bytes):
 160                 argv = [portage._unicode_decode(x) for x in argv]
 161
 162         modes = {
 163                 'commit' : 'Run a scan then commit changes',
 164                 'ci' : 'Run a scan then commit changes',
 165                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 166                 'full' : 'Scan directory tree and print all issues (not a summary)',
 167                 'help' : 'Show this screen',
 168                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 169                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 170                 'scan' : 'Scan directory tree for QA issues'
 171         }
 172
 173         mode_keys = list(modes)
 174         mode_keys.sort()
 175
 176         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 177         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 178         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 179         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 180         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 181
 182         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 183                 help='Request a confirmation before commiting')
 184
 185         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 186                 help='specify a commit message on the command line')
 187
 188         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 189                 help='specify a path to a file that contains a commit message')
 190
 191         parser.add_option('--digest',
 192                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 193                 help='Automatically update Manifest digests for modified files')
 194
 195         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 196                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 197
 198         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 199                 help='do not print unnecessary messages')
 200
 201         parser.add_option(
 202                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 203                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 204                 'regardless of modification if \'force\' is specified)')
 205
 206         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 207                 help='Commit with QA violations')
 208
 209         parser.add_option('--vcs', dest='vcs',
 210                 help='Force using specific VCS instead of autodetection')
 211
 212         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 213                 help='be very verbose in output', default=0)
 214
 215         parser.add_option('-V', '--version', dest='version', action='store_true',
 216                 help='show version info')
 217
 218         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 219                 default=False, help='forces the metadata.xml parse check to be carried out')
 220
 221         parser.add_option(
 222                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 223                 metavar="<y|n>",
 224                 help='only check packages that have uncommitted modifications')
 225
 226         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 227                 default=False, help='ignore arch-specific failures (where arch != host)')
 228
 229         parser.add_option("--ignore-default-opts",
 230                 action="store_true",
 231                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 232
 233         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 234                 default=False, help='ignore masked packages (not allowed with commit mode)')
 235
 236         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 237                 default=False, help='include dev profiles in dependency checks')
 238
 239         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 240                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 241
 242         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 243                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 244
 245         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 246                 help='specify which mode repoman will run in (default=full)')
 247
 248         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 249
 250         for k in mode_keys:
 251                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 252
 253         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 254
 255         sorted_qa = list(qahelp)
 256         sorted_qa.sort()
 257         for k in sorted_qa:
 258                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 259
 260         opts, args = parser.parse_args(argv[1:])
 261
 262         if not opts.ignore_default_opts:
 263                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 264                 if default_opts:
 265                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 266
 267         if opts.mode == 'help':
 268                 parser.print_help(short=False)
 269
 270         for arg in args:
 271                 if arg in modes:
 272                         if not opts.mode:
 273                                 opts.mode = arg
 274                                 break
 275                 else:
 276                         parser.error("invalid mode: %s" % arg)
 277
 278         if not opts.mode:
 279                 opts.mode = 'full'
 280
 281         if opts.mode == 'ci':
 282                 opts.mode = 'commit'  # backwards compat shortcut
 283
 284         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 285                 if opts.ignore_masked:
 286                         parser.error('Commit mode and --ignore-masked are not compatible')
 287                 if opts.without_mask:
 288                         parser.error('Commit mode and --without-mask are not compatible')
 289
 290         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 291         for val in range(opts.verbosity):
 292                 logger = logging.getLogger()
 293                 logger.setLevel(logger.getEffectiveLevel() - 10)
 294
 295         for val in range(opts.quiet):
 296                 logger = logging.getLogger()
 297                 logger.setLevel(logger.getEffectiveLevel() + 10)
 298
 299         return (opts, args)
 300
 301 qahelp={
 302         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 303         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 304         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 305         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 306         "changelog.missing":"Missing ChangeLog files",
 307         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 308         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 309         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 310         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 311         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 312         "file.size":"Files in the files directory must be under 20 KiB",
 313         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 314         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 315         "file.UTF8":"File is not UTF8 compliant",
 316         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 317         "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it",
 318         "inherit.unused":"Ebuild inherits an eclass but does not use it",
 319         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 320         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 321         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 322         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 323         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 324         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 325         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 326         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 327         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 328         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 329         "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 330         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 331         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 332         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 333         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 334         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 335         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 336         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 337         "HDEPEND.bad":"User-visible ebuilds with bad HDEPEND settings (matched against *visible* ebuilds)",
 338         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 339         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 340         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 341         "HDEPEND.badmasked":"Masked ebuilds with bad HDEPEND settings (matched against *all* ebuilds)",
 342         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 343         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 344         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 345         "HDEPEND.badindev":"User-visible ebuilds with bad HDEPEND settings (matched against *visible* ebuilds) in developing arch",
 346         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 347         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 348         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 349         "HDEPEND.badmaskedindev":"Masked ebuilds with bad HDEPEND settings (matched against *all* ebuilds) in developing arch",
 350         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 351         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 352         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 353         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 354         "HDEPEND.syntax":"Syntax error in HDEPEND (usually an extra/missing space/parenthesis)",
 355         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 356         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 357         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 358         "HDEPEND.badtilde":"HDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 359         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 360         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 361         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 362         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 363         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 364         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 365         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 366         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 367         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 368         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 369         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 370         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 371         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 372         "variable.readonly":"Assigning a readonly variable",
 373         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 374         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 375         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 376         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 377         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 378         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 379         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 380         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 381         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 382         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 383         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 384         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 385         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 386         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 387         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 388         "ebuild.badheader":"This ebuild has a malformed header",
 389         "manifest.bad":"Manifest has missing or incorrect digests",
 390         "metadata.missing":"Missing metadata.xml files",
 391         "metadata.bad":"Bad metadata.xml files",
 392         "metadata.warning":"Warnings in metadata.xml files",
 393         "portage.internal":"The ebuild uses an internal Portage function",
 394         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 395         "virtual.suspect":"Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 396         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 397         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 398 }
 399
 400 qacats = list(qahelp)
 401 qacats.sort()
 402
 403 qawarnings = set((
 404 "changelog.missing",
 405 "changelog.notadded",
 406 "dependency.unknown",
 407 "digest.assumed",
 408 "digest.unused",
 409 "ebuild.notadded",
 410 "ebuild.nesteddie",
 411 "DEPEND.badmasked", "HDEPEND.badmasked", "RDEPEND.badmasked", "PDEPEND.badmasked",
 412 "DEPEND.badindev", "HDEPEND.badindev", "RDEPEND.badindev", "PDEPEND.badindev",
 413 "DEPEND.badmaskedindev", "HDEPEND.badmaskedindev", "RDEPEND.badmaskedindev", "PDEPEND.badmaskedindev",
 414 "DEPEND.badtilde", "HDEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 415 "DESCRIPTION.toolong",
 416 "EAPI.deprecated",
 417 "HOMEPAGE.virtual",
 418 "LICENSE.virtual",
 419 "KEYWORDS.dropped",
 420 "KEYWORDS.stupid",
 421 "KEYWORDS.missing",
 422 "PDEPEND.suspect",
 423 "RDEPEND.implicit",
 424 "RDEPEND.suspect",
 425 "virtual.suspect",
 426 "RESTRICT.invalid",
 427 "ebuild.minorsyn",
 428 "ebuild.badheader",
 429 "ebuild.patches",
 430 "file.size",
 431 "inherit.unused",
 432 "inherit.deprecated",
 433 "java.eclassesnotused",
 434 "wxwidgets.eclassnotused",
 435 "metadata.warning",
 436 "portage.internal",
 437 "usage.obsolete",
 438 "upstream.workaround",
 439 "LIVEVCS.stable",
 440 "LIVEVCS.unmasked",
 441 ))
 442
 443 if portage.const._ENABLE_INHERIT_CHECK:
 444         # This is experimental, so it's non-fatal.
 445         qawarnings.add("inherit.missing")
 446
 447 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 448
 449 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 450 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 451 allvars.update(Package.metadata_keys)
 452 allvars = sorted(allvars)
 453 commitmessage=None
 454 for x in missingvars:
 455         x += ".missing"
 456         if x not in qacats:
 457                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 458                 qacats.append(x)
 459                 qawarnings.add(x)
 460
 461 valid_restrict = frozenset(["binchecks", "bindist",
 462         "fetch", "installsources", "mirror",
 463         "primaryuri", "strip", "test", "userpriv"])
 464
 465 live_eclasses = frozenset([
 466         "bzr",
 467         "cvs",
 468         "darcs",
 469         "git",
 470         "git-2",
 471         "mercurial",
 472         "subversion",
 473         "tla",
 474 ])
 475
 476 suspect_rdepend = frozenset([
 477         "app-arch/cabextract",
 478         "app-arch/rpm2targz",
 479         "app-doc/doxygen",
 480         "dev-lang/nasm",
 481         "dev-lang/swig",
 482         "dev-lang/yasm",
 483         "dev-perl/extutils-pkgconfig",
 484         "dev-util/byacc",
 485         "dev-util/cmake",
 486         "dev-util/ftjam",
 487         "dev-util/gperf",
 488         "dev-util/gtk-doc",
 489         "dev-util/gtk-doc-am",
 490         "dev-util/intltool",
 491         "dev-util/jam",
 492         "dev-util/pkg-config-lite",
 493         "dev-util/pkgconf",
 494         "dev-util/pkgconfig",
 495         "dev-util/pkgconfig-openbsd",
 496         "dev-util/scons",
 497         "dev-util/unifdef",
 498         "dev-util/yacc",
 499         "media-gfx/ebdftopcf",
 500         "sys-apps/help2man",
 501         "sys-devel/autoconf",
 502         "sys-devel/automake",
 503         "sys-devel/bin86",
 504         "sys-devel/bison",
 505         "sys-devel/dev86",
 506         "sys-devel/flex",
 507         "sys-devel/m4",
 508         "sys-devel/pmake",
 509         "virtual/linux-sources",
 510         "virtual/pkgconfig",
 511         "x11-misc/bdftopcf",
 512         "x11-misc/imake",
 513 ])
 514
 515 suspect_virtual = {
 516         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 517         "dev-util/pkgconf":"virtual/pkgconfig",
 518         "dev-util/pkgconfig":"virtual/pkgconfig",
 519         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 520         "dev-libs/libusb":"virtual/libusb",
 521         "dev-libs/libusbx":"virtual/libusb",
 522         "dev-libs/libusb-compat":"virtual/libusb",
 523 }
 524
 525 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 526 # force refetch if the local copy creation time is older than this
 527 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 528
 529 # file.executable
 530 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 531
 532 options, arguments = ParseArgs(sys.argv, qahelp)
 533
 534 if options.version:
 535         print("Portage", portage.VERSION)
 536         sys.exit(0)
 537
 538 # Set this to False when an extraordinary issue (generally
 539 # something other than a QA issue) makes it impossible to
 540 # commit (like if Manifest generation fails).
 541 can_force = True
 542
 543 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 544 if portdir is None:
 545         sys.exit(1)
 546
 547 myreporoot = os.path.basename(portdir_overlay)
 548 myreporoot += mydir[len(portdir_overlay):]
 549
 550 if options.vcs:
 551         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 552                 vcs = options.vcs
 553         else:
 554                 vcs = None
 555 else:
 556         vcses = utilities.FindVCS()
 557         if len(vcses) > 1:
 558                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 559                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 560                 sys.exit(1)
 561         elif vcses:
 562                 vcs = vcses[0]
 563         else:
 564                 vcs = None
 565
 566 if options.if_modified == "y" and vcs is None:
 567         logging.info("Not in a version controlled repository; "
 568                 "disabling --if-modified.")
 569         options.if_modified = "n"
 570
 571 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 572 vcs_preserves_mtime = vcs in ('cvs',)
 573
 574 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 575 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 576 if vcs_global_opts is None:
 577         if vcs in ('cvs', 'svn'):
 578                 vcs_global_opts = "-q"
 579         else:
 580                 vcs_global_opts = ""
 581 vcs_global_opts = vcs_global_opts.split()
 582
 583 if options.mode == 'commit' and not options.pretend and not vcs:
 584         logging.info("Not in a version controlled repository; enabling pretend mode.")
 585         options.pretend = True
 586
 587 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 588 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 589         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 590         portage._shell_quote(portdir_overlay))
 591 # We have to call the config constructor again so
 592 # that config.repositories is initialized correctly.
 593 repoman_settings = portage.config(config_root=config_root, local_config=False,
 594         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 595
 596 root = repoman_settings['EROOT']
 597 trees = {
 598         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 599 }
 600 portdb = trees[root]['porttree'].dbapi
 601
 602 # Constrain dependency resolution to the master(s)
 603 # that are specified in layout.conf.
 604 repodir = os.path.realpath(portdir_overlay)
 605 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 606 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 607 portdir = portdb.porttrees[0]
 608
 609 if repo_config.allow_provide_virtual:
 610         qawarnings.add("virtual.oldstyle")
 611
 612 if repo_config.sign_commit:
 613         if vcs == 'git':
 614                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 615                 # the commit arguments. If key_id is unspecified, then it must be
 616                 # configured by `git config user.signingkey key_id`.
 617                 vcs_local_opts.append("--gpg-sign")
 618
 619 # In order to disable manifest signatures, repos may set
 620 # "sign-manifests = false" in metadata/layout.conf. This
 621 # can be used to prevent merge conflicts like those that
 622 # thin-manifests is designed to prevent.
 623 sign_manifests = "sign" in repoman_settings.features and \
 624         repo_config.sign_manifest
 625
 626 manifest_hashes = repo_config.manifest_hashes
 627 if manifest_hashes is None:
 628         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 629
 630 if options.mode in ("commit", "fix", "manifest"):
 631         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 632                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 633                 "metadata/layout.conf does not contain the '%s' hash which "
 634                 "is required by this portage version. You will have to "
 635                 "upgrade portage if you want to generate valid manifests for "
 636                 "this repository.") % \
 637                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 638                 for line in textwrap.wrap(msg, 70):
 639                         logging.error(line)
 640                 sys.exit(1)
 641
 642         unsupported_hashes = manifest_hashes.difference(
 643                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 644         if unsupported_hashes:
 645                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 646                 "metadata/layout.conf contains one or more hash types '%s' "
 647                 "which are not supported by this portage version. You will "
 648                 "have to upgrade portage if you want to generate valid "
 649                 "manifests for this repository.") % \
 650                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 651                 for line in textwrap.wrap(msg, 70):
 652                         logging.error(line)
 653                 sys.exit(1)
 654
 655 if "commit" == options.mode and \
 656         repo_config.name == "gentoo" and \
 657         "RMD160" in manifest_hashes and \
 658         "RMD160" not in portage.checksum.hashorigin_map:
 659         msg = "Please install " \
 660         "pycrypto or enable python's ssl USE flag in order " \
 661         "to enable RMD160 hash support. See bug #198398 for " \
 662         "more information."
 663         prefix = bad(" * ")
 664         for line in textwrap.wrap(msg, 70):
 665                 print(prefix + line)
 666         sys.exit(1)
 667
 668 if options.echangelog is None and repo_config.update_changelog:
 669         options.echangelog = 'y'
 670
 671 if vcs is None:
 672         options.echangelog = 'n'
 673
 674 # The --echangelog option causes automatic ChangeLog generation,
 675 # which invalidates changelog.ebuildadded and changelog.missing
 676 # checks.
 677 # Note: Some don't use ChangeLogs in distributed SCMs.
 678 # It will be generated on server side from scm log,
 679 # before package moves to the rsync server.
 680 # This is needed because they try to avoid merge collisions.
 681 # Gentoo's Council decided to always use the ChangeLog file.
 682 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 683 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 684
 685 if 'digest' in repoman_settings.features and options.digest != 'n':
 686         options.digest = 'y'
 687
 688 logging.debug("vcs: %s" % (vcs,))
 689 logging.debug("repo config: %s" % (repo_config,))
 690 logging.debug("options: %s" % (options,))
 691
 692 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 693 # profile-specific config constructor calls.
 694 env = os.environ.copy()
 695 env['PORTDIR'] = portdir
 696 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 697
 698 logging.info('Setting paths:')
 699 logging.info('PORTDIR = "' + portdir + '"')
 700 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 701
 702 # It's confusing if these warnings are displayed without the user
 703 # being told which profile they come from, so disable them.
 704 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 705
 706 categories = []
 707 for path in repo_config.eclass_db.porttrees:
 708         categories.extend(portage.util.grabfile(
 709                 os.path.join(path, 'profiles', 'categories')))
 710 repoman_settings.categories = frozenset(
 711         portage.util.stack_lists([categories], incremental=1))
 712 categories = repoman_settings.categories
 713
 714 portdb.settings = repoman_settings
 715 root_config = RootConfig(repoman_settings, trees[root], None)
 716 # We really only need to cache the metadata that's necessary for visibility
 717 # filtering. Anything else can be discarded to reduce memory consumption.
 718 portdb._aux_cache_keys.clear()
 719 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 720
 721 reposplit = myreporoot.split(os.path.sep)
 722 repolevel = len(reposplit)
 723
 724 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 725 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 726 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 727 if options.mode == 'commit' and repolevel not in [1,2,3]:
 728         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 729         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 730         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 731         print(red("***"))
 732         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 733
 734 # Make startdir relative to the canonical repodir, so that we can pass
 735 # it to digestgen and it won't have to be canonicalized again.
 736 if repolevel == 1:
 737         startdir = repodir
 738 else:
 739         startdir = normalize_path(mydir)
 740         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 741
 742 def caterror(mycat):
 743         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 744
 745 def repoman_getstatusoutput(cmd):
 746         """
 747         Implements an interface similar to getstatusoutput(), but with
 748         customized unicode handling (see bug #310789) and without the shell.
 749         """
 750         args = portage.util.shlex_split(cmd)
 751         encoding = _encodings['fs']
 752         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 753                 # Python 3.1 does not support bytes in Popen args.
 754                 args = [_unicode_encode(x,
 755                         encoding=encoding, errors='strict') for x in args]
 756         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 757                 stderr=subprocess.STDOUT)
 758         output = portage._unicode_decode(proc.communicate()[0],
 759                 encoding=encoding, errors='strict')
 760         if output and output[-1] == "\n":
 761                 # getstatusoutput strips one newline
 762                 output = output[:-1]
 763         return (proc.wait(), output)
 764
 765 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 766         """
 767         Implements an interface similar to os.popen(), but with customized
 768         unicode handling (see bug #310789) and without the shell.
 769         """
 770
 771         __slots__ = ('_proc', '_stdout')
 772
 773         def __init__(self, cmd):
 774                 args = portage.util.shlex_split(cmd)
 775                 encoding = _encodings['fs']
 776                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 777                         # Python 3.1 does not support bytes in Popen args.
 778                         args = [_unicode_encode(x,
 779                                 encoding=encoding, errors='strict') for x in args]
 780                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 781                 object.__setattr__(self, '_proc', proc)
 782                 object.__setattr__(self, '_stdout',
 783                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 784
 785         def _get_target(self):
 786                 return object.__getattribute__(self, '_stdout')
 787
 788         __enter__ = _get_target
 789
 790         def __exit__(self, exc_type, exc_value, traceback):
 791                 proc = object.__getattribute__(self, '_proc')
 792                 proc.wait()
 793                 proc.stdout.close()
 794
 795 class ProfileDesc(object):
 796         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 797         def __init__(self, arch, status, sub_path, tree_path):
 798                 self.arch = arch
 799                 self.status = status
 800                 if sub_path:
 801                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 802                 self.sub_path = sub_path
 803                 self.tree_path = tree_path
 804                 if tree_path:
 805                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 806                 else:
 807                         self.abs_path = tree_path
 808
 809         def __str__(self):
 810                 if self.sub_path:
 811                         return self.sub_path
 812                 return 'empty profile'
 813
 814 profile_list = []
 815 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 816
 817 # get lists of valid keywords, licenses, and use
 818 kwlist = set()
 819 liclist = set()
 820 uselist = set()
 821 global_pmasklines = []
 822
 823 for path in portdb.porttrees:
 824         try:
 825                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 826         except OSError:
 827                 pass
 828         kwlist.update(portage.grabfile(os.path.join(path,
 829                 "profiles", "arch.list")))
 830
 831         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 832         for x in use_desc:
 833                 x = x.split()
 834                 if x:
 835                         uselist.add(x[0])
 836
 837         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 838         try:
 839                 expand_list = os.listdir(expand_desc_dir)
 840         except OSError:
 841                 pass
 842         else:
 843                 for fn in expand_list:
 844                         if not fn[-5:] == '.desc':
 845                                 continue
 846                         use_prefix = fn[:-5].lower() + '_'
 847                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 848                                 x = x.split()
 849                                 if x:
 850                                         uselist.add(use_prefix + x[0])
 851
 852         global_pmasklines.append(portage.util.grabfile_package(
 853                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 854
 855         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 856         try:
 857                 desc_file = io.open(_unicode_encode(desc_path,
 858                         encoding=_encodings['fs'], errors='strict'),
 859                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 860         except EnvironmentError:
 861                 pass
 862         else:
 863                 for i, x in enumerate(desc_file):
 864                         if x[0] == "#":
 865                                 continue
 866                         arch = x.split()
 867                         if len(arch) == 0:
 868                                 continue
 869                         if len(arch) != 3:
 870                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 871                                         desc_path + " line %d" % (i+1, ))
 872                         elif arch[0] not in kwlist:
 873                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 874                                         desc_path + " line %d" % (i+1, ))
 875                         elif arch[2] not in valid_profile_types:
 876                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 877                                         desc_path + " line %d" % (i+1, ))
 878                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 879                         if not os.path.isdir(profile_desc.abs_path):
 880                                 logging.error(
 881                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 882                                         arch[2], arch[1], arch[0], desc_path, i+1)
 883                                 continue
 884                         if os.path.exists(
 885                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 886                                 continue
 887                         profile_list.append(profile_desc)
 888                 desc_file.close()
 889
 890 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 891 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 892
 893 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 894 global_pmaskdict = {}
 895 for x in global_pmasklines:
 896         global_pmaskdict.setdefault(x.cp, []).append(x)
 897 del global_pmasklines
 898
 899 def has_global_mask(pkg):
 900         mask_atoms = global_pmaskdict.get(pkg.cp)
 901         if mask_atoms:
 902                 pkg_list = [pkg]
 903                 for x in mask_atoms:
 904                         if portage.dep.match_from_list(x, pkg_list):
 905                                 return x
 906         return None
 907
 908 # Ensure that profile sub_path attributes are unique. Process in reverse order
 909 # so that profiles with duplicate sub_path from overlays will override
 910 # profiles with the same sub_path from parent repos.
 911 profiles = {}
 912 profile_list.reverse()
 913 profile_sub_paths = set()
 914 for prof in profile_list:
 915         if prof.sub_path in profile_sub_paths:
 916                 continue
 917         profile_sub_paths.add(prof.sub_path)
 918         profiles.setdefault(prof.arch, []).append(prof)
 919
 920 # Use an empty profile for checking dependencies of
 921 # packages that have empty KEYWORDS.
 922 prof = ProfileDesc('**', 'stable', '', '')
 923 profiles.setdefault(prof.arch, []).append(prof)
 924
 925 for x in repoman_settings.archlist():
 926         if x[0] == "~":
 927                 continue
 928         if x not in profiles:
 929                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 930                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 931                 print(red("up with the "+x+" team."))
 932                 print()
 933
 934 if not liclist:
 935         logging.fatal("Couldn't find licenses?")
 936         sys.exit(1)
 937
 938 if not kwlist:
 939         logging.fatal("Couldn't read KEYWORDS from arch.list")
 940         sys.exit(1)
 941
 942 if not uselist:
 943         logging.fatal("Couldn't find use.desc?")
 944         sys.exit(1)
 945
 946 scanlist=[]
 947 if repolevel==2:
 948         #we are inside a category directory
 949         catdir=reposplit[-1]
 950         if catdir not in categories:
 951                 caterror(catdir)
 952         mydirlist=os.listdir(startdir)
 953         for x in mydirlist:
 954                 if x == "CVS" or x.startswith("."):
 955                         continue
 956                 if os.path.isdir(startdir+"/"+x):
 957                         scanlist.append(catdir+"/"+x)
 958         repo_subdir = catdir + os.sep
 959 elif repolevel==1:
 960         for x in categories:
 961                 if not os.path.isdir(startdir+"/"+x):
 962                         continue
 963                 for y in os.listdir(startdir+"/"+x):
 964                         if y == "CVS" or y.startswith("."):
 965                                 continue
 966                         if os.path.isdir(startdir+"/"+x+"/"+y):
 967                                 scanlist.append(x+"/"+y)
 968         repo_subdir = ""
 969 elif repolevel==3:
 970         catdir = reposplit[-2]
 971         if catdir not in categories:
 972                 caterror(catdir)
 973         scanlist.append(catdir+"/"+reposplit[-1])
 974         repo_subdir = scanlist[-1] + os.sep
 975 else:
 976         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 977                 ' from the current working directory'
 978         logging.critical(msg)
 979         sys.exit(1)
 980
 981 repo_subdir_len = len(repo_subdir)
 982 scanlist.sort()
 983
 984 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 985
 986 def vcs_files_to_cps(vcs_file_iter):
 987         """
 988         Iterate over the given modified file paths returned from the vcs,
 989         and return a frozenset containing category/pn strings for each
 990         modified package.
 991         """
 992
 993         modified_cps = []
 994
 995         if repolevel == 3:
 996                 if reposplit[-2] in categories and \
 997                         next(vcs_file_iter, None) is not None:
 998                         modified_cps.append("/".join(reposplit[-2:]))
 999
1000         elif repolevel == 2:
1001                 category = reposplit[-1]
1002                 if category in categories:
1003                         for filename in vcs_file_iter:
1004                                 f_split = filename.split(os.sep)
1005                                 # ['.', pn,...]
1006                                 if len(f_split) > 2:
1007                                         modified_cps.append(category + "/" + f_split[1])
1008
1009         else:
1010                 # repolevel == 1
1011                 for filename in vcs_file_iter:
1012                         f_split = filename.split(os.sep)
1013                         # ['.', category, pn,...]
1014                         if len(f_split) > 3 and f_split[1] in categories:
1015                                 modified_cps.append("/".join(f_split[1:3]))
1016
1017         return frozenset(modified_cps)
1018
1019 def git_supports_gpg_sign():
1020         status, cmd_output = \
1021                 repoman_getstatusoutput("git --version")
1022         cmd_output = cmd_output.split()
1023         if cmd_output:
1024                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1025                 if version is not None:
1026                         version = [int(x) for x in version.groups()[1:]]
1027                         if version[0] > 1 or \
1028                                 (version[0] == 1 and version[1] > 7) or \
1029                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1030                                 return True
1031         return False
1032
1033 def dev_keywords(profiles):
1034         """
1035         Create a set of KEYWORDS values that exist in 'dev'
1036         profiles. These are used
1037         to trigger a message notifying the user when they might
1038         want to add the --include-dev option.
1039         """
1040         type_arch_map = {}
1041         for arch, arch_profiles in profiles.items():
1042                 for prof in arch_profiles:
1043                         arch_set = type_arch_map.get(prof.status)
1044                         if arch_set is None:
1045                                 arch_set = set()
1046                                 type_arch_map[prof.status] = arch_set
1047                         arch_set.add(arch)
1048
1049         dev_keywords = type_arch_map.get('dev', set())
1050         dev_keywords.update(['~' + arch for arch in dev_keywords])
1051         return frozenset(dev_keywords)
1052
1053 dev_keywords = dev_keywords(profiles)
1054
1055 stats={}
1056 fails={}
1057
1058 for x in qacats:
1059         stats[x]=0
1060         fails[x]=[]
1061
1062 xmllint_capable = False
1063 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1064
1065 def parsedate(s):
1066         """Parse a RFC 822 date and time string.
1067         This is required for python3 compatibility, since the
1068         rfc822.parsedate() function is not available."""
1069
1070         s_split = []
1071         for x in s.upper().split():
1072                 for y in x.split(','):
1073                         if y:
1074                                 s_split.append(y)
1075
1076         if len(s_split) != 6:
1077                 return None
1078
1079         # %a, %d %b %Y %H:%M:%S %Z
1080         a, d, b, Y, H_M_S, Z = s_split
1081
1082         # Convert month to integer, since strptime %w is locale-dependent.
1083         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1084                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1085         m = month_map.get(b)
1086         if m is None:
1087                 return None
1088         m = str(m).rjust(2, '0')
1089
1090         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1091
1092 def fetch_metadata_dtd():
1093         """
1094         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1095         metadata_dtd_ctime_interval.
1096         @rtype: bool
1097         @return: True if successful, otherwise False
1098         """
1099
1100         must_fetch = True
1101         metadata_dtd_st = None
1102         current_time = int(time.time())
1103         try:
1104                 metadata_dtd_st = os.stat(metadata_dtd)
1105         except EnvironmentError as e:
1106                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1107                         raise
1108                 del e
1109         else:
1110                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1111                 if abs(current_time - metadata_dtd_st.st_ctime) \
1112                         < metadata_dtd_ctime_interval:
1113                         must_fetch = False
1114
1115         if must_fetch:
1116                 print()
1117                 print(green("***") + " the local copy of metadata.dtd " + \
1118                         "needs to be refetched, doing that now")
1119                 print()
1120                 try:
1121                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1122                         msg_info = url_f.info()
1123                         last_modified = msg_info.get('last-modified')
1124                         if last_modified is not None:
1125                                 last_modified = parsedate(last_modified)
1126                                 if last_modified is not None:
1127                                         last_modified = calendar.timegm(last_modified)
1128
1129                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1130                         try:
1131                                 local_f = open(metadata_dtd_tmp, mode='wb')
1132                                 local_f.write(url_f.read())
1133                                 local_f.close()
1134                                 if last_modified is not None:
1135                                         try:
1136                                                 os.utime(metadata_dtd_tmp,
1137                                                         (int(last_modified), int(last_modified)))
1138                                         except OSError:
1139                                                 # This fails on some odd non-unix-like filesystems.
1140                                                 # We don't really need the mtime to be preserved
1141                                                 # anyway here (currently we use ctime to trigger
1142                                                 # fetch), so just ignore it.
1143                                                 pass
1144                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1145                         finally:
1146                                 try:
1147                                         os.unlink(metadata_dtd_tmp)
1148                                 except OSError:
1149                                         pass
1150
1151                         url_f.close()
1152
1153                 except EnvironmentError as e:
1154                         print()
1155                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1156                         print(red("!!!")+" exception '%s' though." % (e,))
1157                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1158                         return False
1159
1160         return True
1161
1162 if options.mode == "manifest":
1163         pass
1164 elif not find_binary('xmllint'):
1165         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1166         if options.xml_parse or repolevel==3:
1167                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1168                 sys.exit(1)
1169 else:
1170         if not fetch_metadata_dtd():
1171                 sys.exit(1)
1172         #this can be problematic if xmllint changes their output
1173         xmllint_capable=True
1174
1175 if options.mode == 'commit' and vcs:
1176         utilities.detect_vcs_conflicts(options, vcs)
1177
1178 if options.mode == "manifest":
1179         pass
1180 elif options.pretend:
1181         print(green("\nRepoMan does a once-over of the neighborhood..."))
1182 else:
1183         print(green("\nRepoMan scours the neighborhood..."))
1184
1185 new_ebuilds = set()
1186 modified_ebuilds = set()
1187 modified_changelogs = set()
1188 mychanged = []
1189 mynew = []
1190 myremoved = []
1191
1192 if vcs == "cvs":
1193         mycvstree = cvstree.getentries("./", recursive=1)
1194         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1195         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1196         if options.if_modified == "y":
1197                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1198
1199 elif vcs == "svn":
1200         with repoman_popen("svn status") as f:
1201                 svnstatus = f.readlines()
1202         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1203         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1204         if options.if_modified == "y":
1205                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1206
1207 elif vcs == "git":
1208         with repoman_popen("git diff-index --name-only "
1209                 "--relative --diff-filter=M HEAD") as f:
1210                 mychanged = f.readlines()
1211         mychanged = ["./" + elem[:-1] for elem in mychanged]
1212
1213         with repoman_popen("git diff-index --name-only "
1214                 "--relative --diff-filter=A HEAD") as f:
1215                 mynew = f.readlines()
1216         mynew = ["./" + elem[:-1] for elem in mynew]
1217         if options.if_modified == "y":
1218                 with repoman_popen("git diff-index --name-only "
1219                         "--relative --diff-filter=D HEAD") as f:
1220                         myremoved = f.readlines()
1221                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1222
1223 elif vcs == "bzr":
1224         with repoman_popen("bzr status -S .") as f:
1225                 bzrstatus = f.readlines()
1226         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1227         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1228         if options.if_modified == "y":
1229                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1230
1231 elif vcs == "hg":
1232         with repoman_popen("hg status --no-status --modified .") as f:
1233                 mychanged = f.readlines()
1234         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1235         mynew = repoman_popen("hg status --no-status --added .").readlines()
1236         mynew = ["./" + elem.rstrip() for elem in mynew]
1237         if options.if_modified == "y":
1238                 with repoman_popen("hg status --no-status --removed .") as f:
1239                         myremoved = f.readlines()
1240                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1241
1242 if vcs:
1243         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1244         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1245         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1246                 if os.path.basename(x) == "ChangeLog")
1247
1248 def vcs_new_changed(relative_path):
1249         for x in chain(mychanged, mynew):
1250                 if x == relative_path:
1251                         return True
1252         return False
1253
1254 have_pmasked = False
1255 have_dev_keywords = False
1256 dofail = 0
1257
1258 # NOTE: match-all caches are not shared due to potential
1259 # differences between profiles in _get_implicit_iuse.
1260 arch_caches={}
1261 arch_xmatch_caches = {}
1262 shared_xmatch_caches = {"cp-list":{}}
1263
1264 # Disable the "ebuild.notadded" check when not in commit mode and
1265 # running `svn status` in every package dir will be too expensive.
1266
1267 check_ebuild_notadded = not \
1268         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1269
1270 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1271 thirdpartymirrors = []
1272 for v in repoman_settings.thirdpartymirrors().values():
1273         for v in v:
1274                 if not v.endswith("/"):
1275                         v += "/"
1276                 thirdpartymirrors.append(v)
1277
1278 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1279         """
1280         Implements doctype() as required to avoid deprecation warnings with
1281         >=python-2.7.
1282         """
1283         def doctype(self, name, pubid, system):
1284                 pass
1285
1286 try:
1287         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1288 except (EnvironmentError, ParseError, PermissionDenied) as e:
1289         err(str(e))
1290 except FileNotFound:
1291         # TODO: Download as we do for metadata.dtd, but add a way to
1292         # disable for non-gentoo repoman users who may not have herds.
1293         herd_base = None
1294
1295 effective_scanlist = scanlist
1296 if options.if_modified == "y":
1297         effective_scanlist = sorted(vcs_files_to_cps(
1298                 chain(mychanged, mynew, myremoved)))
1299
1300 for x in effective_scanlist:
1301         #ebuilds and digests added to cvs respectively.
1302         logging.info("checking package %s" % x)
1303         # save memory by discarding xmatch caches from previous package(s)
1304         arch_xmatch_caches.clear()
1305         eadded=[]
1306         catdir,pkgdir=x.split("/")
1307         checkdir=repodir+"/"+x
1308         checkdir_relative = ""
1309         if repolevel < 3:
1310                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1311         if repolevel < 2:
1312                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1313         checkdir_relative = os.path.join(".", checkdir_relative)
1314         generated_manifest = False
1315
1316         if options.mode == "manifest" or \
1317           (options.mode != 'manifest-check' and options.digest == 'y') or \
1318           options.mode in ('commit', 'fix') and not options.pretend:
1319                 auto_assumed = set()
1320                 fetchlist_dict = portage.FetchlistDict(checkdir,
1321                         repoman_settings, portdb)
1322                 if options.mode == 'manifest' and options.force:
1323                         portage._doebuild_manifest_exempt_depend += 1
1324                         try:
1325                                 distdir = repoman_settings['DISTDIR']
1326                                 mf = repoman_settings.repositories.get_repo_for_location(
1327                                         os.path.dirname(os.path.dirname(checkdir)))
1328                                 mf = mf.load_manifest(checkdir, distdir,
1329                                         fetchlist_dict=fetchlist_dict)
1330                                 mf.create(requiredDistfiles=None,
1331                                         assumeDistHashesAlways=True)
1332                                 for distfiles in fetchlist_dict.values():
1333                                         for distfile in distfiles:
1334                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1335                                                         mf.fhashdict['DIST'].pop(distfile, None)
1336                                                 else:
1337                                                         auto_assumed.add(distfile)
1338                                 mf.write()
1339                         finally:
1340                                 portage._doebuild_manifest_exempt_depend -= 1
1341
1342                 repoman_settings["O"] = checkdir
1343                 try:
1344                         generated_manifest = digestgen(
1345                                 mysettings=repoman_settings, myportdb=portdb)
1346                 except portage.exception.PermissionDenied as e:
1347                         generated_manifest = False
1348                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1349                                 level=logging.ERROR, noiselevel=-1)
1350
1351                 if not generated_manifest:
1352                         print("Unable to generate manifest.")
1353                         dofail = 1
1354
1355                 if options.mode == "manifest":
1356                         if not dofail and options.force and auto_assumed and \
1357                                 'assume-digests' in repoman_settings.features:
1358                                 # Show which digests were assumed despite the --force option
1359                                 # being given. This output will already have been shown by
1360                                 # digestgen() if assume-digests is not enabled, so only show
1361                                 # it here if assume-digests is enabled.
1362                                 pkgs = list(fetchlist_dict)
1363                                 pkgs.sort()
1364                                 portage.writemsg_stdout("  digest.assumed" + \
1365                                         portage.output.colorize("WARN",
1366                                         str(len(auto_assumed)).rjust(18)) + "\n")
1367                                 for cpv in pkgs:
1368                                         fetchmap = fetchlist_dict[cpv]
1369                                         pf = portage.catsplit(cpv)[1]
1370                                         for distfile in sorted(fetchmap):
1371                                                 if distfile in auto_assumed:
1372                                                         portage.writemsg_stdout(
1373                                                                 "   %s::%s\n" % (pf, distfile))
1374                         continue
1375                 elif dofail:
1376                         sys.exit(1)
1377
1378         if not generated_manifest:
1379                 repoman_settings['O'] = checkdir
1380                 repoman_settings['PORTAGE_QUIET'] = '1'
1381                 if not portage.digestcheck([], repoman_settings, strict=1):
1382                         stats["manifest.bad"] += 1
1383                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1384                 repoman_settings.pop('PORTAGE_QUIET', None)
1385
1386         if options.mode == 'manifest-check':
1387                 continue
1388
1389         checkdirlist=os.listdir(checkdir)
1390         ebuildlist=[]
1391         pkgs = {}
1392         allvalid = True
1393         for y in checkdirlist:
1394                 if (y in no_exec or y.endswith(".ebuild")) and \
1395                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1396                                 stats["file.executable"] += 1
1397                                 fails["file.executable"].append(os.path.join(checkdir, y))
1398                 if y.endswith(".ebuild"):
1399                         pf = y[:-7]
1400                         ebuildlist.append(pf)
1401                         cpv = "%s/%s" % (catdir, pf)
1402                         try:
1403                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1404                         except KeyError:
1405                                 allvalid = False
1406                                 stats["ebuild.syntax"] += 1
1407                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1408                                 continue
1409                         except IOError:
1410                                 allvalid = False
1411                                 stats["ebuild.output"] += 1
1412                                 fails["ebuild.output"].append(os.path.join(x, y))
1413                                 continue
1414                         if not portage.eapi_is_supported(myaux["EAPI"]):
1415                                 allvalid = False
1416                                 stats["EAPI.unsupported"] += 1
1417                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1418                                 continue
1419                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1420                                 root_config=root_config, type_name="ebuild")
1421
1422         slot_keywords = {}
1423
1424         if len(pkgs) != len(ebuildlist):
1425                 # If we can't access all the metadata then it's totally unsafe to
1426                 # commit since there's no way to generate a correct Manifest.
1427                 # Do not try to do any more QA checks on this package since missing
1428                 # metadata leads to false positives for several checks, and false
1429                 # positives confuse users.
1430                 can_force = False
1431                 continue
1432
1433         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1434         ebuildlist = sorted(pkgs.values())
1435         ebuildlist = [pkg.pf for pkg in ebuildlist]
1436
1437         for y in checkdirlist:
1438                 index = repo_config.find_invalid_path_char(y)
1439                 if index != -1:
1440                         y_relative = os.path.join(checkdir_relative, y)
1441                         if vcs is not None and not vcs_new_changed(y_relative):
1442                                 # If the file isn't in the VCS new or changed set, then
1443                                 # assume that it's an irrelevant temporary file (Manifest
1444                                 # entries are not generated for file names containing
1445                                 # prohibited characters). See bug #406877.
1446                                 index = -1
1447                 if index != -1:
1448                         stats["file.name"] += 1
1449                         fails["file.name"].append("%s/%s: char '%s'" % \
1450                                 (checkdir, y, y[index]))
1451
1452                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1453                         continue
1454                 f = None
1455                 try:
1456                         line = 1
1457                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1458                                 encoding=_encodings['fs'], errors='strict'),
1459                                 mode='r', encoding=_encodings['repo.content'])
1460                         for l in f:
1461                                 line +=1
1462                 except UnicodeDecodeError as ue:
1463                         stats["file.UTF8"] += 1
1464                         s = ue.object[:ue.start]
1465                         l2 = s.count("\n")
1466                         line += l2
1467                         if l2 != 0:
1468                                 s = s[s.rfind("\n") + 1:]
1469                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1470                 finally:
1471                         if f is not None:
1472                                 f.close()
1473
1474         if vcs in ("git", "hg") and check_ebuild_notadded:
1475                 if vcs == "git":
1476                         myf = repoman_popen("git ls-files --others %s" % \
1477                                 (portage._shell_quote(checkdir_relative),))
1478                 if vcs == "hg":
1479                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1480                                 (portage._shell_quote(checkdir_relative),))
1481                 for l in myf:
1482                         if l[:-1][-7:] == ".ebuild":
1483                                 stats["ebuild.notadded"] += 1
1484                                 fails["ebuild.notadded"].append(
1485                                         os.path.join(x, os.path.basename(l[:-1])))
1486                 myf.close()
1487
1488         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1489                 try:
1490                         if vcs == "cvs":
1491                                 myf=open(checkdir+"/CVS/Entries","r")
1492                         if vcs == "svn":
1493                                 myf = repoman_popen("svn status --depth=files --verbose " +
1494                                         portage._shell_quote(checkdir))
1495                         if vcs == "bzr":
1496                                 myf = repoman_popen("bzr ls -v --kind=file " +
1497                                         portage._shell_quote(checkdir))
1498                         myl = myf.readlines()
1499                         myf.close()
1500                         for l in myl:
1501                                 if vcs == "cvs":
1502                                         if l[0]!="/":
1503                                                 continue
1504                                         splitl=l[1:].split("/")
1505                                         if not len(splitl):
1506                                                 continue
1507                                         if splitl[0][-7:]==".ebuild":
1508                                                 eadded.append(splitl[0][:-7])
1509                                 if vcs == "svn":
1510                                         if l[:1] == "?":
1511                                                 continue
1512                                         if l[:7] == '      >':
1513                                                 # tree conflict, new in subversion 1.6
1514                                                 continue
1515                                         l = l.split()[-1]
1516                                         if l[-7:] == ".ebuild":
1517                                                 eadded.append(os.path.basename(l[:-7]))
1518                                 if vcs == "bzr":
1519                                         if l[1:2] == "?":
1520                                                 continue
1521                                         l = l.split()[-1]
1522                                         if l[-7:] == ".ebuild":
1523                                                 eadded.append(os.path.basename(l[:-7]))
1524                         if vcs == "svn":
1525                                 myf = repoman_popen("svn status " +
1526                                         portage._shell_quote(checkdir))
1527                                 myl=myf.readlines()
1528                                 myf.close()
1529                                 for l in myl:
1530                                         if l[0] == "A":
1531                                                 l = l.rstrip().split(' ')[-1]
1532                                                 if l[-7:] == ".ebuild":
1533                                                         eadded.append(os.path.basename(l[:-7]))
1534                 except IOError:
1535                         if vcs == "cvs":
1536                                 stats["CVS/Entries.IO_error"] += 1
1537                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1538                         else:
1539                                 raise
1540                         continue
1541
1542         mf = repoman_settings.repositories.get_repo_for_location(
1543                 os.path.dirname(os.path.dirname(checkdir)))
1544         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1545         mydigests=mf.getTypeDigests("DIST")
1546
1547         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1548         myfiles_all = []
1549         src_uri_error = False
1550         for mykey in fetchlist_dict:
1551                 try:
1552                         myfiles_all.extend(fetchlist_dict[mykey])
1553                 except portage.exception.InvalidDependString as e:
1554                         src_uri_error = True
1555                         try:
1556                                 portdb.aux_get(mykey, ["SRC_URI"])
1557                         except KeyError:
1558                                 # This will be reported as an "ebuild.syntax" error.
1559                                 pass
1560                         else:
1561                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1562                                 fails["SRC_URI.syntax"].append(
1563                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1564         del fetchlist_dict
1565         if not src_uri_error:
1566                 # This test can produce false positives if SRC_URI could not
1567                 # be parsed for one or more ebuilds. There's no point in
1568                 # producing a false error here since the root cause will
1569                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1570                 # or "ebuild.sytax".
1571                 myfiles_all = set(myfiles_all)
1572                 for entry in mydigests:
1573                         if entry not in myfiles_all:
1574                                 stats["digest.unused"] += 1
1575                                 fails["digest.unused"].append(checkdir+"::"+entry)
1576                 for entry in myfiles_all:
1577                         if entry not in mydigests:
1578                                 stats["digest.missing"] += 1
1579                                 fails["digest.missing"].append(checkdir+"::"+entry)
1580         del myfiles_all
1581
1582         if os.path.exists(checkdir+"/files"):
1583                 filesdirlist=os.listdir(checkdir+"/files")
1584
1585                 # recurse through files directory
1586                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1587                 while filesdirlist:
1588                         y = filesdirlist.pop(0)
1589                         relative_path = os.path.join(x, "files", y)
1590                         full_path = os.path.join(repodir, relative_path)
1591                         try:
1592                                 mystat = os.stat(full_path)
1593                         except OSError as oe:
1594                                 if oe.errno == 2:
1595                                         # don't worry about it.  it likely was removed via fix above.
1596                                         continue
1597                                 else:
1598                                         raise oe
1599                         if S_ISDIR(mystat.st_mode):
1600                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1601                                 if y == "CVS" or y == ".svn":
1602                                         continue
1603                                 for z in os.listdir(checkdir+"/files/"+y):
1604                                         if z == "CVS" or z == ".svn":
1605                                                 continue
1606                                         filesdirlist.append(y+"/"+z)
1607                         # Current policy is no files over 20 KiB, these are the checks. File size between
1608                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1609                         elif mystat.st_size > 61440:
1610                                 stats["file.size.fatal"] += 1
1611                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1612                         elif mystat.st_size > 20480:
1613                                 stats["file.size"] += 1
1614                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1615
1616                         index = repo_config.find_invalid_path_char(y)
1617                         if index != -1:
1618                                 y_relative = os.path.join(checkdir_relative, "files", y)
1619                                 if vcs is not None and not vcs_new_changed(y_relative):
1620                                         # If the file isn't in the VCS new or changed set, then
1621                                         # assume that it's an irrelevant temporary file (Manifest
1622                                         # entries are not generated for file names containing
1623                                         # prohibited characters). See bug #406877.
1624                                         index = -1
1625                         if index != -1:
1626                                 stats["file.name"] += 1
1627                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1628                                         (checkdir, y, y[index]))
1629         del mydigests
1630
1631         if check_changelog and "ChangeLog" not in checkdirlist:
1632                 stats["changelog.missing"]+=1
1633                 fails["changelog.missing"].append(x+"/ChangeLog")
1634
1635         musedict = {}
1636         #metadata.xml file check
1637         if "metadata.xml" not in checkdirlist:
1638                 stats["metadata.missing"]+=1
1639                 fails["metadata.missing"].append(x+"/metadata.xml")
1640         #metadata.xml parse check
1641         else:
1642                 metadata_bad = False
1643
1644                 # read metadata.xml into memory
1645                 try:
1646                         _metadata_xml = xml.etree.ElementTree.parse(
1647                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1648                                 encoding=_encodings['fs'], errors='strict'),
1649                                 parser=xml.etree.ElementTree.XMLParser(
1650                                         target=_MetadataTreeBuilder()))
1651                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1652                         metadata_bad = True
1653                         stats["metadata.bad"] += 1
1654                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1655                         del e
1656                 else:
1657                         # load USE flags from metadata.xml
1658                         try:
1659                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1660                         except portage.exception.ParseError as e:
1661                                 metadata_bad = True
1662                                 stats["metadata.bad"] += 1
1663                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1664
1665                         # Run other metadata.xml checkers
1666                         try:
1667                                 utilities.check_metadata(_metadata_xml, herd_base)
1668                         except (utilities.UnknownHerdsError, ) as e:
1669                                 metadata_bad = True
1670                                 stats["metadata.bad"] += 1
1671                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1672                                 del e
1673
1674                 #Only carry out if in package directory or check forced
1675                 if xmllint_capable and not metadata_bad:
1676                         # xmlint can produce garbage output even on success, so only dump
1677                         # the ouput when it fails.
1678                         st, out = repoman_getstatusoutput(
1679                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1680                                  (portage._shell_quote(metadata_dtd),
1681                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1682                         if st != os.EX_OK:
1683                                 print(red("!!!") + " metadata.xml is invalid:")
1684                                 for z in out.splitlines():
1685                                         print(red("!!! ")+z)
1686                                 stats["metadata.bad"]+=1
1687                                 fails["metadata.bad"].append(x+"/metadata.xml")
1688
1689                 del metadata_bad
1690         muselist = frozenset(musedict)
1691
1692         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1693         changelog_modified = changelog_path in modified_changelogs
1694
1695         # detect unused local USE-descriptions
1696         used_useflags = set()
1697
1698         for y in ebuildlist:
1699                 relative_path = os.path.join(x, y + ".ebuild")
1700                 full_path = os.path.join(repodir, relative_path)
1701                 ebuild_path = y + ".ebuild"
1702                 if repolevel < 3:
1703                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1704                 if repolevel < 2:
1705                         ebuild_path = os.path.join(catdir, ebuild_path)
1706                 ebuild_path = os.path.join(".", ebuild_path)
1707                 if check_changelog and not changelog_modified \
1708                         and ebuild_path in new_ebuilds:
1709                         stats['changelog.ebuildadded'] += 1
1710                         fails['changelog.ebuildadded'].append(relative_path)
1711
1712                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1713                         #ebuild not added to vcs
1714                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1715                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1716                 myesplit=portage.pkgsplit(y)
1717                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1718                             or pv_toolong_re.search(myesplit[1]) \
1719                             or pv_toolong_re.search(myesplit[2]):
1720                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1721                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1722                         continue
1723                 elif myesplit[0]!=pkgdir:
1724                         print(pkgdir,myesplit[0])
1725                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1726                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1727                         continue
1728
1729                 pkg = pkgs[y]
1730
1731                 if pkg.invalid:
1732                         allvalid = False
1733                         for k, msgs in pkg.invalid.items():
1734                                 for msg in msgs:
1735                                         stats[k] = stats[k] + 1
1736                                         fails[k].append("%s %s" % (relative_path, msg))
1737                         continue
1738
1739                 myaux = pkg.metadata
1740                 eapi = myaux["EAPI"]
1741                 inherited = pkg.inherited
1742                 live_ebuild = live_eclasses.intersection(inherited)
1743
1744                 for k, v in myaux.items():
1745                         if not isinstance(v, basestring):
1746                                 continue
1747                         m = non_ascii_re.search(v)
1748                         if m is not None:
1749                                 stats["variable.invalidchar"] += 1
1750                                 fails["variable.invalidchar"].append(
1751                                         ("%s: %s variable contains non-ASCII " + \
1752                                         "character at position %s") % \
1753                                         (relative_path, k, m.start() + 1))
1754
1755                 if not src_uri_error:
1756                         # Check that URIs don't reference a server from thirdpartymirrors.
1757                         for uri in portage.dep.use_reduce( \
1758                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1759                                 contains_mirror = False
1760                                 for mirror in thirdpartymirrors:
1761                                         if uri.startswith(mirror):
1762                                                 contains_mirror = True
1763                                                 break
1764                                 if not contains_mirror:
1765                                         continue
1766
1767                                 stats["SRC_URI.mirror"] += 1
1768                                 fails["SRC_URI.mirror"].append(
1769                                         "%s: '%s' found in thirdpartymirrors" % \
1770                                         (relative_path, mirror))
1771
1772                 if myaux.get("PROVIDE"):
1773                         stats["virtual.oldstyle"]+=1
1774                         fails["virtual.oldstyle"].append(relative_path)
1775
1776                 for pos, missing_var in enumerate(missingvars):
1777                         if not myaux.get(missing_var):
1778                                 if catdir == "virtual" and \
1779                                         missing_var in ("HOMEPAGE", "LICENSE"):
1780                                         continue
1781                                 if live_ebuild and missing_var == "KEYWORDS":
1782                                         continue
1783                                 myqakey=missingvars[pos]+".missing"
1784                                 stats[myqakey]=stats[myqakey]+1
1785                                 fails[myqakey].append(x+"/"+y+".ebuild")
1786
1787                 if catdir == "virtual":
1788                         for var in ("HOMEPAGE", "LICENSE"):
1789                                 if myaux.get(var):
1790                                         myqakey = var + ".virtual"
1791                                         stats[myqakey] = stats[myqakey] + 1
1792                                         fails[myqakey].append(relative_path)
1793
1794                 # 14 is the length of DESCRIPTION=""
1795                 if len(myaux['DESCRIPTION']) > max_desc_len:
1796                         stats['DESCRIPTION.toolong'] += 1
1797                         fails['DESCRIPTION.toolong'].append(
1798                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1799                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1800
1801                 keywords = myaux["KEYWORDS"].split()
1802                 stable_keywords = []
1803                 for keyword in keywords:
1804                         if not keyword.startswith("~") and \
1805                                 not keyword.startswith("-"):
1806                                 stable_keywords.append(keyword)
1807                 if stable_keywords:
1808                         if ebuild_path in new_ebuilds:
1809                                 stable_keywords.sort()
1810                                 stats["KEYWORDS.stable"] += 1
1811                                 fails["KEYWORDS.stable"].append(
1812                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1813                                                 " ".join(stable_keywords))
1814
1815                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1816                         if not kw.startswith("-"))
1817
1818                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1819                 if previous_keywords is None:
1820                         slot_keywords[myaux["SLOT"]] = set()
1821                 elif ebuild_archs and not live_ebuild:
1822                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1823                         if dropped_keywords:
1824                                 stats["KEYWORDS.dropped"] += 1
1825                                 fails["KEYWORDS.dropped"].append(
1826                                         relative_path + ": %s" % \
1827                                         " ".join(sorted(dropped_keywords)))
1828
1829                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1830
1831                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1832                 if "-*" in keywords:
1833                         haskeyword = False
1834                         for kw in keywords:
1835                                 if kw[0] == "~":
1836                                         kw = kw[1:]
1837                                 if kw in kwlist:
1838                                         haskeyword = True
1839                         if not haskeyword:
1840                                 stats["KEYWORDS.stupid"] += 1
1841                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1842
1843                 """
1844                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1845                 not be allowed to be marked stable
1846                 """
1847                 if live_ebuild and repo_config.name == "gentoo":
1848                         bad_stable_keywords = []
1849                         for keyword in keywords:
1850                                 if not keyword.startswith("~") and \
1851                                         not keyword.startswith("-"):
1852                                         bad_stable_keywords.append(keyword)
1853                                 del keyword
1854                         if bad_stable_keywords:
1855                                 stats["LIVEVCS.stable"] += 1
1856                                 fails["LIVEVCS.stable"].append(
1857                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1858                                                 bad_stable_keywords)
1859                         del bad_stable_keywords
1860
1861                         if keywords and not has_global_mask(pkg):
1862                                 stats["LIVEVCS.unmasked"] += 1
1863                                 fails["LIVEVCS.unmasked"].append(relative_path)
1864
1865                 if options.ignore_arches:
1866                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1867                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1868                 else:
1869                         arches=[]
1870                         for keyword in myaux["KEYWORDS"].split():
1871                                 if (keyword[0]=="-"):
1872                                         continue
1873                                 elif (keyword[0]=="~"):
1874                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1875                                 else:
1876                                         arches.append([keyword, keyword, [keyword]])
1877                         if not arches:
1878                                 # Use an empty profile for checking dependencies of
1879                                 # packages that have empty KEYWORDS.
1880                                 arches.append(['**', '**', ['**']])
1881
1882                 unknown_pkgs = set()
1883                 baddepsyntax = False
1884                 badlicsyntax = False
1885                 badprovsyntax = False
1886                 catpkg = catdir+"/"+y
1887
1888                 inherited_java_eclass = "java-pkg-2" in inherited or \
1889                         "java-pkg-opt-2" in inherited
1890                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1891                 operator_tokens = set(["||", "(", ")"])
1892                 type_list, badsyntax = [], []
1893                 for mytype in ("DEPEND", "HDEPEND", "RDEPEND", "PDEPEND",
1894                         "LICENSE", "PROPERTIES", "PROVIDE"):
1895                         mydepstr = myaux[mytype]
1896
1897                         buildtime = mytype in ('DEPEND', 'HDEPEND')
1898                         token_class = None
1899                         if mytype.endswith("DEPEND"):
1900                                 token_class=portage.dep.Atom
1901
1902                         try:
1903                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1904                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1905                         except portage.exception.InvalidDependString as e:
1906                                 atoms = None
1907                                 badsyntax.append(str(e))
1908
1909                         if atoms and mytype.endswith("DEPEND"):
1910                                 if mytype in ("RDEPEND", "PDEPEND") and \
1911                                         "test?" in mydepstr.split():
1912                                         stats[mytype + '.suspect'] += 1
1913                                         fails[mytype + '.suspect'].append(relative_path + \
1914                                                 ": 'test?' USE conditional in %s" % mytype)
1915
1916                                 for atom in atoms:
1917                                         if atom == "||":
1918                                                 continue
1919
1920                                         # Skip dependency.unknown for blockers, so that we
1921                                         # don't encourage people to remove necessary blockers,
1922                                         # as discussed in bug #382407.
1923                                         if atom.blocker is None and \
1924                                                 not portdb.xmatch("match-all", atom) and \
1925                                                 not atom.cp.startswith("virtual/"):
1926                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1927
1928                                         is_blocker = atom.blocker
1929
1930                                         if catdir != "virtual":
1931                                                 if not is_blocker and \
1932                                                         atom.cp in suspect_virtual:
1933                                                         stats['virtual.suspect'] += 1
1934                                                         fails['virtual.suspect'].append(
1935                                                                 relative_path +
1936                                                                 ": %s: consider using '%s' instead of '%s'" %
1937                                                                 (mytype, suspect_virtual[atom.cp], atom))
1938
1939                                         if buildtime and \
1940                                                 not is_blocker and \
1941                                                 not inherited_java_eclass and \
1942                                                 atom.cp == "virtual/jdk":
1943                                                 stats['java.eclassesnotused'] += 1
1944                                                 fails['java.eclassesnotused'].append(relative_path)
1945                                         elif buildtime and \
1946                                                 not is_blocker and \
1947                                                 not inherited_wxwidgets_eclass and \
1948                                                 atom.cp == "x11-libs/wxGTK":
1949                                                 stats['wxwidgets.eclassnotused'] += 1
1950                                                 fails['wxwidgets.eclassnotused'].append(
1951                                                         (relative_path + ": %ss on x11-libs/wxGTK"
1952                                                         " without inheriting wxwidgets.eclass") % mytype)
1953                                         elif mytype in ("PDEPEND", "RDEPEND"):
1954                                                 if not is_blocker and \
1955                                                         atom.cp in suspect_rdepend:
1956                                                         stats[mytype + '.suspect'] += 1
1957                                                         fails[mytype + '.suspect'].append(
1958                                                                 relative_path + ": '%s'" % atom)
1959
1960                                         if atom.operator == "~" and \
1961                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1962                                                 stats[mytype + '.badtilde'] += 1
1963                                                 fails[mytype + '.badtilde'].append(
1964                                                         (relative_path + ": %s uses the ~ operator"
1965                                                          " with a non-zero revision:" + \
1966                                                          " '%s'") % (mytype, atom))
1967
1968                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1969
1970                 for m,b in zip(type_list, badsyntax):
1971                         stats[m+".syntax"] += 1
1972                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1973
1974                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1975                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1976                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1977                 badlicsyntax = badlicsyntax > 0
1978                 badprovsyntax = badprovsyntax > 0
1979
1980                 # uselist checks - global
1981                 myuse = []
1982                 default_use = []
1983                 for myflag in myaux["IUSE"].split():
1984                         flag_name = myflag.lstrip("+-")
1985                         used_useflags.add(flag_name)
1986                         if myflag != flag_name:
1987                                 default_use.append(myflag)
1988                         if flag_name not in uselist:
1989                                 myuse.append(flag_name)
1990
1991                 # uselist checks - metadata
1992                 for mypos in range(len(myuse)-1,-1,-1):
1993                         if myuse[mypos] and (myuse[mypos] in muselist):
1994                                 del myuse[mypos]
1995
1996                 if default_use and not eapi_has_iuse_defaults(eapi):
1997                         for myflag in default_use:
1998                                 stats['EAPI.incompatible'] += 1
1999                                 fails['EAPI.incompatible'].append(
2000                                         (relative_path + ": IUSE defaults" + \
2001                                         " not supported with EAPI='%s':" + \
2002                                         " '%s'") % (eapi, myflag))
2003
2004                 for mypos in range(len(myuse)):
2005                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
2006                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
2007
2008                 # license checks
2009                 if not badlicsyntax:
2010                         # Parse the LICENSE variable, remove USE conditions and
2011                         # flatten it.
2012                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2013                         # Check each entry to ensure that it exists in PORTDIR's
2014                         # license directory.
2015                         for lic in licenses:
2016                                 # Need to check for "||" manually as no portage
2017                                 # function will remove it without removing values.
2018                                 if lic not in liclist and lic != "||":
2019                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
2020                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
2021
2022                 #keyword checks
2023                 myuse = myaux["KEYWORDS"].split()
2024                 for mykey in myuse:
2025                         if mykey not in ("-*", "*", "~*"):
2026                                 myskey = mykey
2027                                 if myskey[:1] == "-":
2028                                         myskey = myskey[1:]
2029                                 if myskey[:1] == "~":
2030                                         myskey = myskey[1:]
2031                                 if myskey not in kwlist:
2032                                         stats["KEYWORDS.invalid"] += 1
2033                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
2034                                 elif myskey not in profiles:
2035                                         stats["KEYWORDS.invalid"] += 1
2036                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
2037
2038                 #restrict checks
2039                 myrestrict = None
2040                 try:
2041                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2042                 except portage.exception.InvalidDependString as e:
2043                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
2044                         fails["RESTRICT.syntax"].append(
2045                                 "%s: RESTRICT: %s" % (relative_path, e))
2046                         del e
2047                 if myrestrict:
2048                         myrestrict = set(myrestrict)
2049                         mybadrestrict = myrestrict.difference(valid_restrict)
2050                         if mybadrestrict:
2051                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2052                                 for mybad in mybadrestrict:
2053                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
2054                 #REQUIRED_USE check
2055                 required_use = myaux["REQUIRED_USE"]
2056                 if required_use:
2057                         if not eapi_has_required_use(eapi):
2058                                 stats['EAPI.incompatible'] += 1
2059                                 fails['EAPI.incompatible'].append(
2060                                         relative_path + ": REQUIRED_USE" + \
2061                                         " not supported with EAPI='%s'" % (eapi,))
2062                         try:
2063                                 portage.dep.check_required_use(required_use, (),
2064                                         pkg.iuse.is_valid_flag, eapi=eapi)
2065                         except portage.exception.InvalidDependString as e:
2066                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2067                                 fails["REQUIRED_USE.syntax"].append(
2068                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2069                                 del e
2070
2071                 # Syntax Checks
2072                 relative_path = os.path.join(x, y + ".ebuild")
2073                 full_path = os.path.join(repodir, relative_path)
2074                 if not vcs_preserves_mtime:
2075                         if ebuild_path not in new_ebuilds and \
2076                                 ebuild_path not in modified_ebuilds:
2077                                 pkg.mtime = None
2078                 try:
2079                         # All ebuilds should have utf_8 encoding.
2080                         f = io.open(_unicode_encode(full_path,
2081                                 encoding=_encodings['fs'], errors='strict'),
2082                                 mode='r', encoding=_encodings['repo.content'])
2083                         try:
2084                                 for check_name, e in run_checks(f, pkg):
2085                                         stats[check_name] += 1
2086                                         fails[check_name].append(relative_path + ': %s' % e)
2087                         finally:
2088                                 f.close()
2089                 except UnicodeDecodeError:
2090                         # A file.UTF8 failure will have already been recorded above.
2091                         pass
2092
2093                 if options.force:
2094                         # The dep_check() calls are the most expensive QA test. If --force
2095                         # is enabled, there's no point in wasting time on these since the
2096                         # user is intent on forcing the commit anyway.
2097                         continue
2098
2099                 for keyword,arch,groups in arches:
2100
2101                         if arch not in profiles:
2102                                 # A missing profile will create an error further down
2103                                 # during the KEYWORDS verification.
2104                                 continue
2105
2106                         for prof in profiles[arch]:
2107
2108                                 if prof.status not in ("stable", "dev") or \
2109                                         prof.status == "dev" and not options.include_dev:
2110                                         continue
2111
2112                                 dep_settings = arch_caches.get(prof.sub_path)
2113                                 if dep_settings is None:
2114                                         dep_settings = portage.config(
2115                                                 config_profile_path=prof.abs_path,
2116                                                 config_incrementals=repoman_incrementals,
2117                                                 config_root=config_root,
2118                                                 local_config=False,
2119                                                 _unmatched_removal=options.unmatched_removal,
2120                                                 env=env)
2121                                         dep_settings.categories = repoman_settings.categories
2122                                         if options.without_mask:
2123                                                 dep_settings._mask_manager_obj = \
2124                                                         copy.deepcopy(dep_settings._mask_manager)
2125                                                 dep_settings._mask_manager._pmaskdict.clear()
2126                                         arch_caches[prof.sub_path] = dep_settings
2127
2128                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2129                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2130                                 if xcache is None:
2131                                         portdb.melt()
2132                                         portdb.freeze()
2133                                         xcache = portdb.xcache
2134                                         xcache.update(shared_xmatch_caches)
2135                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2136
2137                                 trees[root]["porttree"].settings = dep_settings
2138                                 portdb.settings = dep_settings
2139                                 portdb.xcache = xcache
2140                                 # for package.use.mask support inside dep_check
2141                                 dep_settings.setcpv(pkg)
2142                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2143                                 # just in case, prevent config.reset() from nuking these.
2144                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2145
2146                                 if not baddepsyntax:
2147                                         ismasked = not ebuild_archs or \
2148                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2149                                         if ismasked:
2150                                                 if not have_pmasked:
2151                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2152                                                                 pkg.cpv, pkg.metadata))
2153                                                 if options.ignore_masked:
2154                                                         continue
2155                                                 #we are testing deps for a masked package; give it some lee-way
2156                                                 suffix="masked"
2157                                                 matchmode = "minimum-all"
2158                                         else:
2159                                                 suffix=""
2160                                                 matchmode = "minimum-visible"
2161
2162                                         if not have_dev_keywords:
2163                                                 have_dev_keywords = \
2164                                                         bool(dev_keywords.intersection(keywords))
2165
2166                                         if prof.status == "dev":
2167                                                 suffix=suffix+"indev"
2168
2169                                         for mytype in ("DEPEND", "HDEPEND", "PDEPEND", "RDEPEND"):
2170
2171                                                 mykey=mytype+".bad"+suffix
2172                                                 myvalue = myaux[mytype]
2173                                                 if not myvalue:
2174                                                         continue
2175
2176                                                 success, atoms = portage.dep_check(myvalue, portdb,
2177                                                         dep_settings, use="all", mode=matchmode,
2178                                                         trees=trees)
2179
2180                                                 if success:
2181                                                         if atoms:
2182
2183                                                                 # Don't bother with dependency.unknown for
2184                                                                 # cases in which *DEPEND.bad is triggered.
2185                                                                 for atom in atoms:
2186                                                                         # dep_check returns all blockers and they
2187                                                                         # aren't counted for *DEPEND.bad, so we
2188                                                                         # ignore them here.
2189                                                                         if not atom.blocker:
2190                                                                                 unknown_pkgs.discard(
2191                                                                                         (mytype, atom.unevaluated_atom))
2192
2193                                                                 if not prof.sub_path:
2194                                                                         # old-style virtuals currently aren't
2195                                                                         # resolvable with empty profile, since
2196                                                                         # 'virtuals' mappings are unavailable
2197                                                                         # (it would be expensive to search
2198                                                                         # for PROVIDE in all ebuilds)
2199                                                                         atoms = [atom for atom in atoms if not \
2200                                                                                 (atom.cp.startswith('virtual/') and \
2201                                                                                 not portdb.cp_list(atom.cp))]
2202
2203                                                                 #we have some unsolvable deps
2204                                                                 #remove ! deps, which always show up as unsatisfiable
2205                                                                 atoms = [str(atom.unevaluated_atom) \
2206                                                                         for atom in atoms if not atom.blocker]
2207
2208                                                                 #if we emptied out our list, continue:
2209                                                                 if not atoms:
2210                                                                         continue
2211                                                                 stats[mykey]=stats[mykey]+1
2212                                                                 fails[mykey].append("%s: %s(%s) %s" % \
2213                                                                         (relative_path, keyword,
2214                                                                         prof, repr(atoms)))
2215                                                 else:
2216                                                         stats[mykey]=stats[mykey]+1
2217                                                         fails[mykey].append("%s: %s(%s) %s" % \
2218                                                                 (relative_path, keyword,
2219                                                                 prof, repr(atoms)))
2220
2221                 if not baddepsyntax and unknown_pkgs:
2222                         type_map = {}
2223                         for mytype, atom in unknown_pkgs:
2224                                 type_map.setdefault(mytype, set()).add(atom)
2225                         for mytype, atoms in type_map.items():
2226                                 stats["dependency.unknown"] += 1
2227                                 fails["dependency.unknown"].append("%s: %s: %s" %
2228                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2229
2230         # check if there are unused local USE-descriptions in metadata.xml
2231         # (unless there are any invalids, to avoid noise)
2232         if allvalid:
2233                 for myflag in muselist.difference(used_useflags):
2234                         stats["metadata.warning"] += 1
2235                         fails["metadata.warning"].append(
2236                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2237                                 (x, myflag))
2238
2239 if options.if_modified == "y" and len(effective_scanlist) < 1:
2240         logging.warn("--if-modified is enabled, but no modified packages were found!")
2241
2242 if options.mode == "manifest":
2243         sys.exit(dofail)
2244
2245 #dofail will be set to 1 if we have failed in at least one non-warning category
2246 dofail=0
2247 #dowarn will be set to 1 if we tripped any warnings
2248 dowarn=0
2249 #dofull will be set if we should print a "repoman full" informational message
2250 dofull = options.mode != 'full'
2251
2252 for x in qacats:
2253         if not stats[x]:
2254                 continue
2255         dowarn = 1
2256         if x not in qawarnings:
2257                 dofail = 1
2258
2259 if dofail or \
2260         (dowarn and not (options.quiet or options.mode == "scan")):
2261         dofull = 0
2262
2263 # Save QA output so that it can be conveniently displayed
2264 # in $EDITOR while the user creates a commit message.
2265 # Otherwise, the user would not be able to see this output
2266 # once the editor has taken over the screen.
2267 qa_output = io.StringIO()
2268 style_file = ConsoleStyleFile(sys.stdout)
2269 if options.mode == 'commit' and \
2270         (not commitmessage or not commitmessage.strip()):
2271         style_file.write_listener = qa_output
2272 console_writer = StyleWriter(file=style_file, maxcol=9999)
2273 console_writer.style_listener = style_file.new_styles
2274
2275 f = formatter.AbstractFormatter(console_writer)
2276
2277 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2278
2279 style_file.flush()
2280 del console_writer, f, style_file
2281 qa_output = qa_output.getvalue()
2282 qa_output = qa_output.splitlines(True)
2283
2284 def grouplist(mylist,seperator="/"):
2285         """(list,seperator="/") -- Takes a list of elements; groups them into
2286         same initial element categories. Returns a dict of {base:[sublist]}
2287         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2288         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2289         mygroups={}
2290         for x in mylist:
2291                 xs=x.split(seperator)
2292                 if xs[0]==".":
2293                         xs=xs[1:]
2294                 if xs[0] not in mygroups:
2295                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2296                 else:
2297                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2298         return mygroups
2299
2300 suggest_ignore_masked = False
2301 suggest_include_dev = False
2302
2303 if have_pmasked and not (options.without_mask or options.ignore_masked):
2304         suggest_ignore_masked = True
2305 if have_dev_keywords and not options.include_dev:
2306         suggest_include_dev = True
2307
2308 if suggest_ignore_masked or suggest_include_dev:
2309         print()
2310         if suggest_ignore_masked:
2311                 print(bold("Note: use --without-mask to check " + \
2312                         "KEYWORDS on dependencies of masked packages"))
2313
2314         if suggest_include_dev:
2315                 print(bold("Note: use --include-dev (-d) to check " + \
2316                         "dependencies for 'dev' profiles"))
2317         print()
2318
2319 if options.mode != 'commit':
2320         if dofull:
2321                 print(bold("Note: type \"repoman full\" for a complete listing."))
2322         if dowarn and not dofail:
2323                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2324         elif not dofail:
2325                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2326         elif dofail:
2327                 print(bad("Please fix these important QA issues first."))
2328                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2329                 sys.exit(1)
2330 else:
2331         if dofail and can_force and options.force and not options.pretend:
2332                 print(green("RepoMan sez:") + \
2333                         " \"You want to commit even with these QA issues?\n" + \
2334                         "              I'll take it this time, but I'm not happy.\"\n")
2335         elif dofail:
2336                 if options.force and not can_force:
2337                         print(bad("The --force option has been disabled due to extraordinary issues."))
2338                 print(bad("Please fix these important QA issues first."))
2339                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2340                 sys.exit(1)
2341
2342         if options.pretend:
2343                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2344
2345         myunadded = []
2346         if vcs == "cvs":
2347                 try:
2348                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2349                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2350                 except SystemExit as e:
2351                         raise  # TODO propagate this
2352                 except:
2353                         err("Error retrieving CVS tree; exiting.")
2354         if vcs == "svn":
2355                 try:
2356                         with repoman_popen("svn status --no-ignore") as f:
2357                                 svnstatus = f.readlines()
2358                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2359                 except SystemExit as e:
2360                         raise  # TODO propagate this
2361                 except:
2362                         err("Error retrieving SVN info; exiting.")
2363         if vcs == "git":
2364                 # get list of files not under version control or missing
2365                 myf = repoman_popen("git ls-files --others")
2366                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2367                 myf.close()
2368         if vcs == "bzr":
2369                 try:
2370                         with repoman_popen("bzr status -S .") as f:
2371                                 bzrstatus = f.readlines()
2372                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2373                 except SystemExit as e:
2374                         raise  # TODO propagate this
2375                 except:
2376                         err("Error retrieving bzr info; exiting.")
2377         if vcs == "hg":
2378                 with repoman_popen("hg status --no-status --unknown .") as f:
2379                         myunadded = f.readlines()
2380                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2381
2382                 # Mercurial doesn't handle manually deleted files as removed from
2383                 # the repository, so the user need to remove them before commit,
2384                 # using "hg remove [FILES]"
2385                 with repoman_popen("hg status --no-status --deleted .") as f:
2386                         mydeleted = f.readlines()
2387                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2388
2389
2390         myautoadd=[]
2391         if myunadded:
2392                 for x in range(len(myunadded)-1,-1,-1):
2393                         xs=myunadded[x].split("/")
2394                         if xs[-1]=="files":
2395                                 print("!!! files dir is not added! Please correct this.")
2396                                 sys.exit(-1)
2397                         elif xs[-1]=="Manifest":
2398                                 # It's a manifest... auto add
2399                                 myautoadd+=[myunadded[x]]
2400                                 del myunadded[x]
2401
2402         if myunadded:
2403                 print(red("!!! The following files are in your local tree but are not added to the master"))
2404                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2405                 for x in myunadded:
2406                         print("   ",x)
2407                 print()
2408                 print()
2409                 sys.exit(1)
2410
2411         if vcs == "hg" and mydeleted:
2412                 print(red("!!! The following files are removed manually from your local tree but are not"))
2413                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2414                 for x in mydeleted:
2415                         print("   ",x)
2416                 print()
2417                 print()
2418                 sys.exit(1)
2419
2420         if vcs == "cvs":
2421                 mycvstree = cvstree.getentries("./", recursive=1)
2422                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2423                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2424                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2425                 bin_blob_pattern = re.compile("^-kb$")
2426                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2427                         recursive=1, basedir="./"))
2428
2429
2430         if vcs == "svn":
2431                 with repoman_popen("svn status") as f:
2432                         svnstatus = f.readlines()
2433                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2434                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2435                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2436
2437                 # Subversion expands keywords specified in svn:keywords properties.
2438                 with repoman_popen("svn propget -R svn:keywords") as f:
2439                         props = f.readlines()
2440                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2441                         for prop in props if " - " in prop)
2442
2443         elif vcs == "git":
2444                 with repoman_popen("git diff-index --name-only "
2445                         "--relative --diff-filter=M HEAD") as f:
2446                         mychanged = f.readlines()
2447                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2448
2449                 with repoman_popen("git diff-index --name-only "
2450                         "--relative --diff-filter=A HEAD") as f:
2451                         mynew = f.readlines()
2452                 mynew = ["./" + elem[:-1] for elem in mynew]
2453
2454                 with repoman_popen("git diff-index --name-only "
2455                         "--relative --diff-filter=D HEAD") as f:
2456                         myremoved = f.readlines()
2457                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2458
2459         if vcs == "bzr":
2460                 with repoman_popen("bzr status -S .") as f:
2461                         bzrstatus = f.readlines()
2462                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2463                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2464                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2465                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2466                 # Bazaar expands nothing.
2467
2468         if vcs == "hg":
2469                 with repoman_popen("hg status --no-status --modified .") as f:
2470                         mychanged = f.readlines()
2471                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2472
2473                 with repoman_popen("hg status --no-status --added .") as f:
2474                         mynew = f.readlines()
2475                 mynew = ["./" + elem.rstrip() for elem in mynew]
2476
2477                 with repoman_popen("hg status --no-status --removed .") as f:
2478                         myremoved = f.readlines()
2479                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2480
2481         if vcs:
2482                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2483                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2484                         print()
2485                         print("(Didn't find any changed files...)")
2486                         print()
2487                         sys.exit(1)
2488
2489         # Manifests need to be regenerated after all other commits, so don't commit
2490         # them now even if they have changed.
2491         mymanifests = set()
2492         myupdates = set()
2493         for f in mychanged + mynew:
2494                 if "Manifest" == os.path.basename(f):
2495                         mymanifests.add(f)
2496                 else:
2497                         myupdates.add(f)
2498         myupdates.difference_update(myremoved)
2499         myupdates = list(myupdates)
2500         mymanifests = list(mymanifests)
2501         myheaders = []
2502         mydirty = []
2503
2504         commitmessage = options.commitmsg
2505         if options.commitmsgfile:
2506                 try:
2507                         f = io.open(_unicode_encode(options.commitmsgfile,
2508                         encoding=_encodings['fs'], errors='strict'),
2509                         mode='r', encoding=_encodings['content'], errors='replace')
2510                         commitmessage = f.read()
2511                         f.close()
2512                         del f
2513                 except (IOError, OSError) as e:
2514                         if e.errno == errno.ENOENT:
2515                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2516                         else:
2517                                 raise
2518                 # We've read the content so the file is no longer needed.
2519                 commitmessagefile = None
2520         if not commitmessage or not commitmessage.strip():
2521                 try:
2522                         editor = os.environ.get("EDITOR")
2523                         if editor and utilities.editor_is_executable(editor):
2524                                 commitmessage = utilities.get_commit_message_with_editor(
2525                                         editor, message=qa_output)
2526                         else:
2527                                 commitmessage = utilities.get_commit_message_with_stdin()
2528                 except KeyboardInterrupt:
2529                         exithandler()
2530                 if not commitmessage or not commitmessage.strip():
2531                         print("* no commit message?  aborting commit.")
2532                         sys.exit(1)
2533         commitmessage = commitmessage.rstrip()
2534         changelog_msg = commitmessage
2535         portage_version = getattr(portage, "VERSION", None)
2536         if portage_version is None:
2537                 sys.stderr.write("Failed to insert portage version in message!\n")
2538                 sys.stderr.flush()
2539                 portage_version = "Unknown"
2540         unameout = platform.system() + " "
2541         if platform.system() in ["Darwin", "SunOS"]:
2542                 unameout += platform.processor()
2543         else:
2544                 unameout += platform.machine()
2545         commitmessage += "\n\n(Portage version: %s/%s/%s" % \
2546                 (portage_version, vcs, unameout)
2547         if options.force:
2548                 commitmessage += ", RepoMan options: --force"
2549         commitmessage += ")"
2550
2551         if options.echangelog in ('y', 'force'):
2552                 logging.info("checking for unmodified ChangeLog files")
2553                 committer_name = utilities.get_committer_name(env=repoman_settings)
2554                 for x in sorted(vcs_files_to_cps(
2555                         chain(myupdates, mymanifests, myremoved))):
2556                         catdir, pkgdir = x.split("/")
2557                         checkdir = repodir + "/" + x
2558                         checkdir_relative = ""
2559                         if repolevel < 3:
2560                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2561                         if repolevel < 2:
2562                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2563                         checkdir_relative = os.path.join(".", checkdir_relative)
2564
2565                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2566                         changelog_modified = changelog_path in modified_changelogs
2567                         if changelog_modified and options.echangelog != 'force':
2568                                 continue
2569
2570                         # get changes for this package
2571                         cdrlen = len(checkdir_relative)
2572                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2573                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2574                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2575
2576                         # Skip ChangeLog generation if only the Manifest was modified,
2577                         # as discussed in bug #398009.
2578                         nontrivial_cl_files = set()
2579                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2580                         nontrivial_cl_files.difference_update(['Manifest'])
2581                         if not nontrivial_cl_files and options.echangelog != 'force':
2582                                 continue
2583
2584                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2585                                 committer_name, changelog_msg,
2586                                 os.path.join(repodir, 'skel.ChangeLog'),
2587                                 catdir, pkgdir,
2588                                 new=clnew, removed=clremoved, changed=clchanged,
2589                                 pretend=options.pretend)
2590                         if new_changelog is None:
2591                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2592                                         level=logging.ERROR, noiselevel=-1)
2593                                 sys.exit(1)
2594
2595                         # if the ChangeLog was just created, add it to vcs
2596                         if new_changelog:
2597                                 myautoadd.append(changelog_path)
2598                                 # myautoadd is appended to myupdates below
2599                         else:
2600                                 myupdates.append(changelog_path)
2601
2602                         if options.ask and not options.pretend:
2603                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2604                                 repoman_settings["O"] = checkdir
2605                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2606
2607         if myautoadd:
2608                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2609                 add_cmd = [vcs, "add"]
2610                 add_cmd += myautoadd
2611                 if options.pretend:
2612                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2613                                 noiselevel=-1)
2614                 else:
2615                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2616                                 # Python 3.1 produces the following TypeError if raw bytes are
2617                                 # passed to subprocess.call():
2618                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2619                                 #     errread, errwrite)
2620                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2621                                 #     raise child_exception
2622                                 # TypeError: expected an object with the buffer interface
2623                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2624                         retcode = subprocess.call(add_cmd)
2625                         if retcode != os.EX_OK:
2626                                 logging.error(
2627                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2628                                 sys.exit(retcode)
2629
2630                 myupdates += myautoadd
2631
2632         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2633
2634         if vcs not in ('cvs', 'svn'):
2635                 # With git, bzr and hg, there's never any keyword expansion, so
2636                 # there's no need to regenerate manifests and all files will be
2637                 # committed in one big commit at the end.
2638                 print()
2639         elif not repo_config.thin_manifest:
2640                 if vcs == 'cvs':
2641                         headerstring = "'\$(Header|Id).*\$'"
2642                 elif vcs == "svn":
2643                         svn_keywords = dict((k.lower(), k) for k in [
2644                                         "Rev",
2645                                         "Revision",
2646                                         "LastChangedRevision",
2647                                         "Date",
2648                                         "LastChangedDate",
2649                                         "Author",
2650                                         "LastChangedBy",
2651                                         "URL",
2652                                         "HeadURL",
2653                                         "Id",
2654                                         "Header",
2655                         ])
2656
2657                 for myfile in myupdates:
2658
2659                         # for CVS, no_expansion contains files that are excluded from expansion
2660                         if vcs == "cvs":
2661                                 if myfile in no_expansion:
2662                                         continue
2663
2664                         # for SVN, expansion contains files that are included in expansion
2665                         elif vcs == "svn":
2666                                 if myfile not in expansion:
2667                                         continue
2668
2669                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2670                                 enabled_keywords = []
2671                                 for k in expansion[myfile]:
2672                                         keyword = svn_keywords.get(k.lower())
2673                                         if keyword is not None:
2674                                                 enabled_keywords.append(keyword)
2675
2676                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2677
2678                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2679                                 portage._shell_quote(myfile))
2680                         if myout[0] == 0:
2681                                 myheaders.append(myfile)
2682
2683                 print("%s have headers that will change." % green(str(len(myheaders))))
2684                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2685
2686         logging.info("myupdates: %s", myupdates)
2687         logging.info("myheaders: %s", myheaders)
2688
2689         if options.ask and userquery('Commit changes?', True) != 'Yes':
2690                 print("* aborting commit.")
2691                 sys.exit(128 + signal.SIGINT)
2692
2693         # Handle the case where committed files have keywords which
2694         # will change and need a priming commit before the Manifest
2695         # can be committed.
2696         if (myupdates or myremoved) and myheaders:
2697                 myfiles = myupdates + myremoved
2698                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2699                 mymsg = os.fdopen(fd, "wb")
2700                 mymsg.write(_unicode_encode(commitmessage))
2701                 mymsg.close()
2702
2703                 print()
2704                 print(green("Using commit message:"))
2705                 print(green("------------------------------------------------------------------------------"))
2706                 print(commitmessage)
2707                 print(green("------------------------------------------------------------------------------"))
2708                 print()
2709
2710                 # Having a leading ./ prefix on file paths can trigger a bug in
2711                 # the cvs server when committing files to multiple directories,
2712                 # so strip the prefix.
2713                 myfiles = [f.lstrip("./") for f in myfiles]
2714
2715                 commit_cmd = [vcs]
2716                 commit_cmd.extend(vcs_global_opts)
2717                 commit_cmd.append("commit")
2718                 commit_cmd.extend(vcs_local_opts)
2719                 commit_cmd.extend(["-F", commitmessagefile])
2720                 commit_cmd.extend(myfiles)
2721
2722                 try:
2723                         if options.pretend:
2724                                 print("(%s)" % (" ".join(commit_cmd),))
2725                         else:
2726                                 retval = spawn(commit_cmd, env=os.environ)
2727                                 if retval != os.EX_OK:
2728                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2729                                                 "error code: %s\n") % (vcs, retval),
2730                                                 level=logging.ERROR, noiselevel=-1)
2731                                         sys.exit(retval)
2732                 finally:
2733                         try:
2734                                 os.unlink(commitmessagefile)
2735                         except OSError:
2736                                 pass
2737
2738         # Setup the GPG commands
2739         def gpgsign(filename):
2740                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2741                 if gpgcmd is None:
2742                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2743                                 " Is make.globals missing?")
2744                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2745                         "PORTAGE_GPG_KEY" not in repoman_settings:
2746                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2747                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2748                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2749                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2750                                         os.path.expanduser("~/.gnupg")
2751                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2752                                         % repoman_settings["PORTAGE_GPG_DIR"])
2753                         else:
2754                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2755                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2756                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2757                                 raise portage.exception.InvalidLocation(
2758                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2759                                         repoman_settings["PORTAGE_GPG_DIR"])
2760                 gpgvars = {"FILE": filename}
2761                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2762                         v = repoman_settings.get(k)
2763                         if v is not None:
2764                                 gpgvars[k] = v
2765                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2766                 if options.pretend:
2767                         print("("+gpgcmd+")")
2768                 else:
2769                         # Encode unicode manually for bug #310789.
2770                         gpgcmd = portage.util.shlex_split(gpgcmd)
2771                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2772                                 # Python 3.1 does not support bytes in Popen args.
2773                                 gpgcmd = [_unicode_encode(arg,
2774                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2775                         rValue = subprocess.call(gpgcmd)
2776                         if rValue == os.EX_OK:
2777                                 os.rename(filename+".asc", filename)
2778                         else:
2779                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2780
2781         # When files are removed and re-added, the cvs server will put /Attic/
2782         # inside the $Header path. This code detects the problem and corrects it
2783         # so that the Manifest will generate correctly. See bug #169500.
2784         # Use binary mode in order to avoid potential character encoding issues.
2785         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2786         attic_str = b'/Attic/'
2787         attic_replace = b'/'
2788         for x in myheaders:
2789                 f = open(_unicode_encode(x,
2790                         encoding=_encodings['fs'], errors='strict'),
2791                         mode='rb')
2792                 mylines = f.readlines()
2793                 f.close()
2794                 modified = False
2795                 for i, line in enumerate(mylines):
2796                         if cvs_header_re.match(line) is not None and \
2797                                 attic_str in line:
2798                                 mylines[i] = line.replace(attic_str, attic_replace)
2799                                 modified = True
2800                 if modified:
2801                         portage.util.write_atomic(x, b''.join(mylines),
2802                                 mode='wb')
2803
2804         if repolevel == 1:
2805                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2806                         "doing the entire repository.\"\n")
2807
2808         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2809
2810                 for x in sorted(vcs_files_to_cps(
2811                         chain(myupdates, myremoved, mymanifests))):
2812                         repoman_settings["O"] = os.path.join(repodir, x)
2813                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2814
2815         signed = False
2816         if sign_manifests:
2817                 signed = True
2818                 try:
2819                         for x in sorted(vcs_files_to_cps(
2820                                 chain(myupdates, myremoved, mymanifests))):
2821                                 repoman_settings["O"] = os.path.join(repodir, x)
2822                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2823                                 if not os.path.exists(manifest_path):
2824                                         continue
2825                                 gpgsign(manifest_path)
2826                 except portage.exception.PortageException as e:
2827                         portage.writemsg("!!! %s\n" % str(e))
2828                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2829                         signed = False
2830
2831         if vcs == 'git':
2832                 # It's not safe to use the git commit -a option since there might
2833                 # be some modified files elsewhere in the working tree that the
2834                 # user doesn't want to commit. Therefore, call git update-index
2835                 # in order to ensure that the index is updated with the latest
2836                 # versions of all new and modified files in the relevant portion
2837                 # of the working tree.
2838                 myfiles = mymanifests + myupdates
2839                 myfiles.sort()
2840                 update_index_cmd = ["git", "update-index"]
2841                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2842                 if options.pretend:
2843                         print("(%s)" % (" ".join(update_index_cmd),))
2844                 else:
2845                         retval = spawn(update_index_cmd, env=os.environ)
2846                         if retval != os.EX_OK:
2847                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2848                                         "error code: %s\n") % (vcs, retval),
2849                                         level=logging.ERROR, noiselevel=-1)
2850                                 sys.exit(retval)
2851
2852         if True:
2853
2854                 myfiles = mymanifests[:]
2855                 # If there are no header (SVN/CVS keywords) changes in
2856                 # the files, this Manifest commit must include the
2857                 # other (yet uncommitted) files.
2858                 if not myheaders:
2859                         myfiles += myupdates
2860                         myfiles += myremoved
2861                 myfiles.sort()
2862
2863                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2864                 mymsg = os.fdopen(fd, "wb")
2865                 # strip the closing parenthesis
2866                 mymsg.write(_unicode_encode(commitmessage[:-1]))
2867                 if signed:
2868                         mymsg.write(_unicode_encode(
2869                                 ", signed Manifest commit with key %s)" % \
2870                                 repoman_settings["PORTAGE_GPG_KEY"]))
2871                 else:
2872                         mymsg.write(b", unsigned Manifest commit)")
2873                 mymsg.close()
2874
2875                 commit_cmd = []
2876                 if options.pretend and vcs is None:
2877                         # substitute a bogus value for pretend output
2878                         commit_cmd.append("cvs")
2879                 else:
2880                         commit_cmd.append(vcs)
2881                 commit_cmd.extend(vcs_global_opts)
2882                 commit_cmd.append("commit")
2883                 commit_cmd.extend(vcs_local_opts)
2884                 if vcs == "hg":
2885                         commit_cmd.extend(["--logfile", commitmessagefile])
2886                         commit_cmd.extend(myfiles)
2887                 else:
2888                         commit_cmd.extend(["-F", commitmessagefile])
2889                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2890
2891                 try:
2892                         if options.pretend:
2893                                 print("(%s)" % (" ".join(commit_cmd),))
2894                         else:
2895                                 retval = spawn(commit_cmd, env=os.environ)
2896                                 if retval != os.EX_OK:
2897
2898                                         if repo_config.sign_commit and vcs == 'git' and \
2899                                                 not git_supports_gpg_sign():
2900                                                 # Inform user that newer git is needed (bug #403323).
2901                                                 logging.error(
2902                                                         "Git >=1.7.9 is required for signed commits!")
2903
2904                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2905                                                 "error code: %s\n") % (vcs, retval),
2906                                                 level=logging.ERROR, noiselevel=-1)
2907                                         sys.exit(retval)
2908                 finally:
2909                         try:
2910                                 os.unlink(commitmessagefile)
2911                         except OSError:
2912                                 pass
2913
2914         print()
2915         if vcs:
2916                 print("Commit complete.")
2917         else:
2918                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2919         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2920 sys.exit(0)
2921