bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2010 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import sys
  22 import tempfile
  23 import time
  24 import platform
  25
  26 try:
  27         from urllib.request import urlopen as urllib_request_urlopen
  28 except ImportError:
  29         from urllib import urlopen as urllib_request_urlopen
  30
  31 from itertools import chain
  32 from stat import S_ISDIR
  33
  34 try:
  35         import portage
  36 except ImportError:
  37         from os import path as osp
  38         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  39         import portage
  40 portage._disable_legacy_globals()
  41 portage.dep._internal_warnings = True
  42
  43 try:
  44         import xml.etree.ElementTree
  45         from xml.parsers.expat import ExpatError
  46 except ImportError:
  47         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  48         from portage.output import EOutput
  49         out = EOutput()
  50         for line in msg:
  51                 out.eerror(line)
  52         sys.exit(1)
  53
  54 from portage import os
  55 from portage import subprocess_getstatusoutput
  56 from portage import _encodings
  57 from portage import _unicode_encode
  58 from portage import StringIO
  59 from repoman.checks import run_checks
  60 from repoman import utilities
  61 from repoman.herdbase import make_herd_base
  62 from _emerge.Package import Package
  63 from _emerge.RootConfig import RootConfig
  64 from _emerge.userquery import userquery
  65 import portage.checksum
  66 import portage.const
  67 from portage import cvstree, normalize_path
  68 from portage import util
  69 from portage.exception import FileNotFound, ParseError, PermissionDenied
  70 from portage.manifest import Manifest
  71 from portage.process import find_binary, spawn
  72 from portage.output import bold, create_color_func, \
  73         green, nocolor, red
  74 from portage.output import ConsoleStyleFile, StyleWriter
  75 from portage.util import cmp_sort_key, writemsg_level
  76 from portage.package.ebuild.digestgen import digestgen
  77 from portage.eapi import eapi_has_slot_deps, \
  78         eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_iuse_defaults, \
  79         eapi_has_required_use, eapi_has_use_dep_defaults
  80
  81 if sys.hexversion >= 0x3000000:
  82         basestring = str
  83
  84 util.initialize_logger()
  85
  86 # 14 is the length of DESCRIPTION=""
  87 max_desc_len = 100
  88 allowed_filename_chars="a-zA-Z0-9._-+:"
  89 disallowed_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]')
  90 pv_toolong_re = re.compile(r'[0-9]{19,}')
  91 bad = create_color_func("BAD")
  92
  93 # A sane umask is needed for files that portage creates.
  94 os.umask(0o22)
  95 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  96 # behave incrementally.
  97 repoman_incrementals = tuple(x for x in \
  98         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
  99 repoman_settings = portage.config(local_config=False)
 100 repoman_settings.lock()
 101
 102 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 103         repoman_settings.get('TERM') == 'dumb' or \
 104         not sys.stdout.isatty():
 105         nocolor()
 106
 107 def warn(txt):
 108         print("repoman: " + txt)
 109
 110 def err(txt):
 111         warn(txt)
 112         sys.exit(1)
 113
 114 def exithandler(signum=None, frame=None):
 115         logging.fatal("Interrupted; exiting...")
 116         sys.exit(1)
 117         os.kill(0, signal.SIGKILL)
 118
 119 signal.signal(signal.SIGINT,exithandler)
 120
 121 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 122         """Repoman needs it's own HelpFormatter for now, because the default ones
 123         murder the help text."""
 124
 125         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 126                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 127
 128         def format_description(self, description):
 129                 return description
 130
 131 class RepomanOptionParser(optparse.OptionParser):
 132         """Add the on_tail function, ruby has it, optionParser should too
 133         """
 134
 135         def __init__(self, *args, **kwargs):
 136                 optparse.OptionParser.__init__(self, *args, **kwargs)
 137                 self.tail = ""
 138
 139         def on_tail(self, description):
 140                 self.tail += description
 141
 142         def format_help(self, formatter=None):
 143                 result = optparse.OptionParser.format_help(self, formatter)
 144                 result += self.tail
 145                 return result
 146
 147
 148 def ParseArgs(argv, qahelp):
 149         """This function uses a customized optionParser to parse command line arguments for repoman
 150         Args:
 151           argv - a sequence of command line arguments
 152                 qahelp - a dict of qa warning to help message
 153         Returns:
 154           (opts, args), just like a call to parser.parse_args()
 155         """
 156
 157         if argv and sys.hexversion < 0x3000000 and not isinstance(argv[0], unicode):
 158                 argv = [portage._unicode_decode(x) for x in argv]
 159
 160         modes = {
 161                 'commit' : 'Run a scan then commit changes',
 162                 'ci' : 'Run a scan then commit changes',
 163                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 164                 'full' : 'Scan directory tree and print all issues (not a summary)',
 165                 'help' : 'Show this screen',
 166                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 167                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 168                 'scan' : 'Scan directory tree for QA issues'
 169         }
 170
 171         mode_keys = list(modes)
 172         mode_keys.sort()
 173
 174         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 175         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 176         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 177         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 178         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 179
 180         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 181                 help='Request a confirmation before commiting')
 182
 183         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 184                 help='specify a commit message on the command line')
 185
 186         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 187                 help='specify a path to a file that contains a commit message')
 188
 189         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 190                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 191
 192         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 193                 help='do not print unnecessary messages')
 194
 195         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 196                 help='Commit with QA violations')
 197
 198         parser.add_option('--vcs', dest='vcs',
 199                 help='Force using specific VCS instead of autodetection')
 200
 201         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 202                 help='be very verbose in output', default=0)
 203
 204         parser.add_option('-V', '--version', dest='version', action='store_true',
 205                 help='show version info')
 206
 207         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 208                 default=False, help='forces the metadata.xml parse check to be carried out')
 209
 210         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 211                 default=False, help='ignore arch-specific failures (where arch != host)')
 212
 213         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 214                 default=False, help='ignore masked packages (not allowed with commit mode)')
 215
 216         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 217                 default=False, help='include dev profiles in dependency checks')
 218
 219         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 220                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 221
 222         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 223                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 224
 225         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 226                 help='specify which mode repoman will run in (default=full)')
 227
 228         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 229
 230         for k in mode_keys:
 231                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 232
 233         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 234
 235         sorted_qa = list(qahelp)
 236         sorted_qa.sort()
 237         for k in sorted_qa:
 238                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 239
 240         opts, args = parser.parse_args(argv[1:])
 241
 242         if opts.mode == 'help':
 243                 parser.print_help(short=False)
 244
 245         for arg in args:
 246                 if arg in modes:
 247                         if not opts.mode:
 248                                 opts.mode = arg
 249                                 break
 250                 else:
 251                         parser.error("invalid mode: %s" % arg)
 252
 253         if not opts.mode:
 254                 opts.mode = 'full'
 255
 256         if opts.mode == 'ci':
 257                 opts.mode = 'commit'  # backwards compat shortcut
 258
 259         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 260                 if opts.ignore_masked:
 261                         parser.error('Commit mode and --ignore-masked are not compatible')
 262                 if opts.without_mask:
 263                         parser.error('Commit mode and --without-mask are not compatible')
 264
 265         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 266         for val in range(opts.verbosity):
 267                 logger = logging.getLogger()
 268                 logger.setLevel(logger.getEffectiveLevel() - 10)
 269
 270         for val in range(opts.quiet):
 271                 logger = logging.getLogger()
 272                 logger.setLevel(logger.getEffectiveLevel() + 10)
 273
 274         return (opts, args)
 275
 276 qahelp={
 277         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 278         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 279         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 280         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 281         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 282         "changelog.missing":"Missing ChangeLog files",
 283         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 284         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 285         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 286         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
 287         "file.size":"Files in the files directory must be under 20 KiB",
 288         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 289         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 290         "file.UTF8":"File is not UTF8 compliant",
 291         "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf",
 292         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 293         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 294         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 295         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 296         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 297         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 298         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 299         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 300         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 301         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 302         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 303         "EAPI.definition":"EAPI is defined after an inherit call (must be defined before)",
 304         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 305         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 306         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 307         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 308         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 309         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 310         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 311         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 312         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 313         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 314         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 315         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 316         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 317         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 318         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 319         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 320         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 321         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 322         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 323         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 324         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 325         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 326         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 327         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 328         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 329         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 330         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 331         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 332         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 333         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 334         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 335         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 336         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 337         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 338         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 339         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 340         "variable.readonly":"Assigning a readonly variable",
 341         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 342         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 343         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 344         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 345         "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
 346         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 347         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 348         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 349         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 350         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 351         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 352         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 353         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 354         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 355         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 356         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 357         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 358         "ebuild.badheader":"This ebuild has a malformed header",
 359         "eprefixify.defined":"The ebuild uses eprefixify, but does not inherit the prefix eclass",
 360         "manifest.bad":"Manifest has missing or incorrect digests",
 361         "metadata.missing":"Missing metadata.xml files",
 362         "metadata.bad":"Bad metadata.xml files",
 363         "metadata.warning":"Warnings in metadata.xml files",
 364         "virtual.versioned":"PROVIDE contains virtuals with versions",
 365         "virtual.exists":"PROVIDE contains existing package names",
 366         "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default",
 367         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 368         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 369 }
 370
 371 qacats = list(qahelp)
 372 qacats.sort()
 373
 374 qawarnings = set((
 375 "changelog.missing",
 376 "changelog.notadded",
 377 "digest.assumed",
 378 "digest.unused",
 379 "ebuild.notadded",
 380 "ebuild.nostable",
 381 "ebuild.allmasked",
 382 "ebuild.nesteddie",
 383 "desktop.invalid",
 384 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 385 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 386 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 387 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 388 "DESCRIPTION.toolong",
 389 "EAPI.deprecated",
 390 "HOMEPAGE.virtual",
 391 "LICENSE.virtual",
 392 "KEYWORDS.dropped",
 393 "KEYWORDS.stupid",
 394 "KEYWORDS.missing",
 395 "IUSE.undefined",
 396 "PDEPEND.suspect",
 397 "RDEPEND.implicit",
 398 "RDEPEND.suspect",
 399 "RESTRICT.invalid",
 400 "SRC_URI.mirror",
 401 "ebuild.minorsyn",
 402 "ebuild.badheader",
 403 "ebuild.patches",
 404 "file.size",
 405 "inherit.autotools",
 406 "inherit.deprecated",
 407 "java.eclassesnotused",
 408 "wxwidgets.eclassnotused",
 409 "metadata.warning",
 410 "virtual.versioned",
 411 "virtual.exists",
 412 "virtual.unavailable",
 413 "usage.obsolete",
 414 "upstream.workaround",
 415 "LIVEVCS.stable",
 416 "LIVEVCS.unmasked",
 417 ))
 418
 419 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 420
 421 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 422 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 423 allvars.update(Package.metadata_keys)
 424 allvars = sorted(allvars)
 425 commitmessage=None
 426 for x in missingvars:
 427         x += ".missing"
 428         if x not in qacats:
 429                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 430                 qacats.append(x)
 431                 qawarnings.add(x)
 432
 433 valid_restrict = frozenset(["binchecks", "bindist",
 434         "fetch", "installsources", "mirror",
 435         "primaryuri", "strip", "test", "userpriv"])
 436
 437 live_eclasses = frozenset([
 438         "bzr",
 439         "cvs",
 440         "darcs",
 441         "git",
 442         "mercurial",
 443         "subversion",
 444         "tla",
 445 ])
 446
 447 suspect_rdepend = frozenset([
 448         "app-arch/cabextract",
 449         "app-arch/rpm2targz",
 450         "app-doc/doxygen",
 451         "dev-lang/nasm",
 452         "dev-lang/swig",
 453         "dev-lang/yasm",
 454         "dev-perl/extutils-pkgconfig",
 455         "dev-util/byacc",
 456         "dev-util/cmake",
 457         "dev-util/ftjam",
 458         "dev-util/gperf",
 459         "dev-util/gtk-doc",
 460         "dev-util/gtk-doc-am",
 461         "dev-util/intltool",
 462         "dev-util/jam",
 463         "dev-util/pkgconfig",
 464         "dev-util/scons",
 465         "dev-util/unifdef",
 466         "dev-util/yacc",
 467         "media-gfx/ebdftopcf",
 468         "sys-apps/help2man",
 469         "sys-devel/autoconf",
 470         "sys-devel/automake",
 471         "sys-devel/bin86",
 472         "sys-devel/bison",
 473         "sys-devel/dev86",
 474         "sys-devel/flex",
 475         "sys-devel/m4",
 476         "sys-devel/pmake",
 477         "virtual/linux-sources",
 478         "x11-misc/bdftopcf",
 479         "x11-misc/imake",
 480 ])
 481
 482 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 483 # force refetch if the local copy creation time is older than this
 484 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 485
 486 # file.executable
 487 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 488
 489 options, arguments = ParseArgs(sys.argv, qahelp)
 490
 491 if options.version:
 492         print("Portage", portage.VERSION)
 493         sys.exit(0)
 494
 495 # Set this to False when an extraordinary issue (generally
 496 # something other than a QA issue) makes it impossible to
 497 # commit (like if Manifest generation fails).
 498 can_force = True
 499
 500 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 501 if portdir is None:
 502         sys.exit(1)
 503
 504 myreporoot = os.path.basename(portdir_overlay)
 505 myreporoot += mydir[len(portdir_overlay):]
 506
 507 if options.vcs:
 508         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 509                 vcs = options.vcs
 510         else:
 511                 vcs = None
 512 else:
 513         vcses = utilities.FindVCS()
 514         if len(vcses) > 1:
 515                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 516                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 517                 sys.exit(1)
 518         elif vcses:
 519                 vcs = vcses[0]
 520         else:
 521                 vcs = None
 522
 523 # Note: We don't use ChangeLogs in distributed SCMs.
 524 # It will be generated on server side from scm log,
 525 # before package moves to the rsync server.
 526 # This is needed because we try to avoid merge collisions.
 527 check_changelog = vcs in ('cvs', 'svn')
 528
 529 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 530 vcs_preserves_mtime = vcs not in ('git',)
 531
 532 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 533 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 534 if vcs_global_opts is None:
 535         if vcs in ('cvs', 'svn'):
 536                 vcs_global_opts = "-q"
 537         else:
 538                 vcs_global_opts = ""
 539 vcs_global_opts = vcs_global_opts.split()
 540
 541 if vcs == "cvs" and \
 542         "commit" == options.mode and \
 543         "RMD160" not in portage.checksum.hashorigin_map:
 544         from portage.util import grablines
 545         repo_lines = grablines("./CVS/Repository")
 546         if repo_lines and \
 547                 "gentoo-x86" == repo_lines[0].strip().split(os.path.sep)[0]:
 548                 msg = "Please install " \
 549                 "pycrypto or enable python's ssl USE flag in order " \
 550                 "to enable RMD160 hash support. See bug #198398 for " \
 551                 "more information."
 552                 prefix = bad(" * ")
 553                 from textwrap import wrap
 554                 for line in wrap(msg, 70):
 555                         print(prefix + line)
 556                 sys.exit(1)
 557         del repo_lines
 558
 559 if options.mode == 'commit' and not options.pretend and not vcs:
 560         logging.info("Not in a version controlled repository; enabling pretend mode.")
 561         options.pretend = True
 562
 563 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 564 repoman_settings = portage.config(local_config=False)
 565 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 566         (repoman_settings.get('PORTDIR_OVERLAY', ''), portdir_overlay)
 567 # We have to call the config constructor again so
 568 # that config.repositories is initialized correctly.
 569 repoman_settings = portage.config(local_config=False, env=dict(os.environ,
 570         PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 571
 572 root = '/'
 573 trees = {
 574         root : {'porttree' : portage.portagetree(root, settings=repoman_settings)}
 575 }
 576 portdb = trees[root]['porttree'].dbapi
 577
 578 # Constrain dependency resolution to the master(s)
 579 # that are specified in layout.conf.
 580 portdir_overlay = os.path.realpath(portdir_overlay)
 581 repo_info = portdb._repo_info[portdir_overlay]
 582 portdb.porttrees = list(repo_info.eclass_db.porttrees)
 583 portdir = portdb.porttrees[0]
 584
 585 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 586 # profile-specific config constructor calls.
 587 env = os.environ.copy()
 588 env['PORTDIR'] = portdir
 589 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 590
 591 logging.info('Setting paths:')
 592 logging.info('PORTDIR = "' + portdir + '"')
 593 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 594
 595 # It's confusing if these warnings are displayed without the user
 596 # being told which profile they come from, so disable them.
 597 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 598
 599 categories = []
 600 for path in set([portdir, portdir_overlay]):
 601         categories.extend(portage.util.grabfile(
 602                 os.path.join(path, 'profiles', 'categories')))
 603 repoman_settings.categories = tuple(sorted(
 604         portage.util.stack_lists([categories], incremental=1)))
 605 del categories
 606
 607 portdb.settings = repoman_settings
 608 root_config = RootConfig(repoman_settings, trees[root], None)
 609 # We really only need to cache the metadata that's necessary for visibility
 610 # filtering. Anything else can be discarded to reduce memory consumption.
 611 portdb._aux_cache_keys.clear()
 612 portdb._aux_cache_keys.update(["EAPI", "KEYWORDS", "SLOT"])
 613
 614 reposplit = myreporoot.split(os.path.sep)
 615 repolevel = len(reposplit)
 616
 617 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 618 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 619 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 620 if options.mode == 'commit' and repolevel not in [1,2,3]:
 621         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 622         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 623         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 624         print(red("***"))
 625         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 626
 627 startdir = normalize_path(mydir)
 628 repodir = startdir
 629 for x in range(0, repolevel - 1):
 630         repodir = os.path.dirname(repodir)
 631 repodir = os.path.realpath(repodir)
 632
 633 def caterror(mycat):
 634         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 635
 636 class ProfileDesc(object):
 637         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 638         def __init__(self, arch, status, sub_path, tree_path):
 639                 self.arch = arch
 640                 self.status = status
 641                 if sub_path:
 642                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 643                 self.sub_path = sub_path
 644                 self.tree_path = tree_path
 645                 if tree_path:
 646                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 647                 else:
 648                         self.abs_path = tree_path
 649
 650         def __str__(self):
 651                 if self.sub_path:
 652                         return self.sub_path
 653                 return 'empty profile'
 654
 655 profile_list = []
 656 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 657
 658 # get lists of valid keywords, licenses, and use
 659 kwlist = set()
 660 liclist = set()
 661 uselist = set()
 662 global_pmasklines = []
 663
 664 for path in portdb.porttrees:
 665         try:
 666                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 667         except OSError:
 668                 pass
 669         kwlist.update(portage.grabfile(os.path.join(path,
 670                 "profiles", "arch.list")))
 671
 672         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 673         for x in use_desc:
 674                 x = x.split()
 675                 if x:
 676                         uselist.add(x[0])
 677
 678         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 679         try:
 680                 expand_list = os.listdir(expand_desc_dir)
 681         except OSError:
 682                 pass
 683         else:
 684                 for fn in expand_list:
 685                         if not fn[-5:] == '.desc':
 686                                 continue
 687                         use_prefix = fn[:-5].lower() + '_'
 688                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 689                                 x = x.split()
 690                                 if x:
 691                                         uselist.add(use_prefix + x[0])
 692
 693         global_pmasklines.append(portage.util.grabfile_package(
 694                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 695
 696         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 697         try:
 698                 desc_file = codecs.open(_unicode_encode(desc_path,
 699                         encoding=_encodings['fs'], errors='strict'),
 700                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 701         except EnvironmentError:
 702                 pass
 703         else:
 704                 for i, x in enumerate(desc_file):
 705                         if x[0] == "#":
 706                                 continue
 707                         arch = x.split()
 708                         if len(arch) == 0:
 709                                 continue
 710                         if len(arch) != 3:
 711                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 712                                         desc_path + " line %d" % (i+1, ))
 713                         elif arch[0] not in kwlist:
 714                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 715                                         desc_path + " line %d" % (i+1, ))
 716                         elif arch[2] not in valid_profile_types:
 717                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 718                                         desc_path + " line %d" % (i+1, ))
 719                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], portdir)
 720                         if not os.path.isdir(profile_desc.abs_path):
 721                                 logging.error(
 722                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 723                                         arch[2], arch[1], arch[0], desc_path, i+1)
 724                                 continue
 725                         profile_list.append(profile_desc)
 726                 desc_file.close()
 727
 728 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 729 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 730
 731 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 732 global_pmaskdict = {}
 733 for x in global_pmasklines:
 734         global_pmaskdict.setdefault(x.cp, []).append(x)
 735 del global_pmasklines
 736
 737 def has_global_mask(pkg):
 738         mask_atoms = global_pmaskdict.get(pkg.cp)
 739         if mask_atoms:
 740                 pkg_list = [pkg]
 741                 for x in mask_atoms:
 742                         if portage.dep.match_from_list(x, pkg_list):
 743                                 return x
 744         return None
 745
 746 # Ensure that profile sub_path attributes are unique. Process in reverse order
 747 # so that profiles with duplicate sub_path from overlays will override
 748 # profiles with the same sub_path from parent repos.
 749 profiles = {}
 750 profile_list.reverse()
 751 profile_sub_paths = set()
 752 for prof in profile_list:
 753         if prof.sub_path in profile_sub_paths:
 754                 continue
 755         profile_sub_paths.add(prof.sub_path)
 756         profiles.setdefault(prof.arch, []).append(prof)
 757
 758 # Use an empty profile for checking dependencies of
 759 # packages that have empty KEYWORDS.
 760 prof = ProfileDesc('**', 'stable', '', '')
 761 profiles.setdefault(prof.arch, []).append(prof)
 762
 763 for x in repoman_settings.archlist():
 764         if x[0] == "~":
 765                 continue
 766         if x not in profiles:
 767                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 768                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 769                 print(red("up with the "+x+" team."))
 770                 print()
 771
 772 if not liclist:
 773         logging.fatal("Couldn't find licenses?")
 774         sys.exit(1)
 775
 776 if not kwlist:
 777         logging.fatal("Couldn't read KEYWORDS from arch.list")
 778         sys.exit(1)
 779
 780 if not uselist:
 781         logging.fatal("Couldn't find use.desc?")
 782         sys.exit(1)
 783
 784 scanlist=[]
 785 if repolevel==2:
 786         #we are inside a category directory
 787         catdir=reposplit[-1]
 788         if catdir not in repoman_settings.categories:
 789                 caterror(catdir)
 790         mydirlist=os.listdir(startdir)
 791         for x in mydirlist:
 792                 if x == "CVS" or x.startswith("."):
 793                         continue
 794                 if os.path.isdir(startdir+"/"+x):
 795                         scanlist.append(catdir+"/"+x)
 796         repo_subdir = catdir + os.sep
 797 elif repolevel==1:
 798         for x in repoman_settings.categories:
 799                 if not os.path.isdir(startdir+"/"+x):
 800                         continue
 801                 for y in os.listdir(startdir+"/"+x):
 802                         if y == "CVS" or y.startswith("."):
 803                                 continue
 804                         if os.path.isdir(startdir+"/"+x+"/"+y):
 805                                 scanlist.append(x+"/"+y)
 806         repo_subdir = ""
 807 elif repolevel==3:
 808         catdir = reposplit[-2]
 809         if catdir not in repoman_settings.categories:
 810                 caterror(catdir)
 811         scanlist.append(catdir+"/"+reposplit[-1])
 812         repo_subdir = scanlist[-1] + os.sep
 813 else:
 814         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 815                 ' from the current working directory'
 816         logging.critical(msg)
 817         sys.exit(1)
 818
 819 repo_subdir_len = len(repo_subdir)
 820 scanlist.sort()
 821
 822 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 823
 824 def dev_keywords(profiles):
 825         """
 826         Create a set of KEYWORDS values that exist in 'dev'
 827         profiles. These are used
 828         to trigger a message notifying the user when they might
 829         want to add the --include-dev option.
 830         """
 831         type_arch_map = {}
 832         for arch, arch_profiles in profiles.items():
 833                 for prof in arch_profiles:
 834                         arch_set = type_arch_map.get(prof.status)
 835                         if arch_set is None:
 836                                 arch_set = set()
 837                                 type_arch_map[prof.status] = arch_set
 838                         arch_set.add(arch)
 839
 840         dev_keywords = type_arch_map.get('dev', set())
 841         dev_keywords.update(['~' + arch for arch in dev_keywords])
 842         return frozenset(dev_keywords)
 843
 844 dev_keywords = dev_keywords(profiles)
 845
 846 stats={}
 847 fails={}
 848
 849 # provided by the desktop-file-utils package
 850 desktop_file_validate = find_binary("desktop-file-validate")
 851 desktop_pattern = re.compile(r'.*\.desktop$')
 852
 853 for x in qacats:
 854         stats[x]=0
 855         fails[x]=[]
 856
 857 xmllint_capable = False
 858 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
 859
 860 def parsedate(s):
 861         """Parse a RFC 822 date and time string.
 862         This is required for python3 compatibility, since the
 863         rfc822.parsedate() function is not available."""
 864
 865         s_split = []
 866         for x in s.upper().split():
 867                 for y in x.split(','):
 868                         if y:
 869                                 s_split.append(y)
 870
 871         if len(s_split) != 6:
 872                 return None
 873
 874         # %a, %d %b %Y %H:%M:%S %Z
 875         a, d, b, Y, H_M_S, Z = s_split
 876
 877         # Convert month to integer, since strptime %w is locale-dependent.
 878         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
 879                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
 880         m = month_map.get(b)
 881         if m is None:
 882                 return None
 883         m = str(m).rjust(2, '0')
 884
 885         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
 886
 887 def fetch_metadata_dtd():
 888         """
 889         Fetch metadata.dtd if it doesn't exist or the ctime is older than
 890         metadata_dtd_ctime_interval.
 891         @rtype: bool
 892         @returns: True if successful, otherwise False
 893         """
 894
 895         must_fetch = True
 896         metadata_dtd_st = None
 897         current_time = int(time.time())
 898         try:
 899                 metadata_dtd_st = os.stat(metadata_dtd)
 900         except EnvironmentError as e:
 901                 if e.errno not in (errno.ENOENT, errno.ESTALE):
 902                         raise
 903                 del e
 904         else:
 905                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
 906                 if abs(current_time - metadata_dtd_st.st_ctime) \
 907                         < metadata_dtd_ctime_interval:
 908                         must_fetch = False
 909
 910         if must_fetch:
 911                 print()
 912                 print(green("***") + " the local copy of metadata.dtd " + \
 913                         "needs to be refetched, doing that now")
 914                 print()
 915                 try:
 916                         url_f = urllib_request_urlopen(metadata_dtd_uri)
 917                         msg_info = url_f.info()
 918                         last_modified = msg_info.get('last-modified')
 919                         if last_modified is not None:
 920                                 last_modified = parsedate(last_modified)
 921                                 if last_modified is not None:
 922                                         last_modified = calendar.timegm(last_modified)
 923
 924                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
 925                         try:
 926                                 local_f = open(metadata_dtd_tmp, mode='wb')
 927                                 local_f.write(url_f.read())
 928                                 local_f.close()
 929                                 if last_modified is not None:
 930                                         try:
 931                                                 os.utime(metadata_dtd_tmp,
 932                                                         (int(last_modified), int(last_modified)))
 933                                         except OSError:
 934                                                 # This fails on some odd non-unix-like filesystems.
 935                                                 # We don't really need the mtime to be preserved
 936                                                 # anyway here (currently we use ctime to trigger
 937                                                 # fetch), so just ignore it.
 938                                                 pass
 939                                 os.rename(metadata_dtd_tmp, metadata_dtd)
 940                         finally:
 941                                 try:
 942                                         os.unlink(metadata_dtd_tmp)
 943                                 except OSError:
 944                                         pass
 945
 946                         url_f.close()
 947
 948                 except EnvironmentError as e:
 949                         print()
 950                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
 951                         print(red("!!!")+" exception '%s' though." % (e,))
 952                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
 953                         return False
 954
 955         return True
 956
 957 if options.mode == "manifest":
 958         pass
 959 elif not find_binary('xmllint'):
 960         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
 961         if options.xml_parse or repolevel==3:
 962                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
 963                 sys.exit(1)
 964 else:
 965         if not fetch_metadata_dtd():
 966                 sys.exit(1)
 967         #this can be problematic if xmllint changes their output
 968         xmllint_capable=True
 969
 970 if options.mode == 'commit' and vcs:
 971         utilities.detect_vcs_conflicts(options, vcs)
 972
 973 if options.mode == "manifest":
 974         pass
 975 elif options.pretend:
 976         print(green("\nRepoMan does a once-over of the neighborhood..."))
 977 else:
 978         print(green("\nRepoMan scours the neighborhood..."))
 979
 980 new_ebuilds = set()
 981 modified_ebuilds = set()
 982 modified_changelogs = set()
 983 mychanged = []
 984 mynew = []
 985 myremoved = []
 986
 987 if vcs == "cvs":
 988         mycvstree = cvstree.getentries("./", recursive=1)
 989         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
 990         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
 991 if vcs == "svn":
 992         svnstatus = os.popen("svn status").readlines()
 993         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
 994         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
 995 elif vcs == "git":
 996         mychanged = os.popen("git diff-index --name-only --relative --diff-filter=M HEAD").readlines()
 997         mychanged = ["./" + elem[:-1] for elem in mychanged]
 998
 999         mynew = os.popen("git diff-index --name-only --relative --diff-filter=A HEAD").readlines()
1000         mynew = ["./" + elem[:-1] for elem in mynew]
1001 elif vcs == "bzr":
1002         bzrstatus = os.popen("bzr status -S .").readlines()
1003         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1004         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1005 elif vcs == "hg":
1006         mychanged = os.popen("hg status --no-status --modified .").readlines()
1007         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1008         mynew = os.popen("hg status --no-status --added .").readlines()
1009         mynew = ["./" + elem.rstrip() for elem in mynew]
1010
1011 if vcs:
1012         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1013         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1014         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1015                 if os.path.basename(x) == "ChangeLog")
1016
1017 have_pmasked = False
1018 have_dev_keywords = False
1019 dofail = 0
1020 arch_caches={}
1021 arch_xmatch_caches = {}
1022 shared_xmatch_caches = {"cp-list":{}}
1023
1024 # Disable the "ebuild.notadded" check when not in commit mode and
1025 # running `svn status` in every package dir will be too expensive.
1026
1027 check_ebuild_notadded = not \
1028         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1029
1030 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1031 thirdpartymirrors = []
1032 for v in repoman_settings.thirdpartymirrors().values():
1033         thirdpartymirrors.extend(v)
1034
1035 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1036         """
1037         Implements doctype() as required to avoid deprecation warnings with
1038         >=python-2.7.
1039         """
1040         def doctype(self, name, pubid, system):
1041                 pass
1042
1043 try:
1044         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1045 except (EnvironmentError, ParseError, PermissionDenied) as e:
1046         err(str(e))
1047 except FileNotFound:
1048         # TODO: Download as we do for metadata.dtd, but add a way to
1049         # disable for non-gentoo repoman users who may not have herds.
1050         herd_base = None
1051
1052 for x in scanlist:
1053         #ebuilds and digests added to cvs respectively.
1054         logging.info("checking package %s" % x)
1055         eadded=[]
1056         catdir,pkgdir=x.split("/")
1057         checkdir=repodir+"/"+x
1058         checkdir_relative = ""
1059         if repolevel < 3:
1060                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1061         if repolevel < 2:
1062                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1063         checkdir_relative = os.path.join(".", checkdir_relative)
1064         generated_manifest = False
1065
1066         if options.mode == "manifest" or \
1067           (options.mode != 'manifest-check' and \
1068           'digest' in repoman_settings.features) or \
1069           options.mode in ('commit', 'fix') and not options.pretend:
1070                 auto_assumed = set()
1071                 fetchlist_dict = portage.FetchlistDict(checkdir,
1072                         repoman_settings, portdb)
1073                 if options.mode == 'manifest' and options.force:
1074                         portage._doebuild_manifest_exempt_depend += 1
1075                         try:
1076                                 distdir = repoman_settings['DISTDIR']
1077                                 mf = portage.manifest.Manifest(checkdir, distdir,
1078                                         fetchlist_dict=fetchlist_dict)
1079                                 mf.create(requiredDistfiles=None,
1080                                         assumeDistHashesAlways=True)
1081                                 for distfiles in fetchlist_dict.values():
1082                                         for distfile in distfiles:
1083                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1084                                                         mf.fhashdict['DIST'].pop(distfile, None)
1085                                                 else:
1086                                                         auto_assumed.add(distfile)
1087                                 mf.write()
1088                         finally:
1089                                 portage._doebuild_manifest_exempt_depend -= 1
1090
1091                 repoman_settings["O"] = checkdir
1092                 generated_manifest = digestgen(
1093                         mysettings=repoman_settings, myportdb=portdb)
1094
1095                 if not generated_manifest:
1096                         print("Unable to generate manifest.")
1097                         dofail = 1
1098
1099                 if options.mode == "manifest":
1100                         if not dofail and options.force and auto_assumed and \
1101                                 'assume-digests' in repoman_settings.features:
1102                                 # Show which digests were assumed despite the --force option
1103                                 # being given. This output will already have been shown by
1104                                 # digestgen() if assume-digests is not enabled, so only show
1105                                 # it here if assume-digests is enabled.
1106                                 pkgs = list(fetchlist_dict)
1107                                 pkgs.sort()
1108                                 portage.writemsg_stdout("  digest.assumed" + \
1109                                         portage.output.colorize("WARN",
1110                                         str(len(auto_assumed)).rjust(18)) + "\n")
1111                                 for cpv in pkgs:
1112                                         fetchmap = fetchlist_dict[cpv]
1113                                         pf = portage.catsplit(cpv)[1]
1114                                         for distfile in sorted(fetchmap):
1115                                                 if distfile in auto_assumed:
1116                                                         portage.writemsg_stdout(
1117                                                                 "   %s::%s\n" % (pf, distfile))
1118                         continue
1119                 elif dofail:
1120                         sys.exit(1)
1121
1122         if not generated_manifest:
1123                 repoman_settings['O'] = checkdir
1124                 repoman_settings['PORTAGE_QUIET'] = '1'
1125                 if not portage.digestcheck([], repoman_settings, strict=1):
1126                         stats["manifest.bad"] += 1
1127                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1128                 repoman_settings.pop('PORTAGE_QUIET', None)
1129
1130         if options.mode == 'manifest-check':
1131                 continue
1132
1133         checkdirlist=os.listdir(checkdir)
1134         ebuildlist=[]
1135         pkgs = {}
1136         allvalid = True
1137         for y in checkdirlist:
1138                 if (y in no_exec or y.endswith(".ebuild")) and \
1139                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1140                                 stats["file.executable"] += 1
1141                                 fails["file.executable"].append(os.path.join(checkdir, y))
1142                 if y.endswith(".ebuild"):
1143                         pf = y[:-7]
1144                         ebuildlist.append(pf)
1145                         cpv = "%s/%s" % (catdir, pf)
1146                         try:
1147                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1148                         except KeyError:
1149                                 allvalid = False
1150                                 stats["ebuild.syntax"] += 1
1151                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1152                                 continue
1153                         except IOError:
1154                                 allvalid = False
1155                                 stats["ebuild.output"] += 1
1156                                 fails["ebuild.output"].append(os.path.join(x, y))
1157                                 continue
1158                         if not portage.eapi_is_supported(myaux["EAPI"]):
1159                                 allvalid = False
1160                                 stats["EAPI.unsupported"] += 1
1161                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1162                                 continue
1163                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1164                                 root_config=root_config)
1165
1166         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1167         pkgsplits = {}
1168         for i in range(len(ebuildlist)):
1169                 ebuild_split = portage.pkgsplit(ebuildlist[i])
1170                 pkgsplits[ebuild_split] = ebuildlist[i]
1171                 ebuildlist[i] = ebuild_split
1172         ebuildlist.sort(key=cmp_sort_key(portage.pkgcmp))
1173         for i in range(len(ebuildlist)):
1174                 ebuildlist[i] = pkgsplits[ebuildlist[i]]
1175         del pkgsplits
1176
1177         slot_keywords = {}
1178
1179         if len(pkgs) != len(ebuildlist):
1180                 # If we can't access all the metadata then it's totally unsafe to
1181                 # commit since there's no way to generate a correct Manifest.
1182                 # Do not try to do any more QA checks on this package since missing
1183                 # metadata leads to false positives for several checks, and false
1184                 # positives confuse users.
1185                 can_force = False
1186                 continue
1187
1188         for y in checkdirlist:
1189                 m = disallowed_filename_chars_re.search(y.strip(os.sep))
1190                 if m is not None:
1191                         stats["file.name"] += 1
1192                         fails["file.name"].append("%s/%s: char '%s'" % \
1193                                 (checkdir, y, m.group(0)))
1194
1195                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1196                         continue
1197                 try:
1198                         line = 1
1199                         for l in codecs.open(_unicode_encode(os.path.join(checkdir, y),
1200                                 encoding=_encodings['fs'], errors='strict'),
1201                                 mode='r', encoding=_encodings['repo.content']):
1202                                 line +=1
1203                 except UnicodeDecodeError as ue:
1204                         stats["file.UTF8"] += 1
1205                         s = ue.object[:ue.start]
1206                         l2 = s.count("\n")
1207                         line += l2
1208                         if l2 != 0:
1209                                 s = s[s.rfind("\n") + 1:]
1210                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1211
1212         if vcs in ("git", "hg") and check_ebuild_notadded:
1213                 if vcs == "git":
1214                         myf = os.popen("git ls-files --others %s" % \
1215                                 (portage._shell_quote(checkdir_relative),))
1216                 if vcs == "hg":
1217                         myf = os.popen("hg status --no-status --unknown %s" % \
1218                                 (portage._shell_quote(checkdir_relative),))
1219                 for l in myf:
1220                         if l[:-1][-7:] == ".ebuild":
1221                                 stats["ebuild.notadded"] += 1
1222                                 fails["ebuild.notadded"].append(
1223                                         os.path.join(x, os.path.basename(l[:-1])))
1224                 myf.close()
1225
1226         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1227                 try:
1228                         if vcs == "cvs":
1229                                 myf=open(checkdir+"/CVS/Entries","r")
1230                         if vcs == "svn":
1231                                 myf = os.popen("svn status --depth=files --verbose " + checkdir)
1232                         if vcs == "bzr":
1233                                 myf = os.popen("bzr ls -v --kind=file " + checkdir)
1234                         myl = myf.readlines()
1235                         myf.close()
1236                         for l in myl:
1237                                 if vcs == "cvs":
1238                                         if l[0]!="/":
1239                                                 continue
1240                                         splitl=l[1:].split("/")
1241                                         if not len(splitl):
1242                                                 continue
1243                                         if splitl[0][-7:]==".ebuild":
1244                                                 eadded.append(splitl[0][:-7])
1245                                 if vcs == "svn":
1246                                         if l[:1] == "?":
1247                                                 continue
1248                                         if l[:7] == '      >':
1249                                                 # tree conflict, new in subversion 1.6
1250                                                 continue
1251                                         l = l.split()[-1]
1252                                         if l[-7:] == ".ebuild":
1253                                                 eadded.append(os.path.basename(l[:-7]))
1254                                 if vcs == "bzr":
1255                                         if l[1:2] == "?":
1256                                                 continue
1257                                         l = l.split()[-1]
1258                                         if l[-7:] == ".ebuild":
1259                                                 eadded.append(os.path.basename(l[:-7]))
1260                         if vcs == "svn":
1261                                 myf = os.popen("svn status " + checkdir)
1262                                 myl=myf.readlines()
1263                                 myf.close()
1264                                 for l in myl:
1265                                         if l[0] == "A":
1266                                                 l = l.rstrip().split(' ')[-1]
1267                                                 if l[-7:] == ".ebuild":
1268                                                         eadded.append(os.path.basename(l[:-7]))
1269                 except IOError:
1270                         if vcs == "cvs":
1271                                 stats["CVS/Entries.IO_error"] += 1
1272                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1273                         else:
1274                                 raise
1275                         continue
1276
1277         mf = Manifest(checkdir, repoman_settings["DISTDIR"])
1278         mydigests=mf.getTypeDigests("DIST")
1279
1280         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1281         myfiles_all = []
1282         src_uri_error = False
1283         for mykey in fetchlist_dict:
1284                 try:
1285                         myfiles_all.extend(fetchlist_dict[mykey])
1286                 except portage.exception.InvalidDependString as e:
1287                         src_uri_error = True
1288                         try:
1289                                 portdb.aux_get(mykey, ["SRC_URI"])
1290                         except KeyError:
1291                                 # This will be reported as an "ebuild.syntax" error.
1292                                 pass
1293                         else:
1294                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1295                                 fails["SRC_URI.syntax"].append(
1296                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1297         del fetchlist_dict
1298         if not src_uri_error:
1299                 # This test can produce false positives if SRC_URI could not
1300                 # be parsed for one or more ebuilds. There's no point in
1301                 # producing a false error here since the root cause will
1302                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1303                 # or "ebuild.sytax".
1304                 myfiles_all = set(myfiles_all)
1305                 for entry in mydigests:
1306                         if entry not in myfiles_all:
1307                                 stats["digest.unused"] += 1
1308                                 fails["digest.unused"].append(checkdir+"::"+entry)
1309                 for entry in myfiles_all:
1310                         if entry not in mydigests:
1311                                 stats["digest.missing"] += 1
1312                                 fails["digest.missing"].append(checkdir+"::"+entry)
1313         del myfiles_all
1314
1315         if os.path.exists(checkdir+"/files"):
1316                 filesdirlist=os.listdir(checkdir+"/files")
1317
1318                 # recurse through files directory
1319                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1320                 while filesdirlist:
1321                         y = filesdirlist.pop(0)
1322                         relative_path = os.path.join(x, "files", y)
1323                         full_path = os.path.join(repodir, relative_path)
1324                         try:
1325                                 mystat = os.stat(full_path)
1326                         except OSError as oe:
1327                                 if oe.errno == 2:
1328                                         # don't worry about it.  it likely was removed via fix above.
1329                                         continue
1330                                 else:
1331                                         raise oe
1332                         if S_ISDIR(mystat.st_mode):
1333                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1334                                 if y == "CVS" or y == ".svn":
1335                                         continue
1336                                 for z in os.listdir(checkdir+"/files/"+y):
1337                                         if z == "CVS" or z == ".svn":
1338                                                 continue
1339                                         filesdirlist.append(y+"/"+z)
1340                         # Current policy is no files over 20 KiB, these are the checks. File size between
1341                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1342                         elif mystat.st_size > 61440:
1343                                 stats["file.size.fatal"] += 1
1344                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1345                         elif mystat.st_size > 20480:
1346                                 stats["file.size"] += 1
1347                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1348
1349                         m = disallowed_filename_chars_re.search(
1350                                 os.path.basename(y.rstrip(os.sep)))
1351                         if m is not None:
1352                                 stats["file.name"] += 1
1353                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1354                                         (checkdir, y, m.group(0)))
1355
1356                         if desktop_file_validate and desktop_pattern.match(y):
1357                                 status, cmd_output = subprocess_getstatusoutput(
1358                                         "'%s' '%s'" % (desktop_file_validate, full_path))
1359                                 if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK:
1360                                         # Note: in the future we may want to grab the
1361                                         # warnings in addition to the errors. We're
1362                                         # just doing errors now since we don't want
1363                                         # to generate too much noise at first.
1364                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1365                                         for line in cmd_output.splitlines():
1366                                                 error_match = error_re.match(line)
1367                                                 if error_match is None:
1368                                                         continue
1369                                                 stats["desktop.invalid"] += 1
1370                                                 fails["desktop.invalid"].append(
1371                                                         relative_path + ': %s' % error_match.group(1))
1372
1373         del mydigests
1374
1375         if check_changelog and "ChangeLog" not in checkdirlist:
1376                 stats["changelog.missing"]+=1
1377                 fails["changelog.missing"].append(x+"/ChangeLog")
1378
1379         musedict = {}
1380         #metadata.xml file check
1381         if "metadata.xml" not in checkdirlist:
1382                 stats["metadata.missing"]+=1
1383                 fails["metadata.missing"].append(x+"/metadata.xml")
1384         #metadata.xml parse check
1385         else:
1386                 metadata_bad = False
1387
1388                 # read metadata.xml into memory
1389                 try:
1390                         _metadata_xml = xml.etree.ElementTree.parse(
1391                                 os.path.join(checkdir, "metadata.xml"),
1392                                 parser=xml.etree.ElementTree.XMLParser(
1393                                         target=_MetadataTreeBuilder()))
1394                 except (ExpatError, EnvironmentError) as e:
1395                         metadata_bad = True
1396                         stats["metadata.bad"] += 1
1397                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1398                         del e
1399                 else:
1400                         # load USE flags from metadata.xml
1401                         try:
1402                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1403                         except portage.exception.ParseError as e:
1404                                 metadata_bad = True
1405                                 stats["metadata.bad"] += 1
1406                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1407
1408                         # Run other metadata.xml checkers
1409                         try:
1410                                 utilities.check_metadata(_metadata_xml, herd_base)
1411                         except (utilities.UnknownHerdsError, ) as e:
1412                                 metadata_bad = True
1413                                 stats["metadata.bad"] += 1
1414                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1415                                 del e
1416
1417                 #Only carry out if in package directory or check forced
1418                 if xmllint_capable and not metadata_bad:
1419                         # xmlint can produce garbage output even on success, so only dump
1420                         # the ouput when it fails.
1421                         st, out = subprocess_getstatusoutput(
1422                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1423                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1424                         if st != os.EX_OK:
1425                                 print(red("!!!") + " metadata.xml is invalid:")
1426                                 for z in out.splitlines():
1427                                         print(red("!!! ")+z)
1428                                 stats["metadata.bad"]+=1
1429                                 fails["metadata.bad"].append(x+"/metadata.xml")
1430
1431                 del metadata_bad
1432         muselist = frozenset(musedict)
1433
1434         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1435         changelog_modified = changelog_path in modified_changelogs
1436
1437         allmasked = True
1438         # detect unused local USE-descriptions
1439         used_useflags = set()
1440
1441         for y in ebuildlist:
1442                 relative_path = os.path.join(x, y + ".ebuild")
1443                 full_path = os.path.join(repodir, relative_path)
1444                 ebuild_path = y + ".ebuild"
1445                 if repolevel < 3:
1446                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1447                 if repolevel < 2:
1448                         ebuild_path = os.path.join(catdir, ebuild_path)
1449                 ebuild_path = os.path.join(".", ebuild_path)
1450                 if check_changelog and not changelog_modified \
1451                         and ebuild_path in new_ebuilds:
1452                         stats['changelog.ebuildadded'] += 1
1453                         fails['changelog.ebuildadded'].append(relative_path)
1454
1455                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1456                         #ebuild not added to vcs
1457                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1458                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1459                 myesplit=portage.pkgsplit(y)
1460                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1461                             or pv_toolong_re.search(myesplit[1]) \
1462                             or pv_toolong_re.search(myesplit[2]):
1463                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1464                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1465                         continue
1466                 elif myesplit[0]!=pkgdir:
1467                         print(pkgdir,myesplit[0])
1468                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1469                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1470                         continue
1471
1472                 pkg = pkgs[y]
1473
1474                 if pkg.invalid:
1475                         allvalid = False
1476                         for k, msgs in pkg.invalid.items():
1477                                 for msg in msgs:
1478                                         stats[k] = stats[k] + 1
1479                                         fails[k].append("%s %s" % (relative_path, msg))
1480                         continue
1481
1482                 myaux = pkg.metadata
1483                 eapi = myaux["EAPI"]
1484                 inherited = pkg.inherited
1485                 live_ebuild = live_eclasses.intersection(inherited)
1486
1487                 for k, v in myaux.items():
1488                         if not isinstance(v, basestring):
1489                                 continue
1490                         m = non_ascii_re.search(v)
1491                         if m is not None:
1492                                 stats["variable.invalidchar"] += 1
1493                                 fails["variable.invalidchar"].append(
1494                                         ("%s: %s variable contains non-ASCII " + \
1495                                         "character at position %s") % \
1496                                         (relative_path, k, m.start() + 1))
1497
1498                 if not src_uri_error:
1499                         # Check that URIs don't reference a server from thirdpartymirrors.
1500                         for uri in portage.dep.use_reduce( \
1501                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1502                                 contains_mirror = False
1503                                 for mirror in thirdpartymirrors:
1504                                         if uri.startswith(mirror):
1505                                                 contains_mirror = True
1506                                                 break
1507                                 if not contains_mirror:
1508                                         continue
1509
1510                                 stats["SRC_URI.mirror"] += 1
1511                                 fails["SRC_URI.mirror"].append(
1512                                         "%s: '%s' found in thirdpartymirrors" % \
1513                                         (relative_path, mirror))
1514
1515                 try:
1516                         provide = portage.dep.use_reduce(pkg.metadata['PROVIDE'],
1517                                 token_class=portage.dep.Atom, matchall=1, flat=True)
1518                 except portage.exception.InvalidDependString:
1519                         stats["PROVIDE.syntax"] = stats["PROVIDE.syntax"] + 1
1520                         fails["PROVIDE.syntax"].append("%s: %s" % \
1521                                 (relative_path, pkg.metadata['PROVIDE']))
1522                         provide = []
1523                 provide_cps = []
1524
1525                 # The Package class automatically evaluates USE conditionals.
1526                 for myprovide in provide:
1527                         if not isinstance(myprovide, portage.dep.Atom):
1528                                 stats["PROVIDE.syntax"] = stats["PROVIDE.syntax"] + 1
1529                                 fails["PROVIDE.syntax"].append("%s: %s" % \
1530                                         (relative_path, myprovide))
1531                                 continue
1532                         prov_cp = myprovide.cp
1533                         provide_cps.append(prov_cp)
1534                         if prov_cp != myprovide:
1535                                 stats["virtual.versioned"]+=1
1536                                 fails["virtual.versioned"].append(x+"/"+y+".ebuild: "+myprovide)
1537                         if portdb.cp_list(prov_cp):
1538                                 stats["virtual.exists"]+=1
1539                                 fails["virtual.exists"].append(x+"/"+y+".ebuild: "+prov_cp)
1540
1541                 for pos, missing_var in enumerate(missingvars):
1542                         if not myaux.get(missing_var):
1543                                 if catdir == "virtual" and \
1544                                         missing_var in ("HOMEPAGE", "LICENSE"):
1545                                         continue
1546                                 if live_ebuild and missing_var == "KEYWORDS":
1547                                         continue
1548                                 myqakey=missingvars[pos]+".missing"
1549                                 stats[myqakey]=stats[myqakey]+1
1550                                 fails[myqakey].append(x+"/"+y+".ebuild")
1551
1552                 if catdir == "virtual":
1553                         for var in ("HOMEPAGE", "LICENSE"):
1554                                 if myaux.get(var):
1555                                         myqakey = var + ".virtual"
1556                                         stats[myqakey] = stats[myqakey] + 1
1557                                         fails[myqakey].append(relative_path)
1558
1559                 # 14 is the length of DESCRIPTION=""
1560                 if len(myaux['DESCRIPTION']) > max_desc_len:
1561                         stats['DESCRIPTION.toolong'] += 1
1562                         fails['DESCRIPTION.toolong'].append(
1563                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1564                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1565
1566                 keywords = myaux["KEYWORDS"].split()
1567                 stable_keywords = []
1568                 for keyword in keywords:
1569                         if not keyword.startswith("~") and \
1570                                 not keyword.startswith("-"):
1571                                 stable_keywords.append(keyword)
1572                 if stable_keywords:
1573                         if ebuild_path in new_ebuilds:
1574                                 stable_keywords.sort()
1575                                 stats["KEYWORDS.stable"] += 1
1576                                 fails["KEYWORDS.stable"].append(
1577                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1578                                                 " ".join(stable_keywords))
1579
1580                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1581                         if not kw.startswith("-"))
1582
1583                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1584                 if previous_keywords is None:
1585                         slot_keywords[myaux["SLOT"]] = set()
1586                 elif ebuild_archs and not live_ebuild:
1587                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1588                         if dropped_keywords:
1589                                 stats["KEYWORDS.dropped"] += 1
1590                                 fails["KEYWORDS.dropped"].append(
1591                                         relative_path + ": %s" % \
1592                                         " ".join(sorted(dropped_keywords)))
1593
1594                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1595
1596                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1597                 if "-*" in keywords:
1598                         haskeyword = False
1599                         for kw in keywords:
1600                                 if kw[0] == "~":
1601                                         kw = kw[1:]
1602                                 if kw in kwlist:
1603                                         haskeyword = True
1604                         if not haskeyword:
1605                                 stats["KEYWORDS.stupid"] += 1
1606                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1607
1608                 """
1609                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1610                 not be allowed to be marked stable
1611                 """
1612                 if live_ebuild:
1613                         bad_stable_keywords = []
1614                         for keyword in keywords:
1615                                 if not keyword.startswith("~") and \
1616                                         not keyword.startswith("-"):
1617                                         bad_stable_keywords.append(keyword)
1618                                 del keyword
1619                         if bad_stable_keywords:
1620                                 stats["LIVEVCS.stable"] += 1
1621                                 fails["LIVEVCS.stable"].append(
1622                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1623                                                 bad_stable_keywords)
1624                         del bad_stable_keywords
1625
1626                         if keywords and not has_global_mask(pkg):
1627                                 stats["LIVEVCS.unmasked"] += 1
1628                                 fails["LIVEVCS.unmasked"].append(relative_path)
1629
1630                 if options.ignore_arches:
1631                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1632                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1633                 else:
1634                         arches=[]
1635                         for keyword in myaux["KEYWORDS"].split():
1636                                 if (keyword[0]=="-"):
1637                                         continue
1638                                 elif (keyword[0]=="~"):
1639                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1640                                 else:
1641                                         arches.append([keyword, keyword, [keyword]])
1642                                         allmasked = False
1643                         if not arches:
1644                                 # Use an empty profile for checking dependencies of
1645                                 # packages that have empty KEYWORDS.
1646                                 arches.append(['**', '**', ['**']])
1647
1648                 baddepsyntax = False
1649                 badlicsyntax = False
1650                 badprovsyntax = False
1651                 catpkg = catdir+"/"+y
1652
1653                 inherited_java_eclass = "java-pkg-2" in inherited or \
1654                         "java-pkg-opt-2" in inherited
1655                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1656                 operator_tokens = set(["||", "(", ")"])
1657                 type_list, badsyntax = [], []
1658                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1659                         "LICENSE", "PROPERTIES", "PROVIDE"):
1660                         mydepstr = myaux[mytype]
1661
1662                         token_class = None
1663                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1664                                 token_class=portage.dep.Atom
1665
1666                         try:
1667                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1668                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1669                         except portage.exception.InvalidDependString as e:
1670                                 atoms = None
1671                                 badsyntax.append(str(e))
1672
1673                         if atoms and mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1674                                 if mytype in ("RDEPEND", "PDEPEND") and \
1675                                         "test?" in mydepstr.split():
1676                                         stats[mytype + '.suspect'] += 1
1677                                         fails[mytype + '.suspect'].append(relative_path + \
1678                                                 ": 'test?' USE conditional in %s" % mytype)
1679
1680                                 for atom in atoms:
1681                                         if atom == "||":
1682                                                 continue
1683
1684                                         is_blocker = atom.blocker
1685
1686                                         if mytype == "DEPEND" and \
1687                                                 not is_blocker and \
1688                                                 not inherited_java_eclass and \
1689                                                 atom.cp == "virtual/jdk":
1690                                                 stats['java.eclassesnotused'] += 1
1691                                                 fails['java.eclassesnotused'].append(relative_path)
1692                                         elif mytype == "DEPEND" and \
1693                                                 not is_blocker and \
1694                                                 not inherited_wxwidgets_eclass and \
1695                                                 atom.cp == "x11-libs/wxGTK":
1696                                                 stats['wxwidgets.eclassnotused'] += 1
1697                                                 fails['wxwidgets.eclassnotused'].append(
1698                                                         relative_path + ": DEPENDs on x11-libs/wxGTK"
1699                                                         " without inheriting wxwidgets.eclass")
1700                                         elif mytype in ("PDEPEND", "RDEPEND"):
1701                                                 if not is_blocker and \
1702                                                         atom.cp in suspect_rdepend:
1703                                                         stats[mytype + '.suspect'] += 1
1704                                                         fails[mytype + '.suspect'].append(
1705                                                                 relative_path + ": '%s'" % atom)
1706
1707                                         if atom.operator == "~" and \
1708                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1709                                                 stats[mytype + '.badtilde'] += 1
1710                                                 fails[mytype + '.badtilde'].append(
1711                                                         (relative_path + ": %s uses the ~ operator"
1712                                                          " with a non-zero revision:" + \
1713                                                          " '%s'") % (mytype, atom))
1714
1715                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1716
1717                 for m,b in zip(type_list, badsyntax):
1718                         stats[m+".syntax"] += 1
1719                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1720
1721                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1722                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1723                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1724                 badlicsyntax = badlicsyntax > 0
1725                 badprovsyntax = badprovsyntax > 0
1726
1727                 # uselist checks - global
1728                 myuse = []
1729                 default_use = []
1730                 for myflag in myaux["IUSE"].split():
1731                         flag_name = myflag.lstrip("+-")
1732                         used_useflags.add(flag_name)
1733                         if myflag != flag_name:
1734                                 default_use.append(myflag)
1735                         if flag_name not in uselist:
1736                                 myuse.append(flag_name)
1737
1738                 # uselist checks - metadata
1739                 for mypos in range(len(myuse)-1,-1,-1):
1740                         if myuse[mypos] and (myuse[mypos] in muselist):
1741                                 del myuse[mypos]
1742
1743                 if default_use and not eapi_has_iuse_defaults(eapi):
1744                         for myflag in default_use:
1745                                 stats['EAPI.incompatible'] += 1
1746                                 fails['EAPI.incompatible'].append(
1747                                         (relative_path + ": IUSE defaults" + \
1748                                         " not supported with EAPI='%s':" + \
1749                                         " '%s'") % (eapi, myflag))
1750
1751                 for mypos in range(len(myuse)):
1752                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1753                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1754
1755                 # license checks
1756                 if not badlicsyntax:
1757                         # Parse the LICENSE variable, remove USE conditions and
1758                         # flatten it.
1759                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
1760                         # Check each entry to ensure that it exists in PORTDIR's
1761                         # license directory.
1762                         for lic in licenses:
1763                                 # Need to check for "||" manually as no portage
1764                                 # function will remove it without removing values.
1765                                 if lic not in liclist and lic != "||":
1766                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1767                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
1768
1769                 #keyword checks
1770                 myuse = myaux["KEYWORDS"].split()
1771                 for mykey in myuse:
1772                         myskey=mykey[:]
1773                         if myskey[0]=="-":
1774                                 myskey=myskey[1:]
1775                         if myskey[0]=="~":
1776                                 myskey=myskey[1:]
1777                         if mykey!="-*":
1778                                 if myskey not in kwlist:
1779                                         stats["KEYWORDS.invalid"] += 1
1780                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1781                                 elif myskey not in profiles:
1782                                         stats["KEYWORDS.invalid"] += 1
1783                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1784
1785                 #restrict checks
1786                 myrestrict = None
1787                 try:
1788                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
1789                 except portage.exception.InvalidDependString as e:
1790                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1791                         fails["RESTRICT.syntax"].append(
1792                                 "%s: RESTRICT: %s" % (relative_path, e))
1793                         del e
1794                 if myrestrict:
1795                         myrestrict = set(myrestrict)
1796                         mybadrestrict = myrestrict.difference(valid_restrict)
1797                         if mybadrestrict:
1798                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
1799                                 for mybad in mybadrestrict:
1800                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1801                 #REQUIRED_USE check
1802                 required_use = myaux["REQUIRED_USE"]
1803                 if required_use:
1804                         if not eapi_has_required_use(eapi):
1805                                 stats['EAPI.incompatible'] += 1
1806                                 fails['EAPI.incompatible'].append(
1807                                         relative_path + ": REQUIRED_USE" + \
1808                                         " not supported with EAPI='%s'" % (eapi,))
1809                         try:
1810                                 portage.dep.check_required_use(required_use, (),
1811                                         pkg.iuse.is_valid_flag)
1812                         except portage.exception.InvalidDependString as e:
1813                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
1814                                 fails["REQUIRED_USE.syntax"].append(
1815                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
1816                                 del e
1817
1818                 # Syntax Checks
1819                 relative_path = os.path.join(x, y + ".ebuild")
1820                 full_path = os.path.join(repodir, relative_path)
1821                 if not vcs_preserves_mtime:
1822                         if ebuild_path not in new_ebuilds and \
1823                                 ebuild_path not in modified_ebuilds:
1824                                 pkg.mtime = None
1825                 try:
1826                         # All ebuilds should have utf_8 encoding.
1827                         f = codecs.open(_unicode_encode(full_path,
1828                                 encoding=_encodings['fs'], errors='strict'),
1829                                 mode='r', encoding=_encodings['repo.content'])
1830                         try:
1831                                 for check_name, e in run_checks(f, pkg):
1832                                         stats[check_name] += 1
1833                                         fails[check_name].append(relative_path + ': %s' % e)
1834                         finally:
1835                                 f.close()
1836                 except UnicodeDecodeError:
1837                         # A file.UTF8 failure will have already been recorded above.
1838                         pass
1839
1840                 if options.force:
1841                         # The dep_check() calls are the most expensive QA test. If --force
1842                         # is enabled, there's no point in wasting time on these since the
1843                         # user is intent on forcing the commit anyway.
1844                         continue
1845
1846                 for keyword,arch,groups in arches:
1847
1848                         if arch not in profiles:
1849                                 # A missing profile will create an error further down
1850                                 # during the KEYWORDS verification.
1851                                 continue
1852
1853                         for prof in profiles[arch]:
1854
1855                                 if prof.status not in ("stable", "dev") or \
1856                                         prof.status == "dev" and not options.include_dev:
1857                                         continue
1858
1859                                 dep_settings = arch_caches.get(prof.sub_path)
1860                                 if dep_settings is None:
1861                                         dep_settings = portage.config(
1862                                                 config_profile_path=prof.abs_path,
1863                                                 config_incrementals=repoman_incrementals,
1864                                                 local_config=False,
1865                                                 _unmatched_removal=options.unmatched_removal,
1866                                                 env=env)
1867                                         if options.without_mask:
1868                                                 dep_settings._mask_manager = \
1869                                                         copy.deepcopy(dep_settings._mask_manager)
1870                                                 dep_settings._mask_manager._pmaskdict.clear()
1871                                         arch_caches[prof.sub_path] = dep_settings
1872
1873                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
1874                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
1875                                 if xcache is None:
1876                                         portdb.melt()
1877                                         portdb.freeze()
1878                                         xcache = portdb.xcache
1879                                         xcache.update(shared_xmatch_caches)
1880                                         arch_xmatch_caches[xmatch_cache_key] = xcache
1881
1882                                 trees["/"]["porttree"].settings = dep_settings
1883                                 portdb.settings = dep_settings
1884                                 portdb.xcache = xcache
1885                                 # for package.use.mask support inside dep_check
1886                                 dep_settings.setcpv(pkg)
1887                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1888                                 # just in case, prevent config.reset() from nuking these.
1889                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1890
1891                                 if prof.sub_path:
1892                                         # old-style virtuals currently aren't
1893                                         # resolvable with empty profile, since
1894                                         # mappings from 'virtuals' files are
1895                                         # unavailable (it would be expensive to
1896                                         # search for PROVIDE in all ebuilds)
1897                                         for prov_cp in provide_cps:
1898                                                 if prov_cp not in dep_settings.getvirtuals():
1899                                                         stats["virtual.unavailable"] += 1
1900                                                         fails["virtual.unavailable"].append(
1901                                                                 "%s: %s(%s) %s" % (relative_path, keyword,
1902                                                                 prof.sub_path, prov_cp))
1903
1904                                 if not baddepsyntax:
1905                                         ismasked = not ebuild_archs or \
1906                                                 pkg.cp not in portdb.xmatch("list-visible", x)
1907                                         if ismasked:
1908                                                 if not have_pmasked:
1909                                                         have_pmasked = bool(dep_settings._getMaskAtom(
1910                                                                 pkg.cpv, pkg.metadata))
1911                                                 if options.ignore_masked:
1912                                                         continue
1913                                                 #we are testing deps for a masked package; give it some lee-way
1914                                                 suffix="masked"
1915                                                 matchmode = "minimum-all"
1916                                         else:
1917                                                 suffix=""
1918                                                 matchmode = "minimum-visible"
1919
1920                                         if not have_dev_keywords:
1921                                                 have_dev_keywords = \
1922                                                         bool(dev_keywords.intersection(keywords))
1923
1924                                         if prof.status == "dev":
1925                                                 suffix=suffix+"indev"
1926
1927                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1928
1929                                                 mykey=mytype+".bad"+suffix
1930                                                 myvalue = myaux[mytype]
1931                                                 if not myvalue:
1932                                                         continue
1933
1934                                                 success, atoms = portage.dep_check(myvalue, portdb,
1935                                                         dep_settings, use="all", mode=matchmode,
1936                                                         trees=trees)
1937
1938                                                 if success:
1939                                                         if atoms:
1940
1941                                                                 if not prof.sub_path:
1942                                                                         # old-style virtuals currently aren't
1943                                                                         # resolvable with empty profile, since
1944                                                                         # 'virtuals' mappings are unavailable
1945                                                                         # (it would be expensive to search
1946                                                                         # for PROVIDE in all ebuilds)
1947                                                                         atoms = [atom for atom in atoms if not \
1948                                                                                 (atom.cp.startswith('virtual/') and \
1949                                                                                 not portdb.cp_list(atom.cp))]
1950
1951                                                                 #we have some unsolvable deps
1952                                                                 #remove ! deps, which always show up as unsatisfiable
1953                                                                 atoms = [str(atom.unevaluated_atom) \
1954                                                                         for atom in atoms if not atom.blocker]
1955
1956                                                                 #if we emptied out our list, continue:
1957                                                                 if not atoms:
1958                                                                         continue
1959                                                                 stats[mykey]=stats[mykey]+1
1960                                                                 fails[mykey].append("%s: %s(%s) %s" % \
1961                                                                         (relative_path, keyword,
1962                                                                         prof, repr(atoms)))
1963                                                 else:
1964                                                         stats[mykey]=stats[mykey]+1
1965                                                         fails[mykey].append("%s: %s(%s) %s" % \
1966                                                                 (relative_path, keyword,
1967                                                                 prof, repr(atoms)))
1968
1969         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1970         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1971         #if not portage.portdb.xmatch("bestmatch-visible",x):
1972         #    stats["ebuild.nostable"]+=1
1973         #    fails["ebuild.nostable"].append(x)
1974         if allmasked and repolevel == 3:
1975                 stats["ebuild.allmasked"]+=1
1976                 fails["ebuild.allmasked"].append(x)
1977
1978         # check if there are unused local USE-descriptions in metadata.xml
1979         # (unless there are any invalids, to avoid noise)
1980         if allvalid:
1981                 for myflag in muselist.difference(used_useflags):
1982                         stats["metadata.warning"] += 1
1983                         fails["metadata.warning"].append(
1984                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
1985                                 (x, myflag))
1986
1987 if options.mode == "manifest":
1988         sys.exit(dofail)
1989
1990 #dofail will be set to 1 if we have failed in at least one non-warning category
1991 dofail=0
1992 #dowarn will be set to 1 if we tripped any warnings
1993 dowarn=0
1994 #dofull will be set if we should print a "repoman full" informational message
1995 dofull = options.mode != 'full'
1996
1997 for x in qacats:
1998         if not stats[x]:
1999                 continue
2000         dowarn = 1
2001         if x not in qawarnings:
2002                 dofail = 1
2003
2004 if dofail or \
2005         (dowarn and not (options.quiet or options.mode == "scan")):
2006         dofull = 0
2007
2008 # Save QA output so that it can be conveniently displayed
2009 # in $EDITOR while the user creates a commit message.
2010 # Otherwise, the user would not be able to see this output
2011 # once the editor has taken over the screen.
2012 qa_output = StringIO()
2013 style_file = ConsoleStyleFile(sys.stdout)
2014 if options.mode == 'commit' and \
2015         (not commitmessage or not commitmessage.strip()):
2016         style_file.write_listener = qa_output
2017 console_writer = StyleWriter(file=style_file, maxcol=9999)
2018 console_writer.style_listener = style_file.new_styles
2019
2020 f = formatter.AbstractFormatter(console_writer)
2021
2022 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2023
2024 style_file.flush()
2025 del console_writer, f, style_file
2026 qa_output = qa_output.getvalue()
2027 qa_output = qa_output.splitlines(True)
2028
2029 def grouplist(mylist,seperator="/"):
2030         """(list,seperator="/") -- Takes a list of elements; groups them into
2031         same initial element categories. Returns a dict of {base:[sublist]}
2032         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2033         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2034         mygroups={}
2035         for x in mylist:
2036                 xs=x.split(seperator)
2037                 if xs[0]==".":
2038                         xs=xs[1:]
2039                 if xs[0] not in mygroups:
2040                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2041                 else:
2042                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2043         return mygroups
2044
2045 suggest_ignore_masked = False
2046 suggest_include_dev = False
2047
2048 if have_pmasked and not (options.without_mask or options.ignore_masked):
2049         suggest_ignore_masked = True
2050 if have_dev_keywords and not options.include_dev:
2051         suggest_include_dev = True
2052
2053 if suggest_ignore_masked or suggest_include_dev:
2054         print()
2055         if suggest_ignore_masked:
2056                 print(bold("Note: use --without-mask to check " + \
2057                         "KEYWORDS on dependencies of masked packages"))
2058
2059         if suggest_include_dev:
2060                 print(bold("Note: use --include-dev (-d) to check " + \
2061                         "dependencies for 'dev' profiles"))
2062         print()
2063
2064 if options.mode != 'commit':
2065         if dofull:
2066                 print(bold("Note: type \"repoman full\" for a complete listing."))
2067         if dowarn and not dofail:
2068                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2069         elif not dofail:
2070                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2071         elif dofail:
2072                 print(bad("Please fix these important QA issues first."))
2073                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2074                 sys.exit(1)
2075 else:
2076         if dofail and can_force and options.force and not options.pretend:
2077                 print(green("RepoMan sez:") + \
2078                         " \"You want to commit even with these QA issues?\n" + \
2079                         "              I'll take it this time, but I'm not happy.\"\n")
2080         elif dofail:
2081                 if options.force and not can_force:
2082                         print(bad("The --force option has been disabled due to extraordinary issues."))
2083                 print(bad("Please fix these important QA issues first."))
2084                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2085                 sys.exit(1)
2086
2087         if options.pretend:
2088                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2089
2090         myunadded = []
2091         if vcs == "cvs":
2092                 try:
2093                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2094                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2095                 except SystemExit as e:
2096                         raise  # TODO propagate this
2097                 except:
2098                         err("Error retrieving CVS tree; exiting.")
2099         if vcs == "svn":
2100                 try:
2101                         svnstatus=os.popen("svn status --no-ignore").readlines()
2102                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2103                 except SystemExit as e:
2104                         raise  # TODO propagate this
2105                 except:
2106                         err("Error retrieving SVN info; exiting.")
2107         if vcs == "git":
2108                 # get list of files not under version control or missing
2109                 myf = os.popen("git ls-files --others")
2110                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2111                 myf.close()
2112         if vcs == "bzr":
2113                 try:
2114                         bzrstatus=os.popen("bzr status -S .").readlines()
2115                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2116                 except SystemExit as e:
2117                         raise  # TODO propagate this
2118                 except:
2119                         err("Error retrieving bzr info; exiting.")
2120         if vcs == "hg":
2121                 myunadded = os.popen("hg status --no-status --unknown .").readlines()
2122                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2123
2124                 # Mercurial doesn't handle manually deleted files as removed from
2125                 # the repository, so the user need to remove them before commit,
2126                 # using "hg remove [FILES]"
2127                 mydeleted = os.popen("hg status --no-status --deleted .").readlines()
2128                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2129
2130
2131         myautoadd=[]
2132         if myunadded:
2133                 for x in range(len(myunadded)-1,-1,-1):
2134                         xs=myunadded[x].split("/")
2135                         if xs[-1]=="files":
2136                                 print("!!! files dir is not added! Please correct this.")
2137                                 sys.exit(-1)
2138                         elif xs[-1]=="Manifest":
2139                                 # It's a manifest... auto add
2140                                 myautoadd+=[myunadded[x]]
2141                                 del myunadded[x]
2142
2143         if myautoadd:
2144                 print(">>> Auto-Adding missing Manifest(s)...")
2145                 if options.pretend:
2146                         if vcs == "cvs":
2147                                 print("(cvs add "+" ".join(myautoadd)+")")
2148                         elif vcs == "svn":
2149                                 print("(svn add "+" ".join(myautoadd)+")")
2150                         elif vcs == "git":
2151                                 print("(git add "+" ".join(myautoadd)+")")
2152                         elif vcs == "bzr":
2153                                 print("(bzr add "+" ".join(myautoadd)+")")
2154                         elif vcs == "hg":
2155                                 print("(hg add "+" ".join(myautoadd)+")")
2156                         retval=0
2157                 else:
2158                         if vcs == "cvs":
2159                                 retval=os.system("cvs add "+" ".join(myautoadd))
2160                         elif vcs == "svn":
2161                                 retval=os.system("svn add "+" ".join(myautoadd))
2162                         elif vcs == "git":
2163                                 retval=os.system("git add "+" ".join(myautoadd))
2164                         elif vcs == "bzr":
2165                                 retval=os.system("bzr add "+" ".join(myautoadd))
2166                         elif vcs == "hg":
2167                                 retval=os.system("hg add "+" ".join(myautoadd))
2168                 if retval:
2169                         writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \
2170                                 (vcs, retval), level=logging.ERROR, noiselevel=-1)
2171                         sys.exit(retval)
2172
2173         if myunadded:
2174                 print(red("!!! The following files are in your local tree but are not added to the master"))
2175                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2176                 for x in myunadded:
2177                         print("   ",x)
2178                 print()
2179                 print()
2180                 sys.exit(1)
2181
2182         if vcs == "hg" and mydeleted:
2183                 print(red("!!! The following files are removed manually from your local tree but are not"))
2184                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2185                 for x in mydeleted:
2186                         print("   ",x)
2187                 print()
2188                 print()
2189                 sys.exit(1)
2190
2191         if vcs == "cvs":
2192                 mycvstree = cvstree.getentries("./", recursive=1)
2193                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2194                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2195                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2196                 bin_blob_pattern = re.compile("^-kb$")
2197                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2198                         recursive=1, basedir="./"))
2199
2200
2201         if vcs == "svn":
2202                 svnstatus = os.popen("svn status").readlines()
2203                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2204                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2205                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2206
2207                 # Subversion expands keywords specified in svn:keywords properties.
2208                 props = os.popen("svn propget -R svn:keywords").readlines()
2209                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2210                         for prop in props if " - " in prop)
2211
2212         elif vcs == "git":
2213                 mychanged = os.popen("git diff-index --name-only --relative --diff-filter=M HEAD").readlines()
2214                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2215
2216                 mynew = os.popen("git diff-index --name-only --relative --diff-filter=A HEAD").readlines()
2217                 mynew = ["./" + elem[:-1] for elem in mynew]
2218
2219                 myremoved = os.popen("git diff-index --name-only --relative --diff-filter=D HEAD").readlines()
2220                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2221
2222         if vcs == "bzr":
2223                 bzrstatus = os.popen("bzr status -S .").readlines()
2224                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2225                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2226                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2227                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2228                 # Bazaar expands nothing.
2229
2230         if vcs == "hg":
2231                 mychanged = os.popen("hg status --no-status --modified .").readlines()
2232                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2233                 mynew = os.popen("hg status --no-status --added .").readlines()
2234                 mynew = ["./" + elem.rstrip() for elem in mynew]
2235                 myremoved = os.popen("hg status --no-status --removed .").readlines()
2236                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2237
2238         if vcs:
2239                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2240                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2241                         print()
2242                         print("(Didn't find any changed files...)")
2243                         print()
2244                         sys.exit(1)
2245
2246         # Manifests need to be regenerated after all other commits, so don't commit
2247         # them now even if they have changed.
2248         mymanifests = set()
2249         myupdates = set()
2250         for f in mychanged + mynew:
2251                 if "Manifest" == os.path.basename(f):
2252                         mymanifests.add(f)
2253                 else:
2254                         myupdates.add(f)
2255         if vcs in ('git', 'hg'):
2256                 myupdates.difference_update(myremoved)
2257         myupdates = list(myupdates)
2258         mymanifests = list(mymanifests)
2259         myheaders = []
2260         mydirty = []
2261
2262         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2263         if vcs not in ('cvs', 'svn'):
2264                 # With git, bzr and hg, there's never any keyword expansion, so
2265                 # there's no need to regenerate manifests and all files will be
2266                 # committed in one big commit at the end.
2267                 print()
2268         else:
2269                 if vcs == 'cvs':
2270                         headerstring = "'\$(Header|Id).*\$'"
2271                 elif vcs == "svn":
2272                         svn_keywords = dict((k.lower(), k) for k in [
2273                                         "Rev",
2274                                         "Revision",
2275                                         "LastChangedRevision",
2276                                         "Date",
2277                                         "LastChangedDate",
2278                                         "Author",
2279                                         "LastChangedBy",
2280                                         "URL",
2281                                         "HeadURL",
2282                                         "Id",
2283                                         "Header",
2284                         ])
2285
2286                 for myfile in myupdates:
2287
2288                         # for CVS, no_expansion contains files that are excluded from expansion
2289                         if vcs == "cvs":
2290                                 if myfile in no_expansion:
2291                                         continue
2292
2293                         # for SVN, expansion contains files that are included in expansion
2294                         elif vcs == "svn":
2295                                 if myfile not in expansion:
2296                                         continue
2297
2298                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2299                                 enabled_keywords = []
2300                                 for k in expansion[myfile]:
2301                                         keyword = svn_keywords.get(k.lower())
2302                                         if keyword is not None:
2303                                                 enabled_keywords.append(keyword)
2304
2305                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2306
2307                         myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile)
2308                         if myout[0] == 0:
2309                                 myheaders.append(myfile)
2310
2311                 print("%s have headers that will change." % green(str(len(myheaders))))
2312                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2313
2314         logging.info("myupdates: %s", myupdates)
2315         logging.info("myheaders: %s", myheaders)
2316
2317         commitmessage = options.commitmsg
2318         if options.commitmsgfile:
2319                 try:
2320                         f = codecs.open(_unicode_encode(options.commitmsgfile,
2321                         encoding=_encodings['fs'], errors='strict'),
2322                         mode='r', encoding=_encodings['content'], errors='replace')
2323                         commitmessage = f.read()
2324                         f.close()
2325                         del f
2326                 except (IOError, OSError) as e:
2327                         if e.errno == errno.ENOENT:
2328                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2329                         else:
2330                                 raise
2331                 # We've read the content so the file is no longer needed.
2332                 commitmessagefile = None
2333         if not commitmessage or not commitmessage.strip():
2334                 try:
2335                         editor = os.environ.get("EDITOR")
2336                         if editor and utilities.editor_is_executable(editor):
2337                                 commitmessage = utilities.get_commit_message_with_editor(
2338                                         editor, message=qa_output)
2339                         else:
2340                                 commitmessage = utilities.get_commit_message_with_stdin()
2341                 except KeyboardInterrupt:
2342                         exithandler()
2343                 if not commitmessage or not commitmessage.strip():
2344                         print("* no commit message?  aborting commit.")
2345                         sys.exit(1)
2346         commitmessage = commitmessage.rstrip()
2347         portage_version = getattr(portage, "VERSION", None)
2348         if portage_version is None:
2349                 sys.stderr.write("Failed to insert portage version in message!\n")
2350                 sys.stderr.flush()
2351                 portage_version = "Unknown"
2352         unameout = platform.system() + " "
2353         if platform.system() in ["Darwin", "SunOS"]:
2354                 unameout += platform.processor()
2355         else:
2356                 unameout += platform.machine()
2357         commitmessage += "\n\n(Portage version: %s/%s/%s" % \
2358                 (portage_version, vcs, unameout)
2359         if options.force:
2360                 commitmessage += ", RepoMan options: --force"
2361         commitmessage += ")"
2362
2363         if options.ask and userquery('Commit changes?', True) != 'Yes':
2364                 print("* aborting commit.")
2365                 sys.exit(1)
2366
2367         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2368                 myfiles = myupdates + myremoved
2369                 if not myheaders and "sign" not in repoman_settings.features:
2370                         myfiles += mymanifests
2371                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2372                 mymsg = os.fdopen(fd, "wb")
2373                 mymsg.write(_unicode_encode(commitmessage))
2374                 mymsg.close()
2375
2376                 print()
2377                 print(green("Using commit message:"))
2378                 print(green("------------------------------------------------------------------------------"))
2379                 print(commitmessage)
2380                 print(green("------------------------------------------------------------------------------"))
2381                 print()
2382
2383                 # Having a leading ./ prefix on file paths can trigger a bug in
2384                 # the cvs server when committing files to multiple directories,
2385                 # so strip the prefix.
2386                 myfiles = [f.lstrip("./") for f in myfiles]
2387
2388                 commit_cmd = [vcs]
2389                 commit_cmd.extend(vcs_global_opts)
2390                 commit_cmd.append("commit")
2391                 commit_cmd.extend(vcs_local_opts)
2392                 commit_cmd.extend(["-F", commitmessagefile])
2393                 commit_cmd.extend(myfiles)
2394
2395                 try:
2396                         if options.pretend:
2397                                 print("(%s)" % (" ".join(commit_cmd),))
2398                         else:
2399                                 retval = spawn(commit_cmd, env=os.environ)
2400                                 if retval != os.EX_OK:
2401                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2402                                                 "error code: %s\n") % (vcs, retval),
2403                                                 level=logging.ERROR, noiselevel=-1)
2404                                         sys.exit(retval)
2405                 finally:
2406                         try:
2407                                 os.unlink(commitmessagefile)
2408                         except OSError:
2409                                 pass
2410
2411         # Setup the GPG commands
2412         def gpgsign(filename):
2413                 if "PORTAGE_GPG_KEY" not in repoman_settings:
2414                         raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
2415                 if "PORTAGE_GPG_DIR" not in repoman_settings:
2416                         if "HOME" in os.environ:
2417                                 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(os.environ["HOME"], ".gnupg")
2418                                 logging.info("Automatically setting PORTAGE_GPG_DIR to %s" % repoman_settings["PORTAGE_GPG_DIR"])
2419                         else:
2420                                 raise portage.exception.MissingParameter("PORTAGE_GPG_DIR is unset!")
2421                 gpg_dir = repoman_settings["PORTAGE_GPG_DIR"]
2422                 if gpg_dir.startswith("~") and "HOME" in os.environ:
2423                         repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(
2424                                 os.environ["HOME"], gpg_dir[1:].lstrip(os.path.sep))
2425                 if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2426                         raise portage.exception.InvalidLocation(
2427                                 "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2428                                 repoman_settings["PORTAGE_GPG_DIR"])
2429                 gpgcmd = "gpg --sign --clearsign --yes "
2430                 gpgcmd+= "--default-key "+repoman_settings["PORTAGE_GPG_KEY"]
2431                 if "PORTAGE_GPG_DIR" in repoman_settings:
2432                         gpgcmd += " --homedir "+repoman_settings["PORTAGE_GPG_DIR"]
2433                 if options.pretend:
2434                         print("("+gpgcmd+" "+filename+")")
2435                 else:
2436                         rValue = os.system(gpgcmd+" "+filename)
2437                         if rValue == os.EX_OK:
2438                                 os.rename(filename+".asc", filename)
2439                         else:
2440                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2441
2442         # When files are removed and re-added, the cvs server will put /Attic/
2443         # inside the $Header path. This code detects the problem and corrects it
2444         # so that the Manifest will generate correctly. See bug #169500.
2445         # Use binary mode in order to avoid potential character encoding issues.
2446         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2447         attic_str = b'/Attic/'
2448         attic_replace = b'/'
2449         for x in myheaders:
2450                 f = open(_unicode_encode(x,
2451                         encoding=_encodings['fs'], errors='strict'),
2452                         mode='rb')
2453                 mylines = f.readlines()
2454                 f.close()
2455                 modified = False
2456                 for i, line in enumerate(mylines):
2457                         if cvs_header_re.match(line) is not None and \
2458                                 attic_str in line:
2459                                 mylines[i] = line.replace(attic_str, attic_replace)
2460                                 modified = True
2461                 if modified:
2462                         portage.util.write_atomic(x, b''.join(mylines),
2463                                 mode='wb')
2464
2465         manifest_commit_required = True
2466         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2467                 myfiles = myupdates + myremoved
2468                 for x in range(len(myfiles)-1, -1, -1):
2469                         if myfiles[x].count("/") < 4-repolevel:
2470                                 del myfiles[x]
2471                 mydone=[]
2472                 if repolevel==3:   # In a package dir
2473                         repoman_settings["O"] = startdir
2474                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2475                 elif repolevel==2: # In a category dir
2476                         for x in myfiles:
2477                                 xs=x.split("/")
2478                                 if len(xs) < 4-repolevel:
2479                                         continue
2480                                 if xs[0]==".":
2481                                         xs=xs[1:]
2482                                 if xs[0] in mydone:
2483                                         continue
2484                                 mydone.append(xs[0])
2485                                 repoman_settings["O"] = os.path.join(startdir, xs[0])
2486                                 if not os.path.isdir(repoman_settings["O"]):
2487                                         continue
2488                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2489                 elif repolevel==1: # repo-cvsroot
2490                         print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
2491                         for x in myfiles:
2492                                 xs=x.split("/")
2493                                 if len(xs) < 4-repolevel:
2494                                         continue
2495                                 if xs[0]==".":
2496                                         xs=xs[1:]
2497                                 if "/".join(xs[:2]) in mydone:
2498                                         continue
2499                                 mydone.append("/".join(xs[:2]))
2500                                 repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1])
2501                                 if not os.path.isdir(repoman_settings["O"]):
2502                                         continue
2503                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2504                 else:
2505                         print(red("I'm confused... I don't know where I am!"))
2506                         sys.exit(1)
2507
2508                 # Force an unsigned commit when more than one Manifest needs to be signed.
2509                 if repolevel < 3 and "sign" in repoman_settings.features:
2510
2511                         fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2512                         mymsg = os.fdopen(fd, "wb")
2513                         mymsg.write(_unicode_encode(commitmessage))
2514                         mymsg.write(b"\n (Unsigned Manifest commit)")
2515                         mymsg.close()
2516
2517                         commit_cmd = [vcs]
2518                         commit_cmd.extend(vcs_global_opts)
2519                         commit_cmd.append("commit")
2520                         commit_cmd.extend(vcs_local_opts)
2521                         commit_cmd.extend(["-F", commitmessagefile])
2522                         commit_cmd.extend(f.lstrip("./") for f in mymanifests)
2523
2524                         try:
2525                                 if options.pretend:
2526                                         print("(%s)" % (" ".join(commit_cmd),))
2527                                 else:
2528                                         retval = spawn(commit_cmd, env=os.environ)
2529                                         if retval:
2530                                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2531                                                         "error code: %s\n") % (vcs, retval),
2532                                                         level=logging.ERROR, noiselevel=-1)
2533                                                 sys.exit(retval)
2534                         finally:
2535                                 try:
2536                                         os.unlink(commitmessagefile)
2537                                 except OSError:
2538                                         pass
2539                         manifest_commit_required = False
2540
2541         signed = False
2542         if "sign" in repoman_settings.features:
2543                 signed = True
2544                 myfiles = myupdates + myremoved + mymanifests
2545                 try:
2546                         if repolevel==3:   # In a package dir
2547                                 repoman_settings["O"] = "."
2548                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2549                         elif repolevel==2: # In a category dir
2550                                 mydone=[]
2551                                 for x in myfiles:
2552                                         xs=x.split("/")
2553                                         if len(xs) < 4-repolevel:
2554                                                 continue
2555                                         if xs[0]==".":
2556                                                 xs=xs[1:]
2557                                         if xs[0] in mydone:
2558                                                 continue
2559                                         mydone.append(xs[0])
2560                                         repoman_settings["O"] = os.path.join(".", xs[0])
2561                                         if not os.path.isdir(repoman_settings["O"]):
2562                                                 continue
2563                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2564                         elif repolevel==1: # repo-cvsroot
2565                                 print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
2566                                 mydone=[]
2567                                 for x in myfiles:
2568                                         xs=x.split("/")
2569                                         if len(xs) < 4-repolevel:
2570                                                 continue
2571                                         if xs[0]==".":
2572                                                 xs=xs[1:]
2573                                         if "/".join(xs[:2]) in mydone:
2574                                                 continue
2575                                         mydone.append("/".join(xs[:2]))
2576                                         repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
2577                                         if not os.path.isdir(repoman_settings["O"]):
2578                                                 continue
2579                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2580                 except portage.exception.PortageException as e:
2581                         portage.writemsg("!!! %s\n" % str(e))
2582                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2583                         signed = False
2584
2585         if vcs == 'git':
2586                 # It's not safe to use the git commit -a option since there might
2587                 # be some modified files elsewhere in the working tree that the
2588                 # user doesn't want to commit. Therefore, call git update-index
2589                 # in order to ensure that the index is updated with the latest
2590                 # versions of all new and modified files in the relevant portion
2591                 # of the working tree.
2592                 myfiles = mymanifests + myupdates
2593                 myfiles.sort()
2594                 update_index_cmd = ["git", "update-index"]
2595                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2596                 if options.pretend:
2597                         print("(%s)" % (" ".join(update_index_cmd),))
2598                 else:
2599                         retval = spawn(update_index_cmd, env=os.environ)
2600                         if retval != os.EX_OK:
2601                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2602                                         "error code: %s\n") % (vcs, retval),
2603                                         level=logging.ERROR, noiselevel=-1)
2604                                 sys.exit(retval)
2605
2606         if vcs in ['git', 'bzr', 'hg'] or manifest_commit_required or signed:
2607
2608                 myfiles = mymanifests[:]
2609                 if vcs in ['git', 'bzr', 'hg']:
2610                         myfiles += myupdates
2611                         myfiles += myremoved
2612                 myfiles.sort()
2613
2614                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2615                 mymsg = os.fdopen(fd, "wb")
2616                 # strip the closing parenthesis
2617                 mymsg.write(_unicode_encode(commitmessage[:-1]))
2618                 if signed:
2619                         mymsg.write(_unicode_encode(
2620                                 ", signed Manifest commit with key %s)" % \
2621                                 repoman_settings["PORTAGE_GPG_KEY"]))
2622                 else:
2623                         mymsg.write(b", unsigned Manifest commit)")
2624                 mymsg.close()
2625
2626                 commit_cmd = []
2627                 if options.pretend and vcs is None:
2628                         # substitute a bogus value for pretend output
2629                         commit_cmd.append("cvs")
2630                 else:
2631                         commit_cmd.append(vcs)
2632                 commit_cmd.extend(vcs_global_opts)
2633                 commit_cmd.append("commit")
2634                 commit_cmd.extend(vcs_local_opts)
2635                 if vcs == "hg":
2636                         commit_cmd.extend(["--logfile", commitmessagefile])
2637                         commit_cmd.extend(myfiles)
2638                 else:
2639                         commit_cmd.extend(["-F", commitmessagefile])
2640                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2641
2642                 try:
2643                         if options.pretend:
2644                                 print("(%s)" % (" ".join(commit_cmd),))
2645                         else:
2646                                 retval = spawn(commit_cmd, env=os.environ)
2647                                 if retval != os.EX_OK:
2648                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2649                                                 "error code: %s\n") % (vcs, retval),
2650                                                 level=logging.ERROR, noiselevel=-1)
2651                                         sys.exit(retval)
2652                 finally:
2653                         try:
2654                                 os.unlink(commitmessagefile)
2655                         except OSError:
2656                                 pass
2657
2658         print()
2659         if vcs:
2660                 print("Commit complete.")
2661         else:
2662                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2663         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2664 sys.exit(0)
2665