bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2006 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 # Next to do: dep syntax checking in mask files
   7 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   8 # that last one is tricky because multiple profiles need to be checked.
   9
  10 import codecs
  11 import commands
  12 import errno
  13 import formatter
  14 import logging
  15 import optparse
  16 import os
  17 import re
  18 import signal
  19 import stat
  20 import sys
  21 import tempfile
  22 import time
  23 import platform
  24
  25 from itertools import chain, izip
  26 from stat import S_ISDIR, ST_CTIME
  27
  28 try:
  29         import cPickle as pickle
  30 except ImportError:
  31         import pickle
  32
  33 try:
  34         import cStringIO as StringIO
  35 except ImportError:
  36         import StringIO
  37
  38 if not hasattr(__builtins__, "set"):
  39         from sets import Set as set
  40
  41 try:
  42         import portage
  43 except ImportError:
  44         from os import path as osp
  45         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  46         import portage
  47 portage._disable_legacy_globals()
  48
  49 try:
  50         from repoman.checks import run_checks
  51         from repoman import utilities
  52 except ImportError:
  53         from os import path as osp
  54         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym'))
  55         from repoman.checks import run_checks
  56         from repoman import utilities
  57
  58 from _emerge import Package, RootConfig
  59 from portage._sets import load_default_config
  60
  61 import portage.checksum
  62 import portage.const
  63 import portage.dep
  64 portage.dep._dep_check_strict = True
  65 import portage.exception
  66 from portage import cvstree, normalize_path
  67 from portage import util
  68 from portage.exception import ParseError
  69 from portage.manifest import Manifest
  70 from portage.process import find_binary, spawn
  71 from portage.output import bold, create_color_func, darkgreen, \
  72         green, nocolor, red, turquoise, yellow
  73 from portage.output import ConsoleStyleFile, StyleWriter
  74 from portage.util import writemsg_level
  75
  76 util.initialize_logger()
  77
  78 # 14 is the length of DESCRIPTION=""
  79 max_desc_len = 100
  80 allowed_filename_chars="a-zA-Z0-9._-+:"
  81 allowed_filename_chars_set = {}
  82 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('a'), ord('z')+1)))
  83 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('A'), ord('Z')+1)))
  84 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('0'), ord('9')+1)))
  85 map(allowed_filename_chars_set.setdefault, map(chr, map(ord, [".", "-", "_", "+", ":"])))
  86 bad = create_color_func("BAD")
  87
  88 # A sane umask is needed for files that portage creates.
  89 os.umask(022)
  90 repoman_settings = portage.config(local_config=False,
  91         config_incrementals=portage.const.INCREMENTALS)
  92 repoman_settings.lock()
  93
  94 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
  95         not sys.stdout.isatty():
  96         nocolor()
  97
  98 def warn(txt):
  99         print "repoman: " + txt
 100
 101 def err(txt):
 102         warn(txt)
 103         sys.exit(1)
 104
 105 def exithandler(signum=None, frame=None):
 106         logging.fatal("Interrupted; exiting...")
 107         sys.exit(1)
 108         os.kill(0, signal.SIGKILL)
 109
 110 signal.signal(signal.SIGINT,exithandler)
 111
 112 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 113         """Repoman needs it's own HelpFormatter for now, because the default ones
 114         murder the help text."""
 115
 116         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 117                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 118
 119         def format_description(self, description):
 120                 return description
 121
 122 class RepomanOptionParser(optparse.OptionParser):
 123         """Add the on_tail function, ruby has it, optionParser should too
 124         """
 125
 126         def __init__(self, *args, **kwargs):
 127                 optparse.OptionParser.__init__(self, *args, **kwargs)
 128                 self.tail = ""
 129
 130         def on_tail(self, description):
 131                 self.tail += description
 132
 133         def format_help(self, formatter=None):
 134                 result = optparse.OptionParser.format_help(self, formatter)
 135                 result += self.tail
 136                 return result
 137
 138
 139 def ParseArgs(args, qahelp):
 140         """This function uses a customized optionParser to parse command line arguments for repoman
 141         Args:
 142           args - a sequence of command line arguments
 143                 qahelp - a dict of qa warning to help message
 144         Returns:
 145           (opts, args), just like a call to parser.parse_args()
 146         """
 147
 148         modes = {
 149                 'commit' : 'Run a scan then commit changes',
 150                 'ci' : 'Run a scan then commit changes',
 151                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 152                 'full' : 'Scan directory tree and print all issues (not a summary)',
 153                 'help' : 'Show this screen',
 154                 'last' : 'Remember report from last run',
 155                 'lfull' : 'Remember report from last run (full listing)',
 156                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 157                 'scan' : 'Scan directory tree for QA issues'
 158         }
 159
 160         mode_keys = modes.keys()
 161         mode_keys.sort()
 162
 163         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 164         parser.description = green(" ".join((os.path.basename(args[0]), "1.2")))
 165         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 166         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 167         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 168
 169         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 170                 help='specify a commit message on the command line')
 171
 172         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 173                 help='specify a path to a file that contains a commit message')
 174
 175         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 176                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 177
 178         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 179                 help='do not print unnecessary messages')
 180
 181         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 182                 help='Commit with QA violations')
 183
 184         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 185                 help='be very verbose in output', default=0)
 186
 187         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 188                 default=False, help='forces the metadata.xml parse check to be carried out')
 189
 190         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 191                 default=False, help='ignore arch-specific failures (where arch != host)')
 192
 193         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 194                 default=False, help='ignore masked packages (not allowed with commit mode)')
 195
 196         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 197                 default=False, help='include dev profiles in dependency checks')
 198
 199         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 200                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 201
 202         parser.add_option('--mode', type='choice', dest='mode', choices=modes.keys(),
 203                 help='specify which mode repoman will run in (default=full)')
 204
 205         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 206
 207         for k in mode_keys:
 208                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 209
 210         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 211
 212         sorted_qa = qahelp.keys()
 213         sorted_qa.sort()
 214         for k in sorted_qa:
 215                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 216
 217         if not args:
 218                 args = sys.argv
 219         opts, args = parser.parse_args(args)
 220
 221         if opts.mode == 'help':
 222                 parser.print_help(short=False)
 223
 224         for arg in args:
 225                 if arg in modes:
 226                         if not opts.mode:
 227                                 opts.mode = arg
 228                                 break
 229
 230         if not opts.mode:
 231                 opts.mode = 'full'      #default to full
 232
 233         if opts.mode == 'ci':
 234                 opts.mode = 'commit'  # backwards compat shortcut
 235
 236         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 237                 if opts.ignore_masked:
 238                         parser.error('Commit mode and --ignore-masked are not compatible')
 239                 if opts.without_mask:
 240                         parser.error('Commit mode and --without-mask are not compatible')
 241
 242         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 243         for val in range(opts.verbosity):
 244                 logger = logging.getLogger()
 245                 logger.setLevel(logger.getEffectiveLevel() - 10)
 246
 247         for val in range(opts.quiet):
 248                 logger = logging.getLogger()
 249                 logger.setLevel(logger.getEffectiveLevel() + 10)
 250
 251         return (opts, args)
 252
 253 qahelp={
 254         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 255         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 256         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 257         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 258         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 259         "changelog.missing":"Missing ChangeLog files",
 260         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 261         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 262         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 263         "filedir.missing":"Package lacks a files directory",
 264         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
 265         "file.size":"Files in the files directory must be under 20k",
 266         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 267         "file.UTF8":"File is not UTF8 compliant",
 268         "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf",
 269         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 270         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 271         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 272         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 273         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 274         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 275         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 276         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 277         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 278         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 279         "SLOT.missing":"Ebuilds that have a missing or empty SLOT variable",
 280         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 281         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 282         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 283         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 284         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 285         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 286         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 287         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 288         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 289         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 290         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 291         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 292         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 293         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 294         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 295         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 296         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 297         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 298         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 299         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 300         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 301         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 302         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 303         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 304         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 305         "variable.readonly":"Assigning a readonly variable",
 306         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 307         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file",
 308         "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
 309         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 310         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 311         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 312         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 313         "digestentry.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 314         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 315         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 316         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 317         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 318         "ebuild.badheader":"This ebuild has a malformed header",
 319         "metadata.missing":"Missing metadata.xml files",
 320         "metadata.bad":"Bad metadata.xml files",
 321         "virtual.versioned":"PROVIDE contains virtuals with versions",
 322         "virtual.exists":"PROVIDE contains existing package names",
 323         "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default",
 324         "usage.obsolete":"The ebuild makes use of an obsolete construct"
 325 }
 326
 327 qacats = qahelp.keys()
 328 qacats.sort()
 329
 330 qawarnings = set((
 331 "changelog.missing",
 332 "changelog.notadded",
 333 "digestentry.unused",
 334 "ebuild.notadded",
 335 "ebuild.nostable",
 336 "ebuild.allmasked",
 337 "ebuild.nesteddie",
 338 "desktop.invalid",
 339 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 340 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 341 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 342 "DESCRIPTION.toolong",
 343 "KEYWORDS.dropped",
 344 "KEYWORDS.stupid",
 345 "KEYWORDS.missing",
 346 "IUSE.undefined",
 347 "RDEPEND.suspect",
 348 "RESTRICT.invalid",
 349 "SRC_URI.mirror",
 350 "ebuild.minorsyn",
 351 "ebuild.badheader",
 352 "ebuild.patches",
 353 "file.size",
 354 "inherit.autotools",
 355 "java.eclassesnotused",
 356 "virtual.versioned",
 357 "virtual.exists",
 358 "virtual.unavailable",
 359 "usage.obsolete",
 360 "LIVEVCS.stable"
 361 ))
 362
 363 missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"]
 364 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 365 allvars.discard("CDEPEND")
 366 allvars.update(Package.metadata_keys)
 367 allvars = sorted(allvars)
 368 commitmessage=None
 369 for x in missingvars:
 370         x += ".missing"
 371         if x not in qacats:
 372                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 373                 qacats.append(x)
 374                 qawarnings.add(x)
 375
 376 valid_restrict = frozenset(["binchecks", "bindist",
 377         "fetch", "installsources", "mirror",
 378         "primaryuri", "strip", "test", "userpriv"])
 379
 380 live_eclasses = frozenset([
 381         "bzr",
 382         "cvs",
 383         "darcs",
 384         "git",
 385         "mercurial",
 386         "subversion"
 387 ])
 388
 389 suspect_rdepend = frozenset([
 390         "app-arch/cabextract",
 391         "app-arch/rpm2targz",
 392         "app-doc/doxygen",
 393         "dev-lang/nasm",
 394         "dev-lang/swig",
 395         "dev-lang/yasm",
 396         "dev-perl/extutils-pkgconfig",
 397         "dev-python/setuptools",
 398         "dev-util/byacc",
 399         "dev-util/cmake",
 400         "dev-util/ftjam",
 401         "dev-util/gtk-doc",
 402         "dev-util/gtk-doc-am",
 403         "dev-util/intltool",
 404         "dev-util/jam",
 405         "dev-util/pkgconfig",
 406         "dev-util/scons",
 407         "dev-util/unifdef",
 408         "dev-util/yacc",
 409         "media-gfx/ebdftopcf",
 410         "sys-apps/help2man",
 411         "sys-devel/autoconf",
 412         "sys-devel/automake",
 413         "sys-devel/bin86",
 414         "sys-devel/bison",
 415         "sys-devel/dev86",
 416         "sys-devel/flex",
 417         "sys-devel/libtool",
 418         "sys-devel/m4",
 419         "sys-devel/pmake",
 420         "x11-misc/bdftopcf",
 421         "x11-misc/imake",
 422 ])
 423
 424 # file.executable
 425 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 426
 427 def last(full=False):
 428         """Print the results of the last repoman run
 429         Args:
 430                 full - Print the complete results, if false, print a summary
 431         Returns:
 432                 Doesn't return (invokes sys.exit()
 433         """
 434         #Retrieve and unpickle stats and fails from saved files
 435         savedf=open(os.path.join(portage.const.CACHE_PATH, 'repo.stats'),'r')
 436         stats = pickle.load(savedf)
 437         savedf.close()
 438         savedf=open(os.path.join(portage.const.CACHE_PATH, 'repo.fails'),'r')
 439         fails = pickle.load(savedf)
 440         savedf.close()
 441
 442         #dofail will be set to 1 if we have failed in at least one non-warning category
 443         dofail=0
 444         #dowarn will be set to 1 if we tripped any warnings
 445         dowarn=0
 446         #dofull will be set if we should print a "repoman full" informational message
 447         dofull=0
 448
 449         dofull = options.mode not in ("full", "lfull")
 450
 451         for x in qacats:
 452                 if not stats[x]:
 453                         continue
 454                 dowarn = 1
 455                 if x not in qawarnings:
 456                         dofail = 1
 457
 458         print
 459         print green("RepoMan remembers...")
 460         print
 461         style_file = ConsoleStyleFile(sys.stdout)
 462         console_writer = StyleWriter(file=style_file, maxcol=9999)
 463         console_writer.style_listener = style_file.new_styles
 464         f = formatter.AbstractFormatter(console_writer)
 465         utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
 466         print
 467         if dofull:
 468                 print bold("Note: type \"repoman lfull\" for a complete listing of repomans last run.")
 469                 print
 470         if dowarn and not dofail:
 471                 print green("RepoMan sez:"),"\"You only gave me a partial QA payment last time?\n              I took it, but I wasn't happy.\""
 472         elif not dofail:
 473                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
 474         print
 475         sys.exit(0)
 476
 477 options, arguments = ParseArgs(sys.argv, qahelp)
 478
 479 if options.mode in ('last', 'lfull'):
 480         last('lfull' in options.mode)
 481
 482 # Set this to False when an extraordinary issue (generally
 483 # something other than a QA issue) makes it impossible to
 484 # commit (like if Manifest generation fails).
 485 can_force = True
 486
 487 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 488 if portdir is None:
 489         sys.exit(1)
 490
 491 vcs = None
 492 if os.path.isdir("CVS"):
 493         vcs = "cvs"
 494 if os.path.isdir(".svn"):
 495         vcs = "svn"
 496 elif os.path.isdir(os.path.join(portdir_overlay, ".git")):
 497         vcs = "git"
 498
 499 if vcs == "cvs" and \
 500         "commit" == options.mode and \
 501         "RMD160" not in portage.checksum.hashorigin_map:
 502         from portage.util import grablines
 503         repo_lines = grablines("./CVS/Repository")
 504         if repo_lines and \
 505                 "gentoo-x86" == repo_lines[0].strip().split(os.path.sep)[0]:
 506                 msg = "Please install " \
 507                 "pycrypto or enable python's ssl USE flag in order " \
 508                 "to enable RMD160 hash support. See bug #198398 for " \
 509                 "more information."
 510                 prefix = bad(" * ")
 511                 from textwrap import wrap
 512                 for line in wrap(msg, 70):
 513                         print prefix + line
 514                 sys.exit(1)
 515         del repo_lines
 516
 517 if options.mode == 'commit' and not options.pretend and not vcs:
 518         logging.info("Not in a version controlled repository; enabling pretend mode.")
 519         options.pretend = True
 520
 521 os.environ["PORTDIR"] = portdir
 522 if portdir_overlay != portdir:
 523         os.environ["PORTDIR_OVERLAY"] = portdir_overlay
 524 else:
 525         os.environ["PORTDIR_OVERLAY"] = ""
 526
 527 logging.info('Setting paths:')
 528 logging.info('PORTDIR = "' + os.environ['PORTDIR'] + '"')
 529 logging.info('PORTDIR_OVERLAY = "' + os.environ['PORTDIR_OVERLAY']+'"')
 530
 531 # Now that PORTDIR_OVERLAY is properly overridden, create the portdb.
 532 repoman_settings = portage.config(local_config=False,
 533         config_incrementals=portage.const.INCREMENTALS)
 534 trees = portage.create_trees()
 535 trees["/"]["porttree"].settings = repoman_settings
 536 portdb = trees["/"]["porttree"].dbapi
 537 portdb.mysettings = repoman_settings
 538 setconfig = load_default_config(repoman_settings, trees["/"])
 539 root_config = RootConfig(repoman_settings, trees["/"], setconfig)
 540 # We really only need to cache the metadata that's necessary for visibility
 541 # filtering. Anything else can be discarded to reduce memory consumption.
 542 portdb._aux_cache_keys.clear()
 543 portdb._aux_cache_keys.update(["EAPI", "KEYWORDS", "SLOT"])
 544 # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman.
 545 del trees["/"]["vartree"]
 546
 547 myreporoot = os.path.basename(portdir_overlay)
 548 myreporoot += mydir[len(portdir_overlay):]
 549
 550 reposplit = myreporoot.split(os.path.sep)
 551 repolevel = len(reposplit)
 552
 553 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 554 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 555 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 556 if options.mode == 'commit' and repolevel not in [1,2,3]:
 557         print red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory."
 558         print red("***")+" Attempting to commit from a packages files directory will be blocked for instance."
 559         print red("***")+" This is intended behaviour, to ensure the manifest is recommited for a package."
 560         print red("***")
 561         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 562
 563 startdir = normalize_path(mydir)
 564 repodir = startdir
 565 for x in range(0, repolevel - 1):
 566         repodir = os.path.dirname(repodir)
 567
 568 def caterror(mycat):
 569         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 570
 571 # setup a uselist from portage
 572 uselist=[]
 573 try:
 574         uselist=portage.grabfile(portdir+"/profiles/use.desc")
 575         for l in range(0,len(uselist)):
 576                 uselist[l]=uselist[l].split()[0]
 577         for var in repoman_settings["USE_EXPAND"].split():
 578                 vardescs = portage.grabfile(portdir+"/profiles/desc/"+var.lower()+".desc")
 579                 for l in range(0, len(vardescs)):
 580                         uselist.append(var.lower() + "_" + vardescs[l].split()[0])
 581 except (IOError, OSError, ParseError), e:
 582         logging.exception("Couldn't read USE flags from use.desc")
 583         sys.exit(1)
 584
 585 # retrieve a list of current licenses in portage
 586 liclist = set(portage.listdir(os.path.join(portdir, "licenses")))
 587 if not liclist:
 588         logging.fatal("Couldn't find licenses?")
 589         sys.exit(1)
 590 if portdir_overlay != portdir:
 591         liclist.update(portage.listdir(os.path.join(portdir_overlay, "licenses")))
 592
 593 # retrieve list of offical keywords
 594 kwlist = set(portage.grabfile(os.path.join(portdir, "profiles", "arch.list")))
 595 if not kwlist:
 596         logging.fatal("Couldn't read KEYWORDS from arch.list")
 597         sys.exit(1)
 598
 599 if portdir_overlay != portdir:
 600         kwlist.update(portage.grabfile(
 601                 os.path.join(portdir_overlay, "profiles", "arch.list")))
 602
 603 scanlist=[]
 604 if repolevel==2:
 605         #we are inside a category directory
 606         catdir=reposplit[-1]
 607         if catdir not in repoman_settings.categories:
 608                 caterror(catdir)
 609         mydirlist=os.listdir(startdir)
 610         for x in mydirlist:
 611                 if x == "CVS" or x.startswith("."):
 612                         continue
 613                 if os.path.isdir(startdir+"/"+x):
 614                         scanlist.append(catdir+"/"+x)
 615 elif repolevel==1:
 616         for x in repoman_settings.categories:
 617                 if not os.path.isdir(startdir+"/"+x):
 618                         continue
 619                 for y in os.listdir(startdir+"/"+x):
 620                         if y == "CVS" or y.startswith("."):
 621                                 continue
 622                         if os.path.isdir(startdir+"/"+x+"/"+y):
 623                                 scanlist.append(x+"/"+y)
 624 elif repolevel==3:
 625         catdir = reposplit[-2]
 626         if catdir not in repoman_settings.categories:
 627                 caterror(catdir)
 628         scanlist.append(catdir+"/"+reposplit[-1])
 629 scanlist.sort()
 630
 631 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 632
 633 profiles={}
 634 valid_profile_types = frozenset(["dev", "exp", "stable"])
 635 descfile=portdir+"/profiles/profiles.desc"
 636 if os.path.exists(descfile):
 637         for i, x in enumerate(open(descfile, 'rb')):
 638                 if x[0]=="#":
 639                         continue
 640                 arch=x.split()
 641                 if len(arch) == 0:
 642                         continue
 643                 if len(arch)!=3:
 644                         err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 645                                 descfile + " line %d" % (i+1, ))
 646                 elif arch[0] not in kwlist:
 647                         err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 648                                 descfile + " line %d" % (i+1, ))
 649                 elif arch[2] not in valid_profile_types:
 650                         err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 651                                 descfile + " line %d" % (i+1, ))
 652                 if not os.path.isdir(portdir+"/profiles/"+arch[1]):
 653                         print "Invalid "+arch[2]+" profile ("+arch[1]+") for arch "+arch[0]
 654                         continue
 655                 if arch[0] in profiles:
 656                         profiles[arch[0]]+= [[arch[1], arch[2]]]
 657                 else:
 658                         profiles[arch[0]] = [[arch[1], arch[2]]]
 659
 660         for x in repoman_settings.archlist():
 661                 if x[0] == "~":
 662                         continue
 663                 if x not in profiles:
 664                         print red("\""+x+"\" doesn't have a valid profile listed in profiles.desc.")
 665                         print red("You need to either \"cvs update\" your profiles dir or follow this")
 666                         print red("up with the "+x+" team.")
 667                         print
 668 else:
 669         print red("profiles.desc does not exist: "+descfile)
 670         print red("You need to do \"cvs update\" in profiles dir.")
 671         print
 672         sys.exit(1)
 673
 674 def dev_keywords(profiles):
 675         """
 676         Create a set of KEYWORDS values that exist in 'dev'
 677         profiles. These are used
 678         to trigger a message notifying the user when they might
 679         want to add the --include-dev option.
 680         """
 681         type_arch_map = {}
 682         for arch, arch_profiles in profiles.iteritems():
 683                 for profile_path, profile_type in arch_profiles:
 684                         arch_set = type_arch_map.get(profile_type)
 685                         if arch_set is None:
 686                                 arch_set = set()
 687                                 type_arch_map[profile_type] = arch_set
 688                         arch_set.add(arch)
 689
 690         dev_keywords = type_arch_map.get('dev', set())
 691         dev_keywords.update(['~' + arch for arch in dev_keywords])
 692         return frozenset(dev_keywords)
 693
 694 dev_keywords = dev_keywords(profiles)
 695
 696 stats={}
 697 fails={}
 698
 699 # provided by the desktop-file-utils package
 700 desktop_file_validate = find_binary("desktop-file-validate")
 701 desktop_pattern = re.compile(r'.*\.desktop$')
 702
 703 for x in qacats:
 704         stats[x]=0
 705         fails[x]=[]
 706 xmllint_capable = False
 707 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
 708 if options.mode == "manifest":
 709         pass
 710 elif not find_binary('xmllint'):
 711         print red("!!! xmllint not found. Can't check metadata.xml.\n")
 712         if options.xml_parse or repolevel==3:
 713                 print red("!!!")+" sorry, xmllint is needed.  failing\n"
 714                 sys.exit(1)
 715 else:
 716         #hardcoded paths/urls suck. :-/
 717         must_fetch=1
 718         backup_exists=0
 719         try:
 720                 # if it's been over a week since fetching (or the system clock is fscked), grab an updated copy of metadata.dtd
 721                 # clock is fscked or it's been a week. time to grab a new one.
 722                 ct = os.stat(metadata_dtd)[ST_CTIME]
 723                 if abs(time.time() - ct) > (60*60*24*7):
 724                         # don't trap the exception, we're watching for errno 2 (file not found), anything else is a bug.
 725                         backup_exists=1
 726                 else:
 727                         must_fetch=0
 728
 729         except (OSError,IOError), e:
 730                 if e.errno != 2:
 731                         print red("!!!")+" caught exception '%s' for %s/metadata.dtd, bailing" % (str(e), portage.CACHE_PATH)
 732                         sys.exit(1)
 733
 734         if must_fetch:
 735                 print
 736                 print green("***")+" the local copy of metadata.dtd needs to be refetched, doing that now"
 737                 print
 738                 val = 0
 739                 try:
 740                         try:
 741                                 os.unlink(metadata_dtd)
 742                         except OSError, e:
 743                                 if e.errno != errno.ENOENT:
 744                                         raise
 745                                 del e
 746                         val=portage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoman_settings,fetchonly=0, \
 747                                 try_mirrors=0)
 748
 749                 except SystemExit, e:
 750                         raise  # Need to propogate this
 751                 except Exception,e:
 752                         print
 753                         print red("!!!")+" attempting to fetch 'http://www.gentoo.org/dtd/metadata.dtd', caught"
 754                         print red("!!!")+" exception '%s' though." % str(e)
 755                         val=0
 756                 if not val:
 757                         print red("!!!")+" fetching new metadata.dtd failed, aborting"
 758                         sys.exit(1)
 759         #this can be problematic if xmllint changes their output
 760         xmllint_capable=True
 761
 762 if options.mode == 'commit' and vcs:
 763         utilities.detect_vcs_conflicts(options, vcs)
 764
 765 if options.mode == "manifest":
 766         pass
 767 elif options.pretend:
 768         print green("\nRepoMan does a once-over of the neighborhood...")
 769 else:
 770         print green("\nRepoMan scours the neighborhood...")
 771
 772 new_ebuilds = set()
 773 modified_changelogs = set()
 774 mychanged = []
 775 mynew = []
 776 myremoved = []
 777 path_lstrip_re = re.compile(r'.*/')
 778
 779 if vcs == "cvs":
 780         mycvstree = cvstree.getentries("./", recursive=1)
 781         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
 782         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
 783 if vcs == "svn":
 784         svnstatus = os.popen("svn status").readlines()
 785         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
 786         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
 787 elif vcs == "git":
 788         mychanged = os.popen("git ls-files -m").readlines()
 789         mychanged = [ "./" +  elem[:-1] for elem in mychanged ]
 790         mynew = os.popen("git diff --cached --name-only --diff-filter=A").readlines()
 791         strip_levels = repolevel - 1
 792         if strip_levels:
 793                 mynew = [path_lstrip_re.sub("", elem, strip_levels) for elem in mynew]
 794         mynew = ["./" + elem[:-1] for elem in mynew]
 795 if vcs:
 796         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
 797         modified_changelogs.update(x for x in chain(mychanged, mynew) \
 798                 if os.path.basename(x) == "ChangeLog")
 799
 800 have_pmasked = False
 801 have_dev_keywords = False
 802 dofail = 0
 803 arch_caches={}
 804 arch_xmatch_caches = {}
 805 shared_xmatch_caches = {"cp-list":{}}
 806
 807 # Disable the "ebuild.notadded" check when not in commit mode and
 808 # running `svn status` in every package dir will be too expensive.
 809
 810 check_ebuild_notadded = not \
 811         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
 812
 813 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
 814 thirdpartymirrors = portage.flatten(repoman_settings.thirdpartymirrors().values())
 815
 816 for x in scanlist:
 817         #ebuilds and digests added to cvs respectively.
 818         logging.info("checking package %s" % x)
 819         eadded=[]
 820         catdir,pkgdir=x.split("/")
 821         checkdir=repodir+"/"+x
 822         checkdir_relative = ""
 823         if repolevel < 3:
 824                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
 825         if repolevel < 2:
 826                 checkdir_relative = os.path.join(catdir, checkdir_relative)
 827         checkdir_relative = os.path.join(".", checkdir_relative)
 828
 829         if options.mode == "manifest" or \
 830           options.mode in ('commit', 'fix') and not options.pretend:
 831                 repoman_settings["O"] = checkdir
 832                 if not portage.digestgen([], repoman_settings, myportdb=portdb):
 833                         print "Unable to generate manifest."
 834                         dofail = 1
 835                 if options.mode == "manifest":
 836                         continue
 837                 elif dofail:
 838                         sys.exit(1)
 839
 840         checkdirlist=os.listdir(checkdir)
 841         ebuildlist=[]
 842         pkgs = {}
 843         for y in checkdirlist:
 844                 if y in no_exec and \
 845                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0111:
 846                                 stats["file.executable"] += 1
 847                                 fails["file.executable"].append(os.path.join(checkdir, y))
 848                 if y.endswith(".ebuild"):
 849                         pf = y[:-7]
 850                         ebuildlist.append(pf)
 851                         cpv = "%s/%s" % (catdir, pf)
 852                         try:
 853                                 myaux = dict(izip(allvars, portdb.aux_get(cpv, allvars)))
 854                         except KeyError:
 855                                 stats["ebuild.syntax"] += 1
 856                                 fails["ebuild.syntax"].append(os.path.join(x, y))
 857                                 continue
 858                         except IOError:
 859                                 stats["ebuild.output"] += 1
 860                                 fails["ebuild.output"].append(os.path.join(x, y))
 861                                 continue
 862                         if not portage.eapi_is_supported(myaux["EAPI"]):
 863                                 stats["EAPI.unsupported"] += 1
 864                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
 865                                 continue
 866                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
 867                                 root_config=root_config)
 868
 869         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
 870         pkgsplits = {}
 871         for i in xrange(len(ebuildlist)):
 872                 ebuild_split = portage.pkgsplit(ebuildlist[i])
 873                 pkgsplits[ebuild_split] = ebuildlist[i]
 874                 ebuildlist[i] = ebuild_split
 875         ebuildlist.sort(portage.pkgcmp)
 876         for i in xrange(len(ebuildlist)):
 877                 ebuildlist[i] = pkgsplits[ebuildlist[i]]
 878         del pkgsplits
 879
 880         slot_keywords = {}
 881
 882         if len(pkgs) != len(ebuildlist):
 883                 # If we can't access all the metadata then it's totally unsafe to
 884                 # commit since there's no way to generate a correct Manifest.
 885                 # Do not try to do any more QA checks on this package since missing
 886                 # metadata leads to false positives for several checks, and false
 887                 # positives confuse users.
 888                 can_force = False
 889                 continue
 890
 891         for y in checkdirlist:
 892                 for c in y.strip(os.path.sep):
 893                         if c not in allowed_filename_chars_set:
 894                                 stats["file.name"] += 1
 895                                 fails["file.name"].append("%s/%s: char '%s'" % (checkdir, y, c))
 896                                 break
 897
 898                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
 899                         continue
 900                 try:
 901                         line = 1
 902                         for l in codecs.open(checkdir+"/"+y, "r", "utf8"):
 903                                 line +=1
 904                 except UnicodeDecodeError, ue:
 905                         stats["file.UTF8"] += 1
 906                         s = ue.object[:ue.start]
 907                         l2 = s.count("\n")
 908                         line += l2
 909                         if l2 != 0:
 910                                 s = s[s.rfind("\n") + 1:]
 911                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
 912
 913         if vcs == "git" and check_ebuild_notadded:
 914                 myf = os.popen("git ls-files --others %s" % \
 915                         (portage._shell_quote(checkdir_relative),))
 916                 for l in myf:
 917                         if l[:-1][-7:] == ".ebuild":
 918                                 stats["ebuild.notadded"] += 1
 919                                 fails["ebuild.notadded"].append(
 920                                         os.path.join(x, os.path.basename(l[:-1])))
 921                 myf.close()
 922
 923         if vcs in ("cvs", "svn") and check_ebuild_notadded:
 924                 try:
 925                         if vcs == "cvs":
 926                                 myf=open(checkdir+"/CVS/Entries","r")
 927                         if vcs == "svn":
 928                                 myf = os.popen("svn status --depth=files --verbose " + checkdir)
 929                         myl = myf.readlines()
 930                         myf.close()
 931                         for l in myl:
 932                                 if vcs == "cvs":
 933                                         if l[0]!="/":
 934                                                 continue
 935                                         splitl=l[1:].split("/")
 936                                         if not len(splitl):
 937                                                 continue
 938                                         if splitl[0][-7:]==".ebuild":
 939                                                 eadded.append(splitl[0][:-7])
 940                                 if vcs == "svn":
 941                                         if l[:1] == "?":
 942                                                 continue
 943                                         l = l.split()[-1]
 944                                         if l[-7:] == ".ebuild":
 945                                                 eadded.append(os.path.basename(l[:-7]))
 946                         if vcs == "svn":
 947                                 myf = os.popen("svn status " + checkdir)
 948                                 myl=myf.readlines()
 949                                 myf.close()
 950                                 for l in myl:
 951                                         if l[0] == "A":
 952                                                 l = l.rstrip().split(' ')[-1]
 953                                                 if l[-7:] == ".ebuild":
 954                                                         eadded.append(os.path.basename(l[:-7]))
 955                 except IOError:
 956                         if options.mode == 'commit' and vcs == "cvs":
 957                                 stats["CVS/Entries.IO_error"] += 1
 958                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
 959                         if options.mode == 'commit' and vcs == "svn":
 960                                 stats["svn.IO_error"] += 1
 961                                 fails["svn.IO_error"].append(checkdir+"svn info")
 962                         continue
 963
 964         mf = Manifest(checkdir, repoman_settings["DISTDIR"])
 965         mydigests=mf.getTypeDigests("DIST")
 966
 967         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
 968         myfiles_all = []
 969         src_uri_error = False
 970         for mykey in fetchlist_dict:
 971                 try:
 972                         myfiles_all.extend(fetchlist_dict[mykey])
 973                 except portage.exception.InvalidDependString, e:
 974                         src_uri_error = True
 975                         try:
 976                                 portdb.aux_get(mykey, ["SRC_URI"])
 977                         except KeyError:
 978                                 # This will be reported as an "ebuild.syntax" error.
 979                                 pass
 980                         else:
 981                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
 982                                 fails["SRC_URI.syntax"].append(
 983                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
 984         del fetchlist_dict
 985         if not src_uri_error:
 986                 # This test can produce false positives if SRC_URI could not
 987                 # be parsed for one or more ebuilds. There's no point in
 988                 # producing a false error here since the root cause will
 989                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
 990                 # or "ebuild.sytax".
 991                 myfiles_all = set(myfiles_all)
 992                 for entry in mydigests:
 993                         if entry not in myfiles_all:
 994                                 stats["digestentry.unused"] += 1
 995                                 fails["digestentry.unused"].append(checkdir+"::"+entry)
 996         del myfiles_all
 997
 998         if os.path.exists(checkdir+"/files"):
 999                 filesdirlist=os.listdir(checkdir+"/files")
1000
1001                 # recurse through files directory
1002                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1003                 while filesdirlist:
1004                         y = filesdirlist.pop(0)
1005                         relative_path = os.path.join(x, "files", y)
1006                         full_path = os.path.join(repodir, relative_path)
1007                         try:
1008                                 mystat = os.stat(full_path)
1009                         except OSError, oe:
1010                                 if oe.errno == 2:
1011                                         # don't worry about it.  it likely was removed via fix above.
1012                                         continue
1013                                 else:
1014                                         raise oe
1015                         if S_ISDIR(mystat.st_mode):
1016                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1017                                 if y == "CVS" or y == ".svn":
1018                                         continue
1019                                 for z in os.listdir(checkdir+"/files/"+y):
1020                                         if z == "CVS" or z == ".svn":
1021                                                 continue
1022                                         filesdirlist.append(y+"/"+z)
1023                         # current policy is no files over 20k, this is the check.
1024                         elif mystat.st_size > 20480:
1025                                 stats["file.size"] += 1
1026                                 fails["file.size"].append("("+ str(mystat.st_size/1024) + "K) "+x+"/files/"+y)
1027
1028                         for c in os.path.basename(y.rstrip(os.path.sep)):
1029                                 if c not in allowed_filename_chars_set:
1030                                         stats["file.name"] += 1
1031                                         fails["file.name"].append("%s/files/%s: char '%s'" % (checkdir, y, c))
1032                                         break
1033
1034                         if desktop_file_validate and desktop_pattern.match(y):
1035                                 status, cmd_output = commands.getstatusoutput(
1036                                         "'%s' '%s'" % (desktop_file_validate, full_path))
1037                                 if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK:
1038                                         # Note: in the future we may want to grab the
1039                                         # warnings in addition to the errors. We're
1040                                         # just doing errors now since we don't want
1041                                         # to generate too much noise at first.
1042                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1043                                         for line in cmd_output.splitlines():
1044                                                 error_match = error_re.match(line)
1045                                                 if error_match is None:
1046                                                         continue
1047                                                 stats["desktop.invalid"] += 1
1048                                                 fails["desktop.invalid"].append(
1049                                                         relative_path + ': %s' % error_match.group(1))
1050
1051         del mydigests
1052
1053         if "ChangeLog" not in checkdirlist:
1054                 stats["changelog.missing"]+=1
1055                 fails["changelog.missing"].append(x+"/ChangeLog")
1056
1057         #metadata.xml file check
1058         muselist = []
1059
1060         if "metadata.xml" not in checkdirlist:
1061                 stats["metadata.missing"]+=1
1062                 fails["metadata.missing"].append(x+"/metadata.xml")
1063         #metadata.xml parse check
1064         else:
1065                 metadata_bad = False
1066
1067                 # load USE flags from metadata.xml
1068                 try:
1069                         f = open(os.path.join(checkdir, "metadata.xml"))
1070                         utilities.parse_metadata_use(f, muselist)
1071                         f.close()
1072                 except (EnvironmentError, ParseError), e:
1073                         metadata_bad = True
1074                         stats["metadata.bad"] += 1
1075                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1076                         del e
1077
1078                 #Only carry out if in package directory or check forced
1079                 if xmllint_capable and not metadata_bad:
1080                         # xmlint can produce garbage output even on success, so only dump
1081                         # the ouput when it fails.
1082                         st, out = commands.getstatusoutput(
1083                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1084                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1085                         if st != os.EX_OK:
1086                                 print red("!!!") + " metadata.xml is invalid:"
1087                                 for z in out.splitlines():
1088                                         print red("!!! ")+z
1089                                 stats["metadata.bad"]+=1
1090                                 fails["metadata.bad"].append(x+"/metadata.xml")
1091
1092                 del metadata_bad
1093
1094         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1095         changelog_modified = changelog_path in modified_changelogs
1096
1097         allmasked = True
1098
1099         for y in ebuildlist:
1100                 relative_path = os.path.join(x, y + ".ebuild")
1101                 full_path = os.path.join(repodir, relative_path)
1102                 ebuild_path = y + ".ebuild"
1103                 if repolevel < 3:
1104                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1105                 if repolevel < 2:
1106                         ebuild_path = os.path.join(catdir, ebuild_path)
1107                 ebuild_path = os.path.join(".", ebuild_path)
1108                 if not changelog_modified and ebuild_path in new_ebuilds:
1109                         stats['changelog.ebuildadded'] += 1
1110                         fails['changelog.ebuildadded'].append(relative_path)
1111
1112                 if stat.S_IMODE(os.stat(full_path).st_mode) & 0111:
1113                         stats["file.executable"] += 1
1114                         fails["file.executable"].append(x+"/"+y+".ebuild")
1115                 if vcs in ("cvs", "svn") and check_ebuild_notadded and y not in eadded:
1116                         #ebuild not added to vcs
1117                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1118                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1119                 myesplit=portage.pkgsplit(y)
1120                 if myesplit is None or myesplit[0] != x.split("/")[-1]:
1121                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1122                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1123                         continue
1124                 elif myesplit[0]!=pkgdir:
1125                         print pkgdir,myesplit[0]
1126                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1127                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1128                         continue
1129
1130                 pkg = pkgs[y]
1131                 myaux = pkg.metadata
1132                 eapi = myaux["EAPI"]
1133                 inherited = pkg.inherited
1134                 live_ebuild = live_eclasses.intersection(inherited)
1135
1136                 if not src_uri_error:
1137                         # Check that URIs don't reference a server from thirdpartymirrors.
1138                         for uri in portage.flatten(portage.dep.use_reduce(
1139                                 portage.dep.paren_reduce(myaux["SRC_URI"]), matchall=True)):
1140                                 contains_mirror = False
1141                                 for mirror in thirdpartymirrors:
1142                                         if uri.startswith(mirror):
1143                                                 contains_mirror = True
1144                                                 break
1145                                 if not contains_mirror:
1146                                         continue
1147
1148                                 stats["SRC_URI.mirror"] += 1
1149                                 fails["SRC_URI.mirror"].append(
1150                                         "%s: '%s' found in thirdpartymirrors" % \
1151                                         (relative_path, mirror))
1152
1153                 # Test for negative logic and bad words in the RESTRICT var.
1154                 #for x in myaux[allvars.index("RESTRICT")].split():
1155                 #       if x.startswith("no"):
1156                 #               print "Bad RESTRICT value: %s" % x
1157                 try:
1158                         myaux["PROVIDE"] = portage.dep.use_reduce(
1159                                 portage.dep.paren_reduce(myaux["PROVIDE"]), matchall=1)
1160                 except portage.exception.InvalidDependString, e:
1161                         stats["PROVIDE.syntax"] = stats["PROVIDE.syntax"] + 1
1162                         fails["PROVIDE.syntax"].append(mykey+".ebuild PROVIDE: "+str(e))
1163                         del e
1164                         continue
1165                 myaux["PROVIDE"] = " ".join(portage.flatten(myaux["PROVIDE"]))
1166                 for myprovide in myaux["PROVIDE"].split():
1167                         prov_cp = portage.dep_getkey(myprovide)
1168                         if prov_cp != myprovide:
1169                                 stats["virtual.versioned"]+=1
1170                                 fails["virtual.versioned"].append(x+"/"+y+".ebuild: "+myprovide)
1171                         prov_pkg = portage.dep_getkey(
1172                                 portage.best(portdb.xmatch("match-all", prov_cp)))
1173                         if prov_cp == prov_pkg:
1174                                 stats["virtual.exists"]+=1
1175                                 fails["virtual.exists"].append(x+"/"+y+".ebuild: "+prov_cp)
1176
1177                 for pos, missing_var in enumerate(missingvars):
1178                         if not myaux.get(missing_var):
1179                                 if catdir == "virtual" and \
1180                                         missing_var in ("HOMEPAGE", "LICENSE"):
1181                                         continue
1182                                 if live_ebuild and missing_var == "KEYWORDS":
1183                                         continue
1184                                 myqakey=missingvars[pos]+".missing"
1185                                 stats[myqakey]=stats[myqakey]+1
1186                                 fails[myqakey].append(x+"/"+y+".ebuild")
1187
1188                 # 14 is the length of DESCRIPTION=""
1189                 if len(myaux['DESCRIPTION']) > max_desc_len:
1190                         stats['DESCRIPTION.toolong'] += 1
1191                         fails['DESCRIPTION.toolong'].append(
1192                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1193                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1194
1195                 keywords = myaux["KEYWORDS"].split()
1196                 stable_keywords = []
1197                 for keyword in keywords:
1198                         if not keyword.startswith("~") and \
1199                                 not keyword.startswith("-"):
1200                                 stable_keywords.append(keyword)
1201                 if stable_keywords:
1202                         if ebuild_path in new_ebuilds:
1203                                 stable_keywords.sort()
1204                                 stats["KEYWORDS.stable"] += 1
1205                                 fails["KEYWORDS.stable"].append(
1206                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1207                                                 " ".join(stable_keywords))
1208
1209                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1210                         if not kw.startswith("-"))
1211
1212                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1213                 if previous_keywords is None:
1214                         slot_keywords[myaux["SLOT"]] = set()
1215                 elif not live_ebuild:
1216                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1217                         if dropped_keywords:
1218                                 stats["KEYWORDS.dropped"] += 1
1219                                 fails["KEYWORDS.dropped"].append(
1220                                         relative_path + ": %s" % \
1221                                         " ".join(sorted(dropped_keywords)))
1222
1223                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1224
1225                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1226                 if "-*" in keywords:
1227                         haskeyword = False
1228                         for kw in keywords:
1229                                 if kw[0] == "~":
1230                                         kw = kw[1:]
1231                                 if kw in kwlist:
1232                                         haskeyword = True
1233                         if not haskeyword:
1234                                 stats["KEYWORDS.stupid"] += 1
1235                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1236
1237                 """
1238                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1239                 not be allowed to be marked stable
1240                 """
1241                 if live_ebuild:
1242                         bad_stable_keywords = []
1243                         for keyword in keywords:
1244                                 if not keyword.startswith("~") and \
1245                                         not keyword.startswith("-"):
1246                                         bad_stable_keywords.append(keyword)
1247                                 del keyword
1248                         if bad_stable_keywords:
1249                                 stats["LIVEVCS.stable"] += 1
1250                                 fails["LIVEVCS.stable"].append(
1251                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1252                                                 bad_stable_keywords)
1253                         del bad_stable_keywords
1254
1255                 if options.ignore_arches:
1256                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1257                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1258                 else:
1259                         arches=[]
1260                         for keyword in myaux["KEYWORDS"].split():
1261                                 if (keyword[0]=="-"):
1262                                         continue
1263                                 elif (keyword[0]=="~"):
1264                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1265                                 else:
1266                                         arches.append([keyword, keyword, [keyword]])
1267                                         allmasked = False
1268
1269                 baddepsyntax = False
1270                 badlicsyntax = False
1271                 badprovsyntax = False
1272                 catpkg = catdir+"/"+y
1273                 myiuse = set(repoman_settings.archlist())
1274                 for myflag in myaux["IUSE"].split():
1275                         if myflag.startswith("+"):
1276                                 myflag = myflag[1:]
1277                         myiuse.add(myflag)
1278
1279                 inherited_java_eclass = "java-pkg-2" in inherited or \
1280                         "java-pkg-opt-2" in inherited
1281                 operator_tokens = set(["||", "(", ")"])
1282                 type_list, badsyntax = [], []
1283                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1284                         "LICENSE", "PROPERTIES", "PROVIDE"):
1285                         mydepstr = myaux[mytype]
1286
1287                         if mydepstr.find(" ?") != -1:
1288                                 badsyntax.append("'?' preceded by space")
1289
1290                         try:
1291                                 # Missing closing parenthesis will result in a ValueError
1292                                 mydeplist = portage.dep.paren_reduce(mydepstr)
1293                                 # Missing opening parenthesis will result in a final "" element
1294                                 if "" in mydeplist or "(" in mydeplist:
1295                                         raise ValueError
1296                         except ValueError:
1297                                 badsyntax.append("parenthesis mismatch")
1298                                 mydeplist = []
1299                         except portage.exception.InvalidDependString, e:
1300                                 badsyntax.append(str(e))
1301                                 del e
1302                                 mydeplist = []
1303
1304                         try:
1305                                 portage.dep.use_reduce(mydeplist, matchall=1)
1306                         except portage.exception.InvalidDependString, e:
1307                                 badsyntax.append(str(e))
1308
1309                         for token in operator_tokens:
1310                                 if mydepstr.startswith(token+" "):
1311                                         myteststr = mydepstr[len(token):]
1312                                 else:
1313                                         myteststr = mydepstr
1314                                 if myteststr.endswith(" "+token):
1315                                         myteststr = myteststr[:-len(token)]
1316                                 while myteststr.find(" "+token+" ") != -1:
1317                                         myteststr = " ".join(myteststr.split(" "+token+" ", 1))
1318                                 if myteststr.find(token) != -1:
1319                                         badsyntax.append("'%s' not separated by space" % (token))
1320
1321                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1322                                 for token in mydepstr.split():
1323                                         if token in operator_tokens or \
1324                                                 token.endswith("?"):
1325                                                 continue
1326                                         try:
1327                                                 atom = portage.dep.Atom(token)
1328                                         except portage.exception.InvalidAtom:
1329                                                 badsyntax.append("'%s' not a valid atom" % token)
1330                                         else:
1331                                                 is_blocker = atom.blocker
1332
1333                                                 if mytype == "DEPEND" and \
1334                                                         not is_blocker and \
1335                                                         not inherited_java_eclass and \
1336                                                         portage.dep_getkey(atom) == "virtual/jdk":
1337                                                         stats['java.eclassesnotused'] += 1
1338                                                         fails['java.eclassesnotused'].append(relative_path)
1339                                                 elif mytype == "RDEPEND":
1340                                                         if not is_blocker and \
1341                                                                 portage.dep_getkey(atom) in suspect_rdepend:
1342                                                                 stats['RDEPEND.suspect'] += 1
1343                                                                 fails['RDEPEND.suspect'].append(
1344                                                                         relative_path + ": '%s'" % atom)
1345                                                 if eapi == "0":
1346                                                         if portage.dep.dep_getslot(atom):
1347                                                                 stats['EAPI.incompatible'] += 1
1348                                                                 fails['EAPI.incompatible'].append(
1349                                                                         (relative_path + ": %s slot dependency" + \
1350                                                                         " not supported with EAPI='%s':" + \
1351                                                                         " '%s'") % (mytype, eapi, atom))
1352                                                 if atom.use and eapi in ("0", "1"):
1353                                                         stats['EAPI.incompatible'] += 1
1354                                                         fails['EAPI.incompatible'].append(
1355                                                                 (relative_path + ": %s use dependency" + \
1356                                                                 " not supported with EAPI='%s':" + \
1357                                                                 " '%s'") % (mytype, eapi, atom))
1358                                                 if atom.blocker and atom.blocker.overlap.forbid \
1359                                                         and eapi in ("0", "1"):
1360                                                         stats['EAPI.incompatible'] += 1
1361                                                         fails['EAPI.incompatible'].append(
1362                                                                 (relative_path + ": %s new blocker syntax" + \
1363                                                                 " not supported with EAPI='%s':" + \
1364                                                                 " '%s'") % (mytype, eapi, atom))
1365
1366                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1367
1368                 for m,b in zip(type_list, badsyntax):
1369                         stats[m+".syntax"] += 1
1370                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1371
1372                 badlicsyntax = len(filter(lambda x:x=="LICENSE", type_list))
1373                 badprovsyntax = len(filter(lambda x:x=="PROVIDE", type_list))
1374                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1375                 badlicsyntax = badlicsyntax > 0
1376                 badprovsyntax = badprovsyntax > 0
1377
1378                 # uselist checks - global
1379                 myuse = []
1380                 default_use = []
1381                 for myflag in myaux["IUSE"].split():
1382                         flag_name = myflag.lstrip("+-")
1383                         if myflag != flag_name:
1384                                 default_use.append(myflag)
1385                         if flag_name not in uselist:
1386                                 myuse.append(flag_name)
1387
1388                 # uselist checks - metadata
1389                 for mypos in range(len(myuse)-1,-1,-1):
1390                         if myuse[mypos] and (myuse[mypos] in muselist):
1391                                 del myuse[mypos]
1392
1393                 if default_use and eapi == "0":
1394                         for myflag in default_use:
1395                                 stats['EAPI.incompatible'] += 1
1396                                 fails['EAPI.incompatible'].append(
1397                                         (relative_path + ": IUSE defaults" + \
1398                                         " not supported with EAPI='%s':" + \
1399                                         " '%s'") % (eapi, myflag))
1400
1401                 for mypos in range(len(myuse)):
1402                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1403                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1404
1405                 # license checks
1406                 if not badlicsyntax:
1407                         myuse = myaux["LICENSE"]
1408                         # Parse the LICENSE variable, remove USE conditions and
1409                         # flatten it.
1410                         myuse=portage.dep.use_reduce(portage.dep.paren_reduce(myuse), matchall=1)
1411                         myuse=portage.flatten(myuse)
1412                         # Check each entry to ensure that it exists in PORTDIR's
1413                         # license directory.
1414                         for mypos in range(0,len(myuse)):
1415                                 # Need to check for "||" manually as no portage
1416                                 # function will remove it without removing values.
1417                                 if myuse[mypos] not in liclist and myuse[mypos] != "||":
1418                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1419                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1420
1421                 #keyword checks
1422                 myuse = myaux["KEYWORDS"].split()
1423                 for mykey in myuse:
1424                         myskey=mykey[:]
1425                         if myskey[0]=="-":
1426                                 myskey=myskey[1:]
1427                         if myskey[0]=="~":
1428                                 myskey=myskey[1:]
1429                         if mykey!="-*":
1430                                 if myskey not in kwlist:
1431                                         stats["KEYWORDS.invalid"] += 1
1432                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1433                                 elif myskey not in profiles:
1434                                         stats["KEYWORDS.invalid"] += 1
1435                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1436
1437                 #restrict checks
1438                 myrestrict = None
1439                 try:
1440                         myrestrict = portage.dep.use_reduce(
1441                                 portage.dep.paren_reduce(myaux["RESTRICT"]), matchall=1)
1442                 except portage.exception.InvalidDependString, e:
1443                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1444                         fails["RESTRICT.syntax"].append(
1445                                 "%s: RESTRICT: %s" % (relative_path, e))
1446                         del e
1447                 if myrestrict:
1448                         myrestrict = set(portage.flatten(myrestrict))
1449                         mybadrestrict = myrestrict.difference(valid_restrict)
1450                         if mybadrestrict:
1451                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
1452                                 for mybad in mybadrestrict:
1453                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1454                 # Syntax Checks
1455                 relative_path = os.path.join(x, y + ".ebuild")
1456                 full_path = os.path.join(repodir, relative_path)
1457                 f = open(full_path, 'rb')
1458                 try:
1459                         for check_name, e in run_checks(f, pkg):
1460                                 stats[check_name] += 1
1461                                 fails[check_name].append(relative_path + ': %s' % e)
1462                 finally:
1463                         f.close()
1464                         del f
1465
1466                 if options.force:
1467                         # The dep_check() calls are the most expensive QA test. If --force
1468                         # is enabled, there's no point in wasting time on these since the
1469                         # user is intent on forcing the commit anyway.
1470                         continue
1471
1472                 for keyword,arch,groups in arches:
1473
1474                         if arch not in profiles:
1475                                 # A missing profile will create an error further down
1476                                 # during the KEYWORDS verification.
1477                                 continue
1478
1479                         for prof in profiles[arch]:
1480
1481                                 if prof[1] not in ("stable", "dev") or \
1482                                         prof[1] == "dev" and not options.include_dev:
1483                                         continue
1484
1485                                 profdir = portdir+"/profiles/"+prof[0]
1486
1487                                 if prof[0] in arch_caches:
1488                                         dep_settings = arch_caches[prof[0]]
1489                                 else:
1490                                         dep_settings = portage.config(
1491                                                 config_profile_path=profdir,
1492                                                 config_incrementals=portage.const.INCREMENTALS,
1493                                                 local_config=False)
1494                                         if options.without_mask:
1495                                                 dep_settings.pmaskdict.clear()
1496                                         arch_caches[prof[0]] = dep_settings
1497                                         while True:
1498                                                 try:
1499                                                         # Protect ACCEPT_KEYWORDS from config.regenerate()
1500                                                         # (just in case)
1501                                                         dep_settings.incrementals.remove("ACCEPT_KEYWORDS")
1502                                                 except ValueError:
1503                                                         break
1504
1505                                 xmatch_cache_key = (prof[0], tuple(groups))
1506                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
1507                                 if xcache is None:
1508                                         portdb.melt()
1509                                         portdb.freeze()
1510                                         xcache = portdb.xcache
1511                                         xcache.update(shared_xmatch_caches)
1512                                         arch_xmatch_caches[xmatch_cache_key] = xcache
1513
1514                                 trees["/"]["porttree"].settings = dep_settings
1515                                 portdb.mysettings = dep_settings
1516                                 portdb.xcache = xcache
1517                                 # for package.use.mask support inside dep_check
1518                                 dep_settings.setcpv(pkg)
1519                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1520                                 # just in case, prevent config.reset() from nuking these.
1521                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1522
1523                                 for myprovide in myaux["PROVIDE"].split():
1524                                         prov_cp = portage.dep_getkey(myprovide)
1525                                         if prov_cp not in dep_settings.getvirtuals():
1526                                                 stats["virtual.unavailable"]+=1
1527                                                 fails["virtual.unavailable"].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+prov_cp)
1528
1529                                 if not baddepsyntax:
1530                                         ismasked = os.path.join(catdir, y) not in \
1531                                                 portdb.xmatch("list-visible", x)
1532                                         if ismasked:
1533                                                 if not have_pmasked:
1534                                                         have_pmasked = bool(dep_settings._getMaskAtom(
1535                                                                 pkg.cpv, pkg.metadata))
1536                                                 if options.ignore_masked:
1537                                                         continue
1538                                                 #we are testing deps for a masked package; give it some lee-way
1539                                                 suffix="masked"
1540                                                 matchmode = "minimum-all"
1541                                         else:
1542                                                 suffix=""
1543                                                 matchmode = "minimum-visible"
1544
1545                                         if not have_dev_keywords:
1546                                                 have_dev_keywords = \
1547                                                         bool(dev_keywords.intersection(keywords))
1548
1549                                         if prof[1] == "dev":
1550                                                 suffix=suffix+"indev"
1551
1552                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1553
1554                                                 mykey=mytype+".bad"+suffix
1555                                                 myvalue = myaux[mytype]
1556                                                 if not myvalue:
1557                                                         continue
1558                                                 try:
1559                                                         mydep = portage.dep_check(myvalue, portdb,
1560                                                                 dep_settings, use="all", mode=matchmode,
1561                                                                 trees=trees)
1562                                                 except KeyError, e:
1563                                                         stats[mykey]=stats[mykey]+1
1564                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(e[0]))
1565                                                         continue
1566
1567                                                 if mydep[0]==1:
1568                                                         if mydep[1]!=[]:
1569                                                                 #we have some unsolvable deps
1570                                                                 #remove ! deps, which always show up as unsatisfiable
1571                                                                 d=0
1572                                                                 while d<len(mydep[1]):
1573                                                                         if mydep[1][d][0]=="!":
1574                                                                                 del mydep[1][d]
1575                                                                         else:
1576                                                                                 d += 1
1577                                                                 #if we emptied out our list, continue:
1578                                                                 if not mydep[1]:
1579                                                                         continue
1580                                                                 stats[mykey]=stats[mykey]+1
1581                                                                 fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1582                                                 else:
1583                                                         stats[mykey]=stats[mykey]+1
1584                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1585
1586         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1587         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1588         #if not portage.portdb.xmatch("bestmatch-visible",x):
1589         #    stats["ebuild.nostable"]+=1
1590         #    fails["ebuild.nostable"].append(x)
1591         if allmasked and repolevel == 3:
1592                 stats["ebuild.allmasked"]+=1
1593                 fails["ebuild.allmasked"].append(x)
1594
1595 if options.mode == "manifest":
1596         sys.exit(dofail)
1597
1598 #Pickle and save results for instant reuse in last and lfull
1599 if os.access(portage.const.CACHE_PATH, os.W_OK):
1600         for myobj, fname in (stats, "repo.stats"), (fails, "repo.fails"):
1601                 fpath = os.path.join(portage.const.CACHE_PATH, fname)
1602                 savef = open(fpath, 'w')
1603                 pickle.dump(myobj, savef)
1604                 savef.close()
1605                 portage.apply_secpass_permissions(fpath, gid=portage.portage_gid,
1606                         mode=0664)
1607
1608 # TODO(antarus) This function and last () look familiar ;)
1609
1610 #dofail will be set to 1 if we have failed in at least one non-warning category
1611 dofail=0
1612 #dowarn will be set to 1 if we tripped any warnings
1613 dowarn=0
1614 #dofull will be set if we should print a "repoman full" informational message
1615 dofull = options.mode not in ("full", "lfull")
1616
1617 for x in qacats:
1618         if not stats[x]:
1619                 continue
1620         dowarn = 1
1621         if x not in qawarnings:
1622                 dofail = 1
1623
1624 if dofail or \
1625         (dowarn and not (options.quiet or options.mode == "scan")):
1626         dofull = 0
1627
1628 # Save QA output so that it can be conveniently displayed
1629 # in $EDITOR while the user creates a commit message.
1630 # Otherwise, the user would not be able to see this output
1631 # once the editor has taken over the screen.
1632 qa_output = StringIO.StringIO()
1633 style_file = ConsoleStyleFile(sys.stdout)
1634 if options.mode == 'commit' and \
1635         (not commitmessage or not commitmessage.strip()):
1636         style_file.write_listener = qa_output
1637 console_writer = StyleWriter(file=style_file, maxcol=9999)
1638 console_writer.style_listener = style_file.new_styles
1639
1640 f = formatter.AbstractFormatter(console_writer)
1641
1642 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
1643
1644 style_file.flush()
1645 del console_writer, f, style_file
1646 qa_output = qa_output.getvalue()
1647 qa_output = qa_output.splitlines(True)
1648
1649 def grouplist(mylist,seperator="/"):
1650         """(list,seperator="/") -- Takes a list of elements; groups them into
1651         same initial element categories. Returns a dict of {base:[sublist]}
1652         From: ["blah/foo","spork/spatula","blah/weee/splat"]
1653         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
1654         mygroups={}
1655         for x in mylist:
1656                 xs=x.split(seperator)
1657                 if xs[0]==".":
1658                         xs=xs[1:]
1659                 if xs[0] not in mygroups:
1660                         mygroups[xs[0]]=[seperator.join(xs[1:])]
1661                 else:
1662                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
1663         return mygroups
1664
1665 if have_pmasked and not (options.without_mask or options.ignore_masked):
1666         print bold("Note: use --without-mask to check " + \
1667                 "KEYWORDS on dependencies of masked packages")
1668
1669 if have_dev_keywords and not options.include_dev:
1670         print bold("Note: use --include-dev (-d) to check " + \
1671                 "dependencies for 'dev' profiles")
1672
1673 if options.mode != 'commit':
1674         if dofull:
1675                 print bold("Note: type \"repoman full\" for a complete listing.")
1676         if dowarn and not dofail:
1677                 print green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\""
1678         elif not dofail:
1679                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
1680         elif dofail:
1681                 print turquoise("Please fix these important QA issues first.")
1682                 print green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n"
1683                 sys.exit(1)
1684 else:
1685         if dofail and can_force and options.force and not options.pretend:
1686                 print green("RepoMan sez:") + \
1687                         " \"You want to commit even with these QA issues?\n" + \
1688                         "              I'll take it this time, but I'm not happy.\"\n"
1689         elif dofail:
1690                 if options.force and not can_force:
1691                         print bad("The --force option has been disabled due to extraordinary issues.")
1692                 print turquoise("Please fix these important QA issues first.")
1693                 print green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n"
1694                 sys.exit(1)
1695
1696         if options.pretend:
1697                 print green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n"
1698
1699         myunadded = []
1700         if vcs == "cvs":
1701                 try:
1702                         myvcstree=portage.cvstree.getentries("./",recursive=1)
1703                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
1704                 except SystemExit, e:
1705                         raise  # TODO propogate this
1706                 except:
1707                         err("Error retrieving CVS tree; exiting.")
1708         if vcs == "svn":
1709                 try:
1710                         svnstatus=os.popen("svn status --no-ignore").readlines()
1711                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
1712                 except SystemExit, e:
1713                         raise  # TODO propogate this
1714                 except:
1715                         err("Error retrieving SVN info; exiting.")
1716         if vcs == "git":
1717                 # get list of files not under version control or missing
1718                 myf = os.popen("git ls-files --others")
1719                 myunadded = [ "./" + elem[:-1] for elem in myf ]
1720                 myf.close()
1721
1722         myautoadd=[]
1723         if myunadded:
1724                 for x in range(len(myunadded)-1,-1,-1):
1725                         xs=myunadded[x].split("/")
1726                         if xs[-1]=="files":
1727                                 print "!!! files dir is not added! Please correct this."
1728                                 sys.exit(-1)
1729                         elif xs[-1]=="Manifest":
1730                                 # It's a manifest... auto add
1731                                 myautoadd+=[myunadded[x]]
1732                                 del myunadded[x]
1733
1734         if myautoadd:
1735                 print ">>> Auto-Adding missing Manifest(s)..."
1736                 if options.pretend:
1737                         if vcs == "cvs":
1738                                 print "(cvs add "+" ".join(myautoadd)+")"
1739                         if vcs == "svn":
1740                                 print "(svn add "+" ".join(myautoadd)+")"
1741                         elif vcs == "git":
1742                                 print "(git add "+" ".join(myautoadd)+")"
1743                         retval=0
1744                 else:
1745                         if vcs == "cvs":
1746                                 retval=os.system("cvs add "+" ".join(myautoadd))
1747                         if vcs == "svn":
1748                                 retval=os.system("svn add "+" ".join(myautoadd))
1749                         elif vcs == "git":
1750                                 retval=os.system("git add "+" ".join(myautoadd))
1751                 if retval:
1752                         writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \
1753                                 (vcs, retval), level=logging.ERROR, noiselevel=-1)
1754                         sys.exit(retval)
1755
1756         if myunadded:
1757                 print red("!!! The following files are in your local tree but are not added to the master")
1758                 print red("!!! tree. Please remove them from the local tree or add them to the master tree.")
1759                 for x in myunadded:
1760                         print "   ",x
1761                 print
1762                 print
1763                 sys.exit(1)
1764
1765         if vcs == "cvs":
1766                 mycvstree = cvstree.getentries("./", recursive=1)
1767                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1768                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1769                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
1770                 bin_blob_pattern = re.compile("^-kb$")
1771                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
1772                         recursive=1, basedir="./"))
1773
1774
1775         if vcs == "svn":
1776                 svnstatus = os.popen("svn status").readlines()
1777                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1778                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1779                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D") ]
1780                 # in contrast to CVS, SVN expands nothing by default.
1781                 # bin_blobs historically
1782                 # were just there to see what files need to be checked for
1783                 # keyword expansion, which is exactly what we do here, so
1784                 # slightly change the semantic meaning of "bin_blob"...  In the
1785                 # future we could store which keyword is expanded.
1786                 props = os.popen("svn propget -R svn:keywords").readlines()
1787
1788                 # For files with multiple props set, props are delimited by newlines,
1789                 # so exclude lines that don't contain " - " since each of those lines
1790                 # only a contain props for a file listed on a previous line.
1791                 expansion = set("./" + prop.split(" - ")[0] \
1792                         for prop in props if " - " in prop)
1793
1794         elif vcs == "git":
1795                 mychanged = os.popen("git ls-files -m").readlines()
1796                 mychanged = [ "./" +  elem[:-1] for elem in mychanged ]
1797                 mynew = os.popen("git diff --cached --name-only --diff-filter=A").readlines()
1798                 strip_levels = repolevel - 1
1799                 if strip_levels:
1800                         mynew = [path_lstrip_re.sub("", elem, strip_levels) for elem in mynew]
1801                 mynew = ["./" + elem[:-1] for elem in mynew]
1802                 myremoved = os.popen("git diff --cached --name-only --diff-filter=D").readlines()
1803                 if strip_levels:
1804                         myremoved = [path_lstrip_re.sub("", elem, strip_levels) for elem in myremoved]
1805                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1806
1807         if vcs:
1808                 if not (mychanged or mynew or myremoved):
1809                         print green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\""
1810                         print
1811                         print "(Didn't find any changed files...)"
1812                         print
1813                         sys.exit(0)
1814
1815         # Manifests need to be regenerated after all other commits, so don't commit
1816         # them now even if they have changed.
1817         mymanifests = set()
1818         changed_set = set()
1819         new_set = set()
1820         for f in mychanged:
1821                 if "Manifest" == os.path.basename(f):
1822                         mymanifests.add(f)
1823                 else:
1824                         changed_set.add(f)
1825         for f in mynew:
1826                 if "Manifest" == os.path.basename(f):
1827                         mymanifests.add(f)
1828                 else:
1829                         new_set.add(f)
1830         mychanged = list(changed_set)
1831         mynew =  list(new_set)
1832         mymanifests = list(mymanifests)
1833         del changed_set, new_set
1834         myupdates = mychanged + mynew
1835         myheaders = []
1836         mydirty = []
1837         headerstring = "'\$(Header|Id)"
1838         headerstring += ".*\$'"
1839         for myfile in myupdates:
1840
1841                 # for CVS, no_expansion contains files that are excluded from expansion
1842                 if vcs == "cvs":
1843                         if myfile in no_expansion:
1844                                 continue
1845
1846                 # for SVN, expansion contains files that are included in expansion
1847                 elif vcs == "svn":
1848                         if myfile not in expansion:
1849                                 continue
1850
1851                 myout = commands.getstatusoutput("egrep -q "+headerstring+" "+myfile)
1852                 if myout[0] == 0:
1853                         myheaders.append(myfile)
1854
1855         print "*",green(str(len(myupdates))),"files being committed...",green(str(len(myheaders))),"have headers that will change."
1856         print "*","Files with headers will cause the manifests to be made and recommited."
1857         logging.info("myupdates:", str(myupdates))
1858         logging.info("myheaders:", str(myheaders))
1859
1860         commitmessage = options.commitmsg
1861         if options.commitmsgfile:
1862                 try:
1863                         f = open(options.commitmsgfile)
1864                         commitmessage = f.read()
1865                         f.close()
1866                         del f
1867                 except (IOError, OSError), e:
1868                         if e.errno == errno.ENOENT:
1869                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
1870                         else:
1871                                 raise
1872                 # We've read the content so the file is no longer needed.
1873                 commitmessagefile = None
1874         if not commitmessage or not commitmessage.strip():
1875                 try:
1876                         editor = os.environ.get("EDITOR")
1877                         if editor and utilities.editor_is_executable(editor):
1878                                 commitmessage = utilities.get_commit_message_with_editor(
1879                                         editor, message=qa_output)
1880                         else:
1881                                 commitmessage = utilities.get_commit_message_with_stdin()
1882                 except KeyboardInterrupt:
1883                         exithandler()
1884                 if not commitmessage or not commitmessage.strip():
1885                         print "* no commit message?  aborting commit."
1886                         sys.exit(1)
1887         commitmessage = commitmessage.rstrip()
1888         portage_version = getattr(portage, "VERSION", None)
1889         if portage_version is None:
1890                 sys.stderr.write("Failed to insert portage version in message!\n")
1891                 sys.stderr.flush()
1892                 portage_version = "Unknown"
1893         unameout = platform.system() + " " + platform.release() + " "
1894         if platform.system() in ["Darwin", "SunOS"]:
1895                 unameout += platform.processor()
1896         else:
1897                 unameout += platform.machine()
1898         commitmessage += "\n(Portage version: %s/%s/%s" % \
1899                 (portage_version, vcs, unameout)
1900         if options.force:
1901                 commitmessage += ", RepoMan options: --force"
1902         commitmessage += ")"
1903
1904         if myupdates or myremoved:
1905                 myfiles = myupdates + myremoved
1906                 if not myheaders and "sign" not in repoman_settings.features:
1907                         myfiles += mymanifests
1908                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1909                 mymsg = os.fdopen(fd, "w")
1910                 mymsg.write(commitmessage)
1911                 mymsg.close()
1912
1913                 print
1914                 print green("Using commit message:")
1915                 print green("------------------------------------------------------------------------------")
1916                 print commitmessage
1917                 print green("------------------------------------------------------------------------------")
1918                 print
1919
1920                 # Having a leading ./ prefix on file paths can trigger a bug in
1921                 # the cvs server when committing files to multiple directories,
1922                 # so strip the prefix.
1923                 myfiles = [f.lstrip("./") for f in myfiles]
1924
1925                 retval = None
1926                 if options.pretend:
1927                         if vcs == "cvs":
1928                                 print "(cvs -q commit -F %s %s)" % \
1929                                         (commitmessagefile, " ".join(myfiles))
1930                         if vcs == "svn":
1931                                 print "(svn commit -F %s %s)" % \
1932                                         (commitmessagefile, " ".join(myfiles))
1933                         elif vcs == "git":
1934                                 print "(git commit -F %s %s)" % \
1935                                         (commitmessagefile, " ".join(myfiles))
1936                 else:
1937                         if vcs == "cvs":
1938                                 retval = spawn(["cvs", "-q", "commit",
1939                                         "-F", commitmessagefile] + myfiles,
1940                                         env=os.environ)
1941                         if vcs == "svn":
1942                                 retval = spawn(["svn", "commit",
1943                                         "-F", commitmessagefile] + myfiles,
1944                                         env=os.environ)
1945                         elif vcs == "git":
1946                                 retval = spawn(["git", "commit", "-F",
1947                                         commitmessagefile] + myfiles,
1948                                         env=os.environ)
1949                 try:
1950                         os.unlink(commitmessagefile)
1951                 except OSError:
1952                         pass
1953                 if retval:
1954                         writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \
1955                                 (vcs, retval), level=logging.ERROR, noiselevel=-1)
1956                         sys.exit(retval)
1957
1958         # Setup the GPG commands
1959         def gpgsign(filename):
1960                 if "PORTAGE_GPG_KEY" not in repoman_settings:
1961                         raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
1962                 if "PORTAGE_GPG_DIR" not in repoman_settings:
1963                         if "HOME" in os.environ:
1964                                 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(os.environ["HOME"], ".gnupg")
1965                                 logging.info("Automatically setting PORTAGE_GPG_DIR to %s" % repoman_settings["PORTAGE_GPG_DIR"])
1966                         else:
1967                                 raise portage.exception.MissingParameter("PORTAGE_GPG_DIR is unset!")
1968                 gpg_dir = repoman_settings["PORTAGE_GPG_DIR"]
1969                 if gpg_dir.startswith("~") and "HOME" in os.environ:
1970                         repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(
1971                                 os.environ["HOME"], gpg_dir[1:].lstrip(os.path.sep))
1972                 if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
1973                         raise portage.exception.InvalidLocation(
1974                                 "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
1975                                 repoman_settings["PORTAGE_GPG_DIR"])
1976                 gpgcmd = "gpg --sign --clearsign --yes "
1977                 gpgcmd+= "--default-key "+repoman_settings["PORTAGE_GPG_KEY"]
1978                 if "PORTAGE_GPG_DIR" in repoman_settings:
1979                         gpgcmd += " --homedir "+repoman_settings["PORTAGE_GPG_DIR"]
1980                 if options.pretend:
1981                         print "("+gpgcmd+" "+filename+")"
1982                 else:
1983                         rValue = os.system(gpgcmd+" "+filename)
1984                         if rValue == os.EX_OK:
1985                                 os.rename(filename+".asc", filename)
1986                         else:
1987                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
1988
1989         # When files are removed and re-added, the cvs server will put /Attic/
1990         # inside the $Header path. This code detects the problem and corrects it
1991         # so that the Manifest will generate correctly. See bug #169500.
1992         from portage.util import write_atomic
1993         cvs_header = re.compile(r'^#\s*\$Header.*\$$')
1994         for x in myheaders:
1995                 f = open(x)
1996                 mylines = f.readlines()
1997                 f.close()
1998                 modified = False
1999                 for i, line in enumerate(mylines):
2000                         if cvs_header.match(line) and "/Attic/" in line:
2001                                 mylines[i] = line.replace("/Attic/", "/")
2002                                 modified = True
2003                 if modified:
2004                         write_atomic(x, "".join(mylines))
2005
2006         manifest_commit_required = True
2007         if myupdates or myremoved or mynew:
2008                 myfiles=myupdates+myremoved+mynew
2009                 for x in range(len(myfiles)-1, -1, -1):
2010                         if myfiles[x].count("/") < 4-repolevel:
2011                                 del myfiles[x]
2012                 mydone=[]
2013                 if repolevel==3:   # In a package dir
2014                         repoman_settings["O"] = startdir
2015                         portage.digestgen([], repoman_settings, manifestonly=1,
2016                                 myportdb=portdb)
2017                 elif repolevel==2: # In a category dir
2018                         for x in myfiles:
2019                                 xs=x.split("/")
2020                                 if len(xs) < 4-repolevel:
2021                                         continue
2022                                 if xs[0]==".":
2023                                         xs=xs[1:]
2024                                 if xs[0] in mydone:
2025                                         continue
2026                                 mydone.append(xs[0])
2027                                 repoman_settings["O"] = os.path.join(startdir, xs[0])
2028                                 if not os.path.isdir(repoman_settings["O"]):
2029                                         continue
2030                                 portage.digestgen([], repoman_settings, manifestonly=1,
2031                                         myportdb=portdb)
2032                 elif repolevel==1: # repo-cvsroot
2033                         print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
2034                         for x in myfiles:
2035                                 xs=x.split("/")
2036                                 if len(xs) < 4-repolevel:
2037                                         continue
2038                                 if xs[0]==".":
2039                                         xs=xs[1:]
2040                                 if "/".join(xs[:2]) in mydone:
2041                                         continue
2042                                 mydone.append("/".join(xs[:2]))
2043                                 repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1])
2044                                 if not os.path.isdir(repoman_settings["O"]):
2045                                         continue
2046                                 portage.digestgen([], repoman_settings, manifestonly=1,
2047                                         myportdb=portdb)
2048                 else:
2049                         print red("I'm confused... I don't know where I am!")
2050                         sys.exit(1)
2051
2052                 # Force an unsigned commit when more than one Manifest needs to be signed.
2053                 if repolevel < 3 and "sign" in repoman_settings.features:
2054                         if options.pretend:
2055                                 if vcs == "cvs":
2056                                         print "(cvs -q commit -F commitmessagefile)"
2057                                 if vcs == "svn":
2058                                         print "(svn -q commit -F commitmessagefile)"
2059                                 elif vcs == "git":
2060                                         print "(git commit -F commitmessagefile)"
2061                         else:
2062                                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2063                                 mymsg = os.fdopen(fd, "w")
2064                                 mymsg.write(commitmessage)
2065                                 mymsg.write("\n (Unsigned Manifest commit)")
2066                                 mymsg.close()
2067                                 if vcs == "cvs":
2068                                         retval=os.system("cvs -q commit -F "+commitmessagefile)
2069                                 if vcs == "svn":
2070                                         retval=os.system("svn -q commit -F "+commitmessagefile)
2071                                 elif vcs == "git":
2072                                         retval=os.system("git commit -F "+commitmessagefile)
2073                                 try:
2074                                         os.unlink(commitmessagefile)
2075                                 except OSError:
2076                                         pass
2077                                 if retval:
2078                                         writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \
2079                                                 (vcs, retval), level=logging.ERROR, noiselevel=-1)
2080                                         sys.exit(retval)
2081                         manifest_commit_required = False
2082
2083         signed = False
2084         if "sign" in repoman_settings.features:
2085                 signed = True
2086                 myfiles = myupdates + myremoved + mymanifests
2087                 try:
2088                         if repolevel==3:   # In a package dir
2089                                 repoman_settings["O"] = "."
2090                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2091                         elif repolevel==2: # In a category dir
2092                                 mydone=[]
2093                                 for x in myfiles:
2094                                         xs=x.split("/")
2095                                         if len(xs) < 4-repolevel:
2096                                                 continue
2097                                         if xs[0]==".":
2098                                                 xs=xs[1:]
2099                                         if xs[0] in mydone:
2100                                                 continue
2101                                         mydone.append(xs[0])
2102                                         repoman_settings["O"] = os.path.join(".", xs[0])
2103                                         if not os.path.isdir(repoman_settings["O"]):
2104                                                 continue
2105                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2106                         elif repolevel==1: # repo-cvsroot
2107                                 print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
2108                                 mydone=[]
2109                                 for x in myfiles:
2110                                         xs=x.split("/")
2111                                         if len(xs) < 4-repolevel:
2112                                                 continue
2113                                         if xs[0]==".":
2114                                                 xs=xs[1:]
2115                                         if "/".join(xs[:2]) in mydone:
2116                                                 continue
2117                                         mydone.append("/".join(xs[:2]))
2118                                         repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
2119                                         if not os.path.isdir(repoman_settings["O"]):
2120                                                 continue
2121                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2122                 except portage.exception.PortageException, e:
2123                         portage.writemsg("!!! %s\n" % str(e))
2124                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2125                         signed = False
2126
2127         if manifest_commit_required or signed:
2128                 if options.pretend:
2129                         if vcs == "cvs":
2130                                 print "(cvs -q commit -F commitmessagefile)"
2131                         if vcs == "svn":
2132                                 print "(svn -q commit -F commitmessagefile)"
2133                         elif vcs == "git":
2134                                 print "(git commit -a -F commitmessagefile)"
2135                 else:
2136                         fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2137                         mymsg = os.fdopen(fd, "w")
2138                         mymsg.write(commitmessage)
2139                         if signed:
2140                                 mymsg.write("\n (Signed Manifest commit)")
2141                         else:
2142                                 mymsg.write("\n (Unsigned Manifest commit)")
2143                         mymsg.close()
2144                         if vcs == "cvs":
2145                                 retval=os.system("cvs -q commit -F "+commitmessagefile)
2146                         if vcs == "svn":
2147                                 retval=os.system("svn -q commit -F "+commitmessagefile)
2148                         elif vcs == "git":
2149                                 retval=os.system("git commit -a -F "+commitmessagefile)
2150                         try:
2151                                 os.unlink(commitmessagefile)
2152                         except OSError:
2153                                 pass
2154                         if retval:
2155                                 writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \
2156                                         (vcs, retval), level=logging.ERROR, noiselevel=-1)
2157                                 sys.exit(retval)
2158
2159         print
2160         if vcs:
2161                 print "Commit complete."
2162         else:
2163                 print "repoman was too scared by not seeing any familiar version control file that he forgot to commit anything"
2164         print green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n"
2165 sys.exit(0)
2166