bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2006 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 # Next to do: dep syntax checking in mask files
   7 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   8 # that last one is tricky because multiple profiles need to be checked.
   9
  10 from __future__ import print_function
  11
  12 import codecs
  13 try:
  14         from subprocess import getstatusoutput as subprocess_getstatusoutput
  15 except ImportError:
  16         from commands import getstatusoutput as subprocess_getstatusoutput
  17 import errno
  18 import formatter
  19 import logging
  20 import optparse
  21 import re
  22 import signal
  23 import stat
  24 import sys
  25 import tempfile
  26 import time
  27 import platform
  28
  29 from io import StringIO
  30 from itertools import chain
  31 from stat import S_ISDIR, ST_CTIME
  32
  33 if not hasattr(__builtins__, "set"):
  34         from sets import Set as set
  35
  36 try:
  37         import portage
  38 except ImportError:
  39         from os import path as osp
  40         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  41         import portage
  42 portage._disable_legacy_globals()
  43 from portage import os
  44 from portage import _encodings
  45 from portage import _unicode_encode
  46
  47 try:
  48         from repoman.checks import run_checks
  49         from repoman import utilities
  50 except ImportError:
  51         from os import path as osp
  52         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym'))
  53         from repoman.checks import run_checks
  54         from repoman import utilities
  55
  56 from _emerge.Package import Package
  57 from _emerge.RootConfig import RootConfig
  58 from portage.sets import load_default_config
  59
  60 import portage.checksum
  61 import portage.const
  62 import portage.dep
  63 portage.dep._dep_check_strict = True
  64 import portage.exception
  65 from portage import cvstree, normalize_path
  66 from portage import util
  67 from portage.exception import ParseError
  68 from portage.manifest import Manifest
  69 from portage.process import find_binary, spawn
  70 from portage.output import bold, create_color_func, darkgreen, \
  71         green, nocolor, red, turquoise, yellow
  72 from portage.output import ConsoleStyleFile, StyleWriter
  73 from portage.util import cmp_sort_key, writemsg_level
  74
  75 if sys.hexversion >= 0x3000000:
  76         basestring = str
  77
  78 util.initialize_logger()
  79
  80 # 14 is the length of DESCRIPTION=""
  81 max_desc_len = 100
  82 allowed_filename_chars="a-zA-Z0-9._-+:"
  83 disallowed_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]')
  84 bad = create_color_func("BAD")
  85
  86 # A sane umask is needed for files that portage creates.
  87 os.umask(0o22)
  88 repoman_settings = portage.config(local_config=False,
  89         config_incrementals=portage.const.INCREMENTALS)
  90 repoman_settings.lock()
  91
  92 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
  93         not sys.stdout.isatty():
  94         nocolor()
  95
  96 def warn(txt):
  97         print("repoman: " + txt)
  98
  99 def err(txt):
 100         warn(txt)
 101         sys.exit(1)
 102
 103 def exithandler(signum=None, frame=None):
 104         logging.fatal("Interrupted; exiting...")
 105         sys.exit(1)
 106         os.kill(0, signal.SIGKILL)
 107
 108 signal.signal(signal.SIGINT,exithandler)
 109
 110 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 111         """Repoman needs it's own HelpFormatter for now, because the default ones
 112         murder the help text."""
 113
 114         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 115                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 116
 117         def format_description(self, description):
 118                 return description
 119
 120 class RepomanOptionParser(optparse.OptionParser):
 121         """Add the on_tail function, ruby has it, optionParser should too
 122         """
 123
 124         def __init__(self, *args, **kwargs):
 125                 optparse.OptionParser.__init__(self, *args, **kwargs)
 126                 self.tail = ""
 127
 128         def on_tail(self, description):
 129                 self.tail += description
 130
 131         def format_help(self, formatter=None):
 132                 result = optparse.OptionParser.format_help(self, formatter)
 133                 result += self.tail
 134                 return result
 135
 136
 137 def ParseArgs(args, qahelp):
 138         """This function uses a customized optionParser to parse command line arguments for repoman
 139         Args:
 140           args - a sequence of command line arguments
 141                 qahelp - a dict of qa warning to help message
 142         Returns:
 143           (opts, args), just like a call to parser.parse_args()
 144         """
 145
 146         modes = {
 147                 'commit' : 'Run a scan then commit changes',
 148                 'ci' : 'Run a scan then commit changes',
 149                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 150                 'full' : 'Scan directory tree and print all issues (not a summary)',
 151                 'help' : 'Show this screen',
 152                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 153                 'scan' : 'Scan directory tree for QA issues'
 154         }
 155
 156         mode_keys = list(modes)
 157         mode_keys.sort()
 158
 159         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 160         parser.description = green(" ".join((os.path.basename(args[0]), "1.2")))
 161         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 162         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 163         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 164
 165         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 166                 help='specify a commit message on the command line')
 167
 168         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 169                 help='specify a path to a file that contains a commit message')
 170
 171         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 172                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 173
 174         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 175                 help='do not print unnecessary messages')
 176
 177         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 178                 help='Commit with QA violations')
 179
 180         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 181                 help='be very verbose in output', default=0)
 182
 183         parser.add_option('-V', '--version', dest='version', action='store_true',
 184                 help='show version info')
 185
 186         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 187                 default=False, help='forces the metadata.xml parse check to be carried out')
 188
 189         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 190                 default=False, help='ignore arch-specific failures (where arch != host)')
 191
 192         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 193                 default=False, help='ignore masked packages (not allowed with commit mode)')
 194
 195         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 196                 default=False, help='include dev profiles in dependency checks')
 197
 198         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 199                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 200
 201         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 202                 help='specify which mode repoman will run in (default=full)')
 203
 204         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 205
 206         for k in mode_keys:
 207                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 208
 209         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 210
 211         sorted_qa = list(qahelp)
 212         sorted_qa.sort()
 213         for k in sorted_qa:
 214                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 215
 216         if not args:
 217                 args = sys.argv
 218         opts, args = parser.parse_args(args)
 219
 220         if opts.mode == 'help':
 221                 parser.print_help(short=False)
 222
 223         for arg in args:
 224                 if arg in modes:
 225                         if not opts.mode:
 226                                 opts.mode = arg
 227                                 break
 228
 229         if not opts.mode:
 230                 opts.mode = 'full'
 231
 232         if opts.mode == 'ci':
 233                 opts.mode = 'commit'  # backwards compat shortcut
 234
 235         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 236                 if opts.ignore_masked:
 237                         parser.error('Commit mode and --ignore-masked are not compatible')
 238                 if opts.without_mask:
 239                         parser.error('Commit mode and --without-mask are not compatible')
 240
 241         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 242         for val in range(opts.verbosity):
 243                 logger = logging.getLogger()
 244                 logger.setLevel(logger.getEffectiveLevel() - 10)
 245
 246         for val in range(opts.quiet):
 247                 logger = logging.getLogger()
 248                 logger.setLevel(logger.getEffectiveLevel() + 10)
 249
 250         return (opts, args)
 251
 252 qahelp={
 253         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 254         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 255         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 256         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 257         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 258         "changelog.missing":"Missing ChangeLog files",
 259         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 260         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 261         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 262         "filedir.missing":"Package lacks a files directory",
 263         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
 264         "file.size":"Files in the files directory must be under 20 KiB",
 265         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 266         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 267         "file.UTF8":"File is not UTF8 compliant",
 268         "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf",
 269         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 270         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 271         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 272         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 273         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 274         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 275         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 276         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 277         "EAPI.definition":"EAPI is defined after an inherit call (must be defined before)",
 278         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 279         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 280         "SLOT.missing":"Ebuilds that have a missing or empty SLOT variable",
 281         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 282         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 283         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 284         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 285         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 286         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 287         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 288         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 289         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 290         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 291         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 292         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 293         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 294         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 295         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 296         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 297         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 298         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 299         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 300         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 301         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 302         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 303         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 304         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 305         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 306         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 307         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 308         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 309         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 310         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 311         "variable.readonly":"Assigning a readonly variable",
 312         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 313         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 314         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file",
 315         "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
 316         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 317         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 318         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment",
 319         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 320         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 321         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 322         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 323         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 324         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 325         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 326         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 327         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 328         "ebuild.badheader":"This ebuild has a malformed header",
 329         "metadata.missing":"Missing metadata.xml files",
 330         "metadata.bad":"Bad metadata.xml files",
 331         "metadata.warning":"Warnings in metadata.xml files",
 332         "virtual.versioned":"PROVIDE contains virtuals with versions",
 333         "virtual.exists":"PROVIDE contains existing package names",
 334         "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default",
 335         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 336         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 337 }
 338
 339 qacats = list(qahelp)
 340 qacats.sort()
 341
 342 qawarnings = set((
 343 "changelog.missing",
 344 "changelog.notadded",
 345 "digest.assumed",
 346 "digest.unused",
 347 "ebuild.notadded",
 348 "ebuild.nostable",
 349 "ebuild.allmasked",
 350 "ebuild.nesteddie",
 351 "desktop.invalid",
 352 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 353 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 354 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 355 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 356 "DESCRIPTION.toolong",
 357 "KEYWORDS.dropped",
 358 "KEYWORDS.stupid",
 359 "KEYWORDS.missing",
 360 "IUSE.undefined",
 361 "PDEPEND.suspect",
 362 "RDEPEND.implicit",
 363 "RDEPEND.suspect",
 364 "RESTRICT.invalid",
 365 "SRC_URI.mirror",
 366 "ebuild.minorsyn",
 367 "ebuild.badheader",
 368 "ebuild.patches",
 369 "file.size",
 370 "inherit.autotools",
 371 "java.eclassesnotused",
 372 "metadata.warning",
 373 "virtual.versioned",
 374 "virtual.exists",
 375 "virtual.unavailable",
 376 "usage.obsolete",
 377 "upstream.workaround",
 378 "LIVEVCS.stable",
 379 "LIVEVCS.unmasked",
 380 ))
 381
 382 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 383
 384 missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"]
 385 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 386 allvars.update(Package.metadata_keys)
 387 allvars = sorted(allvars)
 388 commitmessage=None
 389 for x in missingvars:
 390         x += ".missing"
 391         if x not in qacats:
 392                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 393                 qacats.append(x)
 394                 qawarnings.add(x)
 395
 396 valid_restrict = frozenset(["binchecks", "bindist",
 397         "fetch", "installsources", "mirror",
 398         "primaryuri", "strip", "test", "userpriv"])
 399
 400 live_eclasses = frozenset([
 401         "bzr",
 402         "cvs",
 403         "darcs",
 404         "git",
 405         "mercurial",
 406         "subversion"
 407 ])
 408
 409 suspect_rdepend = frozenset([
 410         "app-arch/cabextract",
 411         "app-arch/rpm2targz",
 412         "app-doc/doxygen",
 413         "dev-lang/nasm",
 414         "dev-lang/swig",
 415         "dev-lang/yasm",
 416         "dev-perl/extutils-pkgconfig",
 417         "dev-util/byacc",
 418         "dev-util/cmake",
 419         "dev-util/ftjam",
 420         "dev-util/gperf",
 421         "dev-util/gtk-doc",
 422         "dev-util/gtk-doc-am",
 423         "dev-util/intltool",
 424         "dev-util/jam",
 425         "dev-util/pkgconfig",
 426         "dev-util/scons",
 427         "dev-util/unifdef",
 428         "dev-util/yacc",
 429         "media-gfx/ebdftopcf",
 430         "sys-apps/help2man",
 431         "sys-devel/autoconf",
 432         "sys-devel/automake",
 433         "sys-devel/bin86",
 434         "sys-devel/bison",
 435         "sys-devel/dev86",
 436         "sys-devel/flex",
 437         "sys-devel/m4",
 438         "sys-devel/pmake",
 439         "x11-misc/bdftopcf",
 440         "x11-misc/imake",
 441 ])
 442
 443 # file.executable
 444 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 445
 446 options, arguments = ParseArgs(sys.argv, qahelp)
 447
 448 if options.version:
 449         print("Portage", portage.VERSION)
 450         sys.exit(0)
 451
 452 # Set this to False when an extraordinary issue (generally
 453 # something other than a QA issue) makes it impossible to
 454 # commit (like if Manifest generation fails).
 455 can_force = True
 456
 457 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 458 if portdir is None:
 459         sys.exit(1)
 460
 461 myreporoot = os.path.basename(portdir_overlay)
 462 myreporoot += mydir[len(portdir_overlay):]
 463
 464 vcs = None
 465 if os.path.isdir("CVS"):
 466         vcs = "cvs"
 467 if os.path.isdir(".svn"):
 468         vcs = "svn"
 469 elif os.path.isdir(os.path.join(portdir_overlay, ".git")):
 470         vcs = "git"
 471
 472 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 473 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 474 if vcs_global_opts is None:
 475         if vcs != "git":
 476                 vcs_global_opts = "-q"
 477         else:
 478                 vcs_global_opts = ""
 479 vcs_global_opts = vcs_global_opts.split()
 480
 481 if vcs == "cvs" and \
 482         "commit" == options.mode and \
 483         "RMD160" not in portage.checksum.hashorigin_map:
 484         from portage.util import grablines
 485         repo_lines = grablines("./CVS/Repository")
 486         if repo_lines and \
 487                 "gentoo-x86" == repo_lines[0].strip().split(os.path.sep)[0]:
 488                 msg = "Please install " \
 489                 "pycrypto or enable python's ssl USE flag in order " \
 490                 "to enable RMD160 hash support. See bug #198398 for " \
 491                 "more information."
 492                 prefix = bad(" * ")
 493                 from textwrap import wrap
 494                 for line in wrap(msg, 70):
 495                         print(prefix + line)
 496                 sys.exit(1)
 497         del repo_lines
 498
 499 if options.mode == 'commit' and not options.pretend and not vcs:
 500         logging.info("Not in a version controlled repository; enabling pretend mode.")
 501         options.pretend = True
 502
 503 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 504 repoman_settings = portage.config(local_config=False)
 505 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 506         (repoman_settings.get('PORTDIR_OVERLAY', ''), portdir_overlay)
 507 repoman_settings.backup_changes('PORTDIR_OVERLAY')
 508
 509 root = '/'
 510 trees = {
 511         root : {'porttree' : portage.portagetree(root, settings=repoman_settings)}
 512 }
 513 portdb = trees[root]['porttree'].dbapi
 514
 515 # Constrain dependency resolution to the master(s)
 516 # that are specified in layout.conf.
 517 portdir_overlay = os.path.realpath(portdir_overlay)
 518 repo_info = portdb._repo_info[portdir_overlay]
 519 portdb.porttrees = list(repo_info.eclass_db.porttrees)
 520 portdir = portdb.porttrees[0]
 521
 522 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 523 # profile-specific config constructor calls.
 524 env = os.environ.copy()
 525 env['PORTDIR'] = portdir
 526 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 527
 528 logging.info('Setting paths:')
 529 logging.info('PORTDIR = "' + portdir + '"')
 530 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 531
 532 categories = []
 533 for path in set([portdir, portdir_overlay]):
 534         categories.extend(portage.util.grabfile(
 535                 os.path.join(path, 'profiles', 'categories')))
 536 repoman_settings.categories = tuple(sorted(
 537         portage.util.stack_lists([categories], incremental=1)))
 538 del categories
 539
 540 portdb.mysettings = repoman_settings
 541 root_config = RootConfig(repoman_settings, trees[root], None)
 542 # We really only need to cache the metadata that's necessary for visibility
 543 # filtering. Anything else can be discarded to reduce memory consumption.
 544 portdb._aux_cache_keys.clear()
 545 portdb._aux_cache_keys.update(["EAPI", "KEYWORDS", "SLOT"])
 546
 547 reposplit = myreporoot.split(os.path.sep)
 548 repolevel = len(reposplit)
 549
 550 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 551 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 552 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 553 if options.mode == 'commit' and repolevel not in [1,2,3]:
 554         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 555         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 556         print(red("***")+" This is intended behaviour, to ensure the manifest is recommited for a package.")
 557         print(red("***"))
 558         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 559
 560 startdir = normalize_path(mydir)
 561 repodir = startdir
 562 for x in range(0, repolevel - 1):
 563         repodir = os.path.dirname(repodir)
 564 repodir = os.path.realpath(repodir)
 565
 566 def caterror(mycat):
 567         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 568
 569 class ProfileDesc(object):
 570         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 571         def __init__(self, arch, status, sub_path, tree_path):
 572                 self.arch = arch
 573                 self.status = status
 574                 self.sub_path = normalize_path(sub_path.lstrip(os.sep))
 575                 self.tree_path = tree_path
 576                 self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 577
 578 profile_list = []
 579 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 580
 581 # get lists of valid keywords, licenses, and use
 582 kwlist = set()
 583 liclist = set()
 584 uselist = set()
 585 global_pmasklines = []
 586
 587 for path in portdb.porttrees:
 588         try:
 589                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 590         except OSError:
 591                 pass
 592         kwlist.update(portage.grabfile(os.path.join(path,
 593                 "profiles", "arch.list")))
 594
 595         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 596         for x in use_desc:
 597                 x = x.split()
 598                 if x:
 599                         uselist.add(x[0])
 600
 601         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 602         try:
 603                 expand_list = os.listdir(expand_desc_dir)
 604         except OSError:
 605                 pass
 606         else:
 607                 for fn in expand_list:
 608                         if not fn[-5:] == '.desc':
 609                                 continue
 610                         use_prefix = fn[:-5].lower() + '_'
 611                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 612                                 x = x.split()
 613                                 if x:
 614                                         uselist.add(use_prefix + x[0])
 615
 616         global_pmasklines.append(portage.util.grabfile_package(
 617                 os.path.join(path, 'profiles', 'package.mask'), recursive=1))
 618
 619         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 620         try:
 621                 desc_file = codecs.open(_unicode_encode(desc_path,
 622                         encoding=_encodings['fs'], errors='strict'),
 623                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 624         except EnvironmentError:
 625                 pass
 626         else:
 627                 for i, x in enumerate(desc_file):
 628                         if x[0] == "#":
 629                                 continue
 630                         arch = x.split()
 631                         if len(arch) == 0:
 632                                 continue
 633                         if len(arch) != 3:
 634                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 635                                         desc_path + " line %d" % (i+1, ))
 636                         elif arch[0] not in kwlist:
 637                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 638                                         desc_path + " line %d" % (i+1, ))
 639                         elif arch[2] not in valid_profile_types:
 640                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 641                                         desc_path + " line %d" % (i+1, ))
 642                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], portdir)
 643                         if not os.path.isdir(profile_desc.abs_path):
 644                                 logging.error(
 645                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 646                                         arch[2], arch[1], arch[0], desc_path, i+1)
 647                                 continue
 648                         profile_list.append(profile_desc)
 649                 desc_file.close()
 650
 651 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 652 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 653
 654 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 655 global_pmaskdict = {}
 656 for x in global_pmasklines:
 657         global_pmaskdict.setdefault(portage.dep_getkey(x), []).append(x)
 658 del global_pmasklines
 659
 660 def has_global_mask(pkg):
 661         mask_atoms = global_pmaskdict.get(pkg.cp)
 662         if mask_atoms:
 663                 pkg_list = [pkg]
 664                 for x in mask_atoms:
 665                         if portage.dep.match_from_list(x, pkg_list):
 666                                 return x
 667         return None
 668
 669 # Ensure that profile sub_path attributes are unique. Process in reverse order
 670 # so that profiles with duplicate sub_path from overlays will override
 671 # profiles with the same sub_path from parent repos.
 672 profiles = {}
 673 profile_list.reverse()
 674 profile_sub_paths = set()
 675 for prof in profile_list:
 676         if prof.sub_path in profile_sub_paths:
 677                 continue
 678         profile_sub_paths.add(prof.sub_path)
 679         profiles.setdefault(prof.arch, []).append(prof)
 680
 681 for x in repoman_settings.archlist():
 682         if x[0] == "~":
 683                 continue
 684         if x not in profiles:
 685                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 686                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 687                 print(red("up with the "+x+" team."))
 688                 print()
 689
 690 if not liclist:
 691         logging.fatal("Couldn't find licenses?")
 692         sys.exit(1)
 693
 694 if not kwlist:
 695         logging.fatal("Couldn't read KEYWORDS from arch.list")
 696         sys.exit(1)
 697
 698 if not uselist:
 699         logging.fatal("Couldn't find use.desc?")
 700         sys.exit(1)
 701
 702 scanlist=[]
 703 if repolevel==2:
 704         #we are inside a category directory
 705         catdir=reposplit[-1]
 706         if catdir not in repoman_settings.categories:
 707                 caterror(catdir)
 708         mydirlist=os.listdir(startdir)
 709         for x in mydirlist:
 710                 if x == "CVS" or x.startswith("."):
 711                         continue
 712                 if os.path.isdir(startdir+"/"+x):
 713                         scanlist.append(catdir+"/"+x)
 714         repo_subdir = catdir + os.sep
 715 elif repolevel==1:
 716         for x in repoman_settings.categories:
 717                 if not os.path.isdir(startdir+"/"+x):
 718                         continue
 719                 for y in os.listdir(startdir+"/"+x):
 720                         if y == "CVS" or y.startswith("."):
 721                                 continue
 722                         if os.path.isdir(startdir+"/"+x+"/"+y):
 723                                 scanlist.append(x+"/"+y)
 724         repo_subdir = ""
 725 elif repolevel==3:
 726         catdir = reposplit[-2]
 727         if catdir not in repoman_settings.categories:
 728                 caterror(catdir)
 729         scanlist.append(catdir+"/"+reposplit[-1])
 730         repo_subdir = scanlist[-1] + os.sep
 731 repo_subdir_len = len(repo_subdir)
 732 scanlist.sort()
 733
 734 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 735
 736 def dev_keywords(profiles):
 737         """
 738         Create a set of KEYWORDS values that exist in 'dev'
 739         profiles. These are used
 740         to trigger a message notifying the user when they might
 741         want to add the --include-dev option.
 742         """
 743         type_arch_map = {}
 744         for arch, arch_profiles in profiles.items():
 745                 for prof in arch_profiles:
 746                         arch_set = type_arch_map.get(prof.status)
 747                         if arch_set is None:
 748                                 arch_set = set()
 749                                 type_arch_map[prof.status] = arch_set
 750                         arch_set.add(arch)
 751
 752         dev_keywords = type_arch_map.get('dev', set())
 753         dev_keywords.update(['~' + arch for arch in dev_keywords])
 754         return frozenset(dev_keywords)
 755
 756 dev_keywords = dev_keywords(profiles)
 757
 758 stats={}
 759 fails={}
 760
 761 # provided by the desktop-file-utils package
 762 desktop_file_validate = find_binary("desktop-file-validate")
 763 desktop_pattern = re.compile(r'.*\.desktop$')
 764
 765 for x in qacats:
 766         stats[x]=0
 767         fails[x]=[]
 768 xmllint_capable = False
 769 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
 770 if options.mode == "manifest":
 771         pass
 772 elif not find_binary('xmllint'):
 773         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
 774         if options.xml_parse or repolevel==3:
 775                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
 776                 sys.exit(1)
 777 else:
 778         #hardcoded paths/urls suck. :-/
 779         must_fetch=1
 780         backup_exists=0
 781         try:
 782                 # if it's been over a week since fetching (or the system clock is fscked), grab an updated copy of metadata.dtd
 783                 # clock is fscked or it's been a week. time to grab a new one.
 784                 ct = os.stat(metadata_dtd)[ST_CTIME]
 785                 if abs(time.time() - ct) > (60*60*24*7):
 786                         # don't trap the exception, we're watching for errno 2 (file not found), anything else is a bug.
 787                         backup_exists=1
 788                 else:
 789                         must_fetch=0
 790
 791         except (OSError,IOError) as e:
 792                 if e.errno != 2:
 793                         print(red("!!!")+" caught exception '%s' for %s/metadata.dtd, bailing" % (str(e), portage.CACHE_PATH))
 794                         sys.exit(1)
 795
 796         if must_fetch:
 797                 print()
 798                 print(green("***")+" the local copy of metadata.dtd needs to be refetched, doing that now")
 799                 print()
 800                 val = 0
 801                 try:
 802                         try:
 803                                 os.unlink(metadata_dtd)
 804                         except OSError as e:
 805                                 if e.errno != errno.ENOENT:
 806                                         raise
 807                                 del e
 808                         val=portage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoman_settings,fetchonly=0, \
 809                                 try_mirrors=0)
 810
 811                 except SystemExit as e:
 812                         raise  # Need to propogate this
 813                 except Exception as e:
 814                         print()
 815                         print(red("!!!")+" attempting to fetch 'http://www.gentoo.org/dtd/metadata.dtd', caught")
 816                         print(red("!!!")+" exception '%s' though." % str(e))
 817                         val=0
 818                 if not val:
 819                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
 820                         sys.exit(1)
 821         #this can be problematic if xmllint changes their output
 822         xmllint_capable=True
 823
 824 if options.mode == 'commit' and vcs:
 825         utilities.detect_vcs_conflicts(options, vcs)
 826
 827 if options.mode == "manifest":
 828         pass
 829 elif options.pretend:
 830         print(green("\nRepoMan does a once-over of the neighborhood..."))
 831 else:
 832         print(green("\nRepoMan scours the neighborhood..."))
 833
 834 new_ebuilds = set()
 835 modified_changelogs = set()
 836 mychanged = []
 837 mynew = []
 838 myremoved = []
 839
 840 if vcs == "cvs":
 841         mycvstree = cvstree.getentries("./", recursive=1)
 842         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
 843         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
 844 if vcs == "svn":
 845         svnstatus = os.popen("svn status").readlines()
 846         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
 847         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
 848 elif vcs == "git":
 849         strip_levels = repolevel - 1
 850
 851         mychanged = os.popen("git diff-index --name-only --diff-filter=M HEAD").readlines()
 852         if strip_levels:
 853                 mychanged = [elem[repo_subdir_len:] for elem in mychanged \
 854                         if elem[:repo_subdir_len] == repo_subdir]
 855         mychanged = ["./" + elem[:-1] for elem in mychanged]
 856
 857         mynew = os.popen("git diff-index --name-only --diff-filter=A HEAD").readlines()
 858         if strip_levels:
 859                 mynew = [elem[repo_subdir_len:] for elem in mynew \
 860                         if elem[:repo_subdir_len] == repo_subdir]
 861         mynew = ["./" + elem[:-1] for elem in mynew]
 862 if vcs:
 863         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
 864         modified_changelogs.update(x for x in chain(mychanged, mynew) \
 865                 if os.path.basename(x) == "ChangeLog")
 866
 867 have_pmasked = False
 868 have_dev_keywords = False
 869 dofail = 0
 870 arch_caches={}
 871 arch_xmatch_caches = {}
 872 shared_xmatch_caches = {"cp-list":{}}
 873
 874 # Disable the "ebuild.notadded" check when not in commit mode and
 875 # running `svn status` in every package dir will be too expensive.
 876
 877 check_ebuild_notadded = not \
 878         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
 879
 880 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
 881 thirdpartymirrors = portage.flatten(list(repoman_settings.thirdpartymirrors().values()))
 882
 883 for x in scanlist:
 884         #ebuilds and digests added to cvs respectively.
 885         logging.info("checking package %s" % x)
 886         eadded=[]
 887         catdir,pkgdir=x.split("/")
 888         checkdir=repodir+"/"+x
 889         checkdir_relative = ""
 890         if repolevel < 3:
 891                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
 892         if repolevel < 2:
 893                 checkdir_relative = os.path.join(catdir, checkdir_relative)
 894         checkdir_relative = os.path.join(".", checkdir_relative)
 895
 896         if options.mode == "manifest" or \
 897           options.mode in ('commit', 'fix') and not options.pretend:
 898                 auto_assumed = set()
 899                 fetchlist_dict = portage.FetchlistDict(checkdir,
 900                         repoman_settings, portdb)
 901                 if options.mode == 'manifest' and options.force:
 902                         portage._doebuild_manifest_exempt_depend += 1
 903                         try:
 904                                 distdir = repoman_settings['DISTDIR']
 905                                 mf = portage.manifest.Manifest(checkdir, distdir,
 906                                         fetchlist_dict=fetchlist_dict)
 907                                 mf.create(requiredDistfiles=None,
 908                                         assumeDistHashesAlways=True)
 909                                 for distfiles in fetchlist_dict.values():
 910                                         for distfile in distfiles:
 911                                                 if os.path.isfile(os.path.join(distdir, distfile)):
 912                                                         mf.fhashdict['DIST'].pop(distfile, None)
 913                                                 else:
 914                                                         auto_assumed.add(distfile)
 915                                 mf.write()
 916                         finally:
 917                                 portage._doebuild_manifest_exempt_depend -= 1
 918
 919                 repoman_settings["O"] = checkdir
 920                 if not portage.digestgen([], repoman_settings, myportdb=portdb):
 921                         print("Unable to generate manifest.")
 922                         dofail = 1
 923                 if options.mode == "manifest":
 924                         if not dofail and options.force and auto_assumed and \
 925                                 'assume-digests' in repoman_settings.features:
 926                                 # Show which digests were assumed despite the --force option
 927                                 # being given. This output will already have been shown by
 928                                 # digestgen() if assume-digests is not enabled, so only show
 929                                 # it here if assume-digests is enabled.
 930                                 pkgs = list(fetchlist_dict)
 931                                 pkgs.sort()
 932                                 portage.writemsg_stdout("  digest.assumed" + \
 933                                         portage.output.colorize("WARN",
 934                                         str(len(auto_assumed)).rjust(18)) + "\n")
 935                                 for cpv in pkgs:
 936                                         fetchmap = fetchlist_dict[cpv]
 937                                         pf = portage.catsplit(cpv)[1]
 938                                         for distfile in sorted(fetchmap):
 939                                                 if distfile in auto_assumed:
 940                                                         portage.writemsg_stdout(
 941                                                                 "   %s::%s\n" % (pf, distfile))
 942                         continue
 943                 elif dofail:
 944                         sys.exit(1)
 945
 946         checkdirlist=os.listdir(checkdir)
 947         ebuildlist=[]
 948         pkgs = {}
 949         for y in checkdirlist:
 950                 if y in no_exec and \
 951                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
 952                                 stats["file.executable"] += 1
 953                                 fails["file.executable"].append(os.path.join(checkdir, y))
 954                 if y.endswith(".ebuild"):
 955                         pf = y[:-7]
 956                         ebuildlist.append(pf)
 957                         cpv = "%s/%s" % (catdir, pf)
 958                         try:
 959                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
 960                         except KeyError:
 961                                 stats["ebuild.syntax"] += 1
 962                                 fails["ebuild.syntax"].append(os.path.join(x, y))
 963                                 continue
 964                         except IOError:
 965                                 stats["ebuild.output"] += 1
 966                                 fails["ebuild.output"].append(os.path.join(x, y))
 967                                 continue
 968                         if not portage.eapi_is_supported(myaux["EAPI"]):
 969                                 stats["EAPI.unsupported"] += 1
 970                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
 971                                 continue
 972                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
 973                                 root_config=root_config)
 974
 975         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
 976         pkgsplits = {}
 977         for i in range(len(ebuildlist)):
 978                 ebuild_split = portage.pkgsplit(ebuildlist[i])
 979                 pkgsplits[ebuild_split] = ebuildlist[i]
 980                 ebuildlist[i] = ebuild_split
 981         ebuildlist.sort(key=cmp_sort_key(portage.pkgcmp))
 982         for i in range(len(ebuildlist)):
 983                 ebuildlist[i] = pkgsplits[ebuildlist[i]]
 984         del pkgsplits
 985
 986         slot_keywords = {}
 987
 988         if len(pkgs) != len(ebuildlist):
 989                 # If we can't access all the metadata then it's totally unsafe to
 990                 # commit since there's no way to generate a correct Manifest.
 991                 # Do not try to do any more QA checks on this package since missing
 992                 # metadata leads to false positives for several checks, and false
 993                 # positives confuse users.
 994                 can_force = False
 995                 continue
 996
 997         for y in checkdirlist:
 998                 m = disallowed_filename_chars_re.search(y.strip(os.sep))
 999                 if m is not None:
1000                         stats["file.name"] += 1
1001                         fails["file.name"].append("%s/%s: char '%s'" % \
1002                                 (checkdir, y, m.group(0)))
1003
1004                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1005                         continue
1006                 try:
1007                         line = 1
1008                         for l in codecs.open(_unicode_encode(os.path.join(checkdir, y),
1009                                 encoding=_encodings['fs'], errors='strict'),
1010                                 mode='r', encoding=_encodings['repo.content']):
1011                                 line +=1
1012                 except UnicodeDecodeError as ue:
1013                         stats["file.UTF8"] += 1
1014                         s = ue.object[:ue.start]
1015                         l2 = s.count("\n")
1016                         line += l2
1017                         if l2 != 0:
1018                                 s = s[s.rfind("\n") + 1:]
1019                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1020
1021         if vcs == "git" and check_ebuild_notadded:
1022                 myf = os.popen("git ls-files --others %s" % \
1023                         (portage._shell_quote(checkdir_relative),))
1024                 for l in myf:
1025                         if l[:-1][-7:] == ".ebuild":
1026                                 stats["ebuild.notadded"] += 1
1027                                 fails["ebuild.notadded"].append(
1028                                         os.path.join(x, os.path.basename(l[:-1])))
1029                 myf.close()
1030
1031         if vcs in ("cvs", "svn") and check_ebuild_notadded:
1032                 try:
1033                         if vcs == "cvs":
1034                                 myf=open(checkdir+"/CVS/Entries","r")
1035                         if vcs == "svn":
1036                                 myf = os.popen("svn status --depth=files --verbose " + checkdir)
1037                         myl = myf.readlines()
1038                         myf.close()
1039                         for l in myl:
1040                                 if vcs == "cvs":
1041                                         if l[0]!="/":
1042                                                 continue
1043                                         splitl=l[1:].split("/")
1044                                         if not len(splitl):
1045                                                 continue
1046                                         if splitl[0][-7:]==".ebuild":
1047                                                 eadded.append(splitl[0][:-7])
1048                                 if vcs == "svn":
1049                                         if l[:1] == "?":
1050                                                 continue
1051                                         if l[:7] == '      >':
1052                                                 # tree conflict, new in subversion 1.6
1053                                                 continue
1054                                         l = l.split()[-1]
1055                                         if l[-7:] == ".ebuild":
1056                                                 eadded.append(os.path.basename(l[:-7]))
1057                         if vcs == "svn":
1058                                 myf = os.popen("svn status " + checkdir)
1059                                 myl=myf.readlines()
1060                                 myf.close()
1061                                 for l in myl:
1062                                         if l[0] == "A":
1063                                                 l = l.rstrip().split(' ')[-1]
1064                                                 if l[-7:] == ".ebuild":
1065                                                         eadded.append(os.path.basename(l[:-7]))
1066                 except IOError:
1067                         if options.mode == 'commit' and vcs == "cvs":
1068                                 stats["CVS/Entries.IO_error"] += 1
1069                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1070                         if options.mode == 'commit' and vcs == "svn":
1071                                 stats["svn.IO_error"] += 1
1072                                 fails["svn.IO_error"].append(checkdir+"svn info")
1073                         continue
1074
1075         mf = Manifest(checkdir, repoman_settings["DISTDIR"])
1076         mydigests=mf.getTypeDigests("DIST")
1077
1078         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1079         myfiles_all = []
1080         src_uri_error = False
1081         for mykey in fetchlist_dict:
1082                 try:
1083                         myfiles_all.extend(fetchlist_dict[mykey])
1084                 except portage.exception.InvalidDependString as e:
1085                         src_uri_error = True
1086                         try:
1087                                 portdb.aux_get(mykey, ["SRC_URI"])
1088                         except KeyError:
1089                                 # This will be reported as an "ebuild.syntax" error.
1090                                 pass
1091                         else:
1092                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1093                                 fails["SRC_URI.syntax"].append(
1094                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1095         del fetchlist_dict
1096         if not src_uri_error:
1097                 # This test can produce false positives if SRC_URI could not
1098                 # be parsed for one or more ebuilds. There's no point in
1099                 # producing a false error here since the root cause will
1100                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1101                 # or "ebuild.sytax".
1102                 myfiles_all = set(myfiles_all)
1103                 for entry in mydigests:
1104                         if entry not in myfiles_all:
1105                                 stats["digest.unused"] += 1
1106                                 fails["digest.unused"].append(checkdir+"::"+entry)
1107                 for entry in myfiles_all:
1108                         if entry not in mydigests:
1109                                 stats["digest.missing"] += 1
1110                                 fails["digest.missing"].append(checkdir+"::"+entry)
1111         del myfiles_all
1112
1113         if os.path.exists(checkdir+"/files"):
1114                 filesdirlist=os.listdir(checkdir+"/files")
1115
1116                 # recurse through files directory
1117                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1118                 while filesdirlist:
1119                         y = filesdirlist.pop(0)
1120                         relative_path = os.path.join(x, "files", y)
1121                         full_path = os.path.join(repodir, relative_path)
1122                         try:
1123                                 mystat = os.stat(full_path)
1124                         except OSError as oe:
1125                                 if oe.errno == 2:
1126                                         # don't worry about it.  it likely was removed via fix above.
1127                                         continue
1128                                 else:
1129                                         raise oe
1130                         if S_ISDIR(mystat.st_mode):
1131                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1132                                 if y == "CVS" or y == ".svn":
1133                                         continue
1134                                 for z in os.listdir(checkdir+"/files/"+y):
1135                                         if z == "CVS" or z == ".svn":
1136                                                 continue
1137                                         filesdirlist.append(y+"/"+z)
1138                         # Current policy is no files over 20 KiB, these are the checks. File size between
1139                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1140                         elif mystat.st_size > 61440:
1141                                 stats["file.size.fatal"] += 1
1142                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1143                         elif mystat.st_size > 20480:
1144                                 stats["file.size"] += 1
1145                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1146
1147                         m = disallowed_filename_chars_re.search(
1148                                 os.path.basename(y.rstrip(os.sep)))
1149                         if m is not None:
1150                                 stats["file.name"] += 1
1151                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1152                                         (checkdir, y, m.group(0)))
1153
1154                         if desktop_file_validate and desktop_pattern.match(y):
1155                                 status, cmd_output = subprocess_getstatusoutput(
1156                                         "'%s' '%s'" % (desktop_file_validate, full_path))
1157                                 if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK:
1158                                         # Note: in the future we may want to grab the
1159                                         # warnings in addition to the errors. We're
1160                                         # just doing errors now since we don't want
1161                                         # to generate too much noise at first.
1162                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1163                                         for line in cmd_output.splitlines():
1164                                                 error_match = error_re.match(line)
1165                                                 if error_match is None:
1166                                                         continue
1167                                                 stats["desktop.invalid"] += 1
1168                                                 fails["desktop.invalid"].append(
1169                                                         relative_path + ': %s' % error_match.group(1))
1170
1171         del mydigests
1172         # Note: We don't use ChangeLogs in distributed SCMs.
1173         # It will be generated on server side from scm log,
1174         # before package moves to the rsync server.
1175         # This is needed because we try to avoid merge collisions.
1176         if vcs not in ( "git", ) and "ChangeLog" not in checkdirlist:
1177                 stats["changelog.missing"]+=1
1178                 fails["changelog.missing"].append(x+"/ChangeLog")
1179
1180         #metadata.xml file check
1181         muselist = []
1182
1183         if "metadata.xml" not in checkdirlist:
1184                 stats["metadata.missing"]+=1
1185                 fails["metadata.missing"].append(x+"/metadata.xml")
1186         #metadata.xml parse check
1187         else:
1188                 metadata_bad = False
1189
1190                 # load USE flags from metadata.xml
1191                 try:
1192                         f = open(os.path.join(checkdir, "metadata.xml"))
1193                         utilities.parse_metadata_use(f, muselist)
1194                         f.close()
1195                 except (EnvironmentError, ParseError) as e:
1196                         metadata_bad = True
1197                         stats["metadata.bad"] += 1
1198                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1199                         del e
1200
1201                 #Only carry out if in package directory or check forced
1202                 if xmllint_capable and not metadata_bad:
1203                         # xmlint can produce garbage output even on success, so only dump
1204                         # the ouput when it fails.
1205                         st, out = subprocess_getstatusoutput(
1206                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1207                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1208                         if st != os.EX_OK:
1209                                 print(red("!!!") + " metadata.xml is invalid:")
1210                                 for z in out.splitlines():
1211                                         print(red("!!! ")+z)
1212                                 stats["metadata.bad"]+=1
1213                                 fails["metadata.bad"].append(x+"/metadata.xml")
1214
1215                 del metadata_bad
1216         muselist = frozenset(muselist)
1217
1218         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1219         changelog_modified = changelog_path in modified_changelogs
1220
1221         allmasked = True
1222         # detect unused local USE-descriptions
1223         used_useflags = set()
1224
1225         for y in ebuildlist:
1226                 relative_path = os.path.join(x, y + ".ebuild")
1227                 full_path = os.path.join(repodir, relative_path)
1228                 ebuild_path = y + ".ebuild"
1229                 if repolevel < 3:
1230                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1231                 if repolevel < 2:
1232                         ebuild_path = os.path.join(catdir, ebuild_path)
1233                 ebuild_path = os.path.join(".", ebuild_path)
1234                 if not changelog_modified and ebuild_path in new_ebuilds:
1235                         stats['changelog.ebuildadded'] += 1
1236                         fails['changelog.ebuildadded'].append(relative_path)
1237
1238                 if stat.S_IMODE(os.stat(full_path).st_mode) & 0o111:
1239                         stats["file.executable"] += 1
1240                         fails["file.executable"].append(x+"/"+y+".ebuild")
1241                 if vcs in ("cvs", "svn") and check_ebuild_notadded and y not in eadded:
1242                         #ebuild not added to vcs
1243                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1244                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1245                 myesplit=portage.pkgsplit(y)
1246                 if myesplit is None or myesplit[0] != x.split("/")[-1]:
1247                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1248                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1249                         continue
1250                 elif myesplit[0]!=pkgdir:
1251                         print(pkgdir,myesplit[0])
1252                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1253                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1254                         continue
1255
1256                 pkg = pkgs[y]
1257
1258                 if pkg.invalid:
1259                         for k, msgs in pkg.invalid.items():
1260                                 for msg in msgs:
1261                                         stats[k] = stats[k] + 1
1262                                         fails[k].append("%s %s" % (relative_path, msg))
1263                         continue
1264
1265                 myaux = pkg.metadata
1266                 eapi = myaux["EAPI"]
1267                 inherited = pkg.inherited
1268                 live_ebuild = live_eclasses.intersection(inherited)
1269
1270                 for k, v in myaux.items():
1271                         if not isinstance(v, basestring):
1272                                 continue
1273                         m = non_ascii_re.search(v)
1274                         if m is not None:
1275                                 stats["variable.invalidchar"] += 1
1276                                 fails["variable.invalidchar"].append(
1277                                         ("%s: %s variable contains non-ASCII " + \
1278                                         "character at position %s") % \
1279                                         (relative_path, k, m.start() + 1))
1280
1281                 if not src_uri_error:
1282                         # Check that URIs don't reference a server from thirdpartymirrors.
1283                         for uri in portage.flatten(portage.dep.use_reduce(
1284                                 portage.dep.paren_reduce(myaux["SRC_URI"]), matchall=True)):
1285                                 contains_mirror = False
1286                                 for mirror in thirdpartymirrors:
1287                                         if uri.startswith(mirror):
1288                                                 contains_mirror = True
1289                                                 break
1290                                 if not contains_mirror:
1291                                         continue
1292
1293                                 stats["SRC_URI.mirror"] += 1
1294                                 fails["SRC_URI.mirror"].append(
1295                                         "%s: '%s' found in thirdpartymirrors" % \
1296                                         (relative_path, mirror))
1297
1298                 # The Package class automatically evaluates USE conditionals.
1299                 for myprovide in portage.flatten(portage.dep.use_reduce(
1300                         portage.dep.paren_reduce(pkg.metadata['PROVIDE']), matchall=1)):
1301                         prov_cp = portage.dep_getkey(myprovide)
1302                         if prov_cp != myprovide:
1303                                 stats["virtual.versioned"]+=1
1304                                 fails["virtual.versioned"].append(x+"/"+y+".ebuild: "+myprovide)
1305                         prov_pkg = portage.dep_getkey(
1306                                 portage.best(portdb.xmatch("match-all", prov_cp)))
1307                         if prov_cp == prov_pkg:
1308                                 stats["virtual.exists"]+=1
1309                                 fails["virtual.exists"].append(x+"/"+y+".ebuild: "+prov_cp)
1310
1311                 for pos, missing_var in enumerate(missingvars):
1312                         if not myaux.get(missing_var):
1313                                 if catdir == "virtual" and \
1314                                         missing_var in ("HOMEPAGE", "LICENSE"):
1315                                         continue
1316                                 if live_ebuild and missing_var == "KEYWORDS":
1317                                         continue
1318                                 myqakey=missingvars[pos]+".missing"
1319                                 stats[myqakey]=stats[myqakey]+1
1320                                 fails[myqakey].append(x+"/"+y+".ebuild")
1321
1322                 # 14 is the length of DESCRIPTION=""
1323                 if len(myaux['DESCRIPTION']) > max_desc_len:
1324                         stats['DESCRIPTION.toolong'] += 1
1325                         fails['DESCRIPTION.toolong'].append(
1326                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1327                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1328
1329                 keywords = myaux["KEYWORDS"].split()
1330                 stable_keywords = []
1331                 for keyword in keywords:
1332                         if not keyword.startswith("~") and \
1333                                 not keyword.startswith("-"):
1334                                 stable_keywords.append(keyword)
1335                 if stable_keywords:
1336                         if ebuild_path in new_ebuilds:
1337                                 stable_keywords.sort()
1338                                 stats["KEYWORDS.stable"] += 1
1339                                 fails["KEYWORDS.stable"].append(
1340                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1341                                                 " ".join(stable_keywords))
1342
1343                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1344                         if not kw.startswith("-"))
1345
1346                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1347                 if previous_keywords is None:
1348                         slot_keywords[myaux["SLOT"]] = set()
1349                 elif not live_ebuild:
1350                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1351                         if dropped_keywords:
1352                                 stats["KEYWORDS.dropped"] += 1
1353                                 fails["KEYWORDS.dropped"].append(
1354                                         relative_path + ": %s" % \
1355                                         " ".join(sorted(dropped_keywords)))
1356
1357                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1358
1359                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1360                 if "-*" in keywords:
1361                         haskeyword = False
1362                         for kw in keywords:
1363                                 if kw[0] == "~":
1364                                         kw = kw[1:]
1365                                 if kw in kwlist:
1366                                         haskeyword = True
1367                         if not haskeyword:
1368                                 stats["KEYWORDS.stupid"] += 1
1369                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1370
1371                 """
1372                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1373                 not be allowed to be marked stable
1374                 """
1375                 if live_ebuild:
1376                         bad_stable_keywords = []
1377                         for keyword in keywords:
1378                                 if not keyword.startswith("~") and \
1379                                         not keyword.startswith("-"):
1380                                         bad_stable_keywords.append(keyword)
1381                                 del keyword
1382                         if bad_stable_keywords:
1383                                 stats["LIVEVCS.stable"] += 1
1384                                 fails["LIVEVCS.stable"].append(
1385                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1386                                                 bad_stable_keywords)
1387                         del bad_stable_keywords
1388
1389                         if keywords and not has_global_mask(pkg):
1390                                 stats["LIVEVCS.unmasked"] += 1
1391                                 fails["LIVEVCS.unmasked"].append(relative_path)
1392
1393                 if options.ignore_arches:
1394                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1395                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1396                 else:
1397                         arches=[]
1398                         for keyword in myaux["KEYWORDS"].split():
1399                                 if (keyword[0]=="-"):
1400                                         continue
1401                                 elif (keyword[0]=="~"):
1402                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1403                                 else:
1404                                         arches.append([keyword, keyword, [keyword]])
1405                                         allmasked = False
1406
1407                 baddepsyntax = False
1408                 badlicsyntax = False
1409                 badprovsyntax = False
1410                 catpkg = catdir+"/"+y
1411                 myiuse = set(repoman_settings.archlist())
1412                 for myflag in myaux["IUSE"].split():
1413                         if myflag.startswith("+"):
1414                                 myflag = myflag[1:]
1415                         myiuse.add(myflag)
1416
1417                 inherited_java_eclass = "java-pkg-2" in inherited or \
1418                         "java-pkg-opt-2" in inherited
1419                 operator_tokens = set(["||", "(", ")"])
1420                 type_list, badsyntax = [], []
1421                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1422                         "LICENSE", "PROPERTIES", "PROVIDE"):
1423                         mydepstr = myaux[mytype]
1424
1425                         if mydepstr.find(" ?") != -1:
1426                                 badsyntax.append("'?' preceded by space")
1427
1428                         try:
1429                                 # Missing closing parenthesis will result in a ValueError
1430                                 mydeplist = portage.dep.paren_reduce(mydepstr)
1431                                 # Missing opening parenthesis will result in a final "" element
1432                                 if "" in mydeplist or "(" in mydeplist:
1433                                         raise ValueError
1434                         except ValueError:
1435                                 badsyntax.append("parenthesis mismatch")
1436                                 mydeplist = []
1437                         except portage.exception.InvalidDependString as e:
1438                                 badsyntax.append(str(e))
1439                                 del e
1440                                 mydeplist = []
1441
1442                         try:
1443                                 portage.dep.use_reduce(mydeplist, matchall=1)
1444                         except portage.exception.InvalidDependString as e:
1445                                 badsyntax.append(str(e))
1446
1447                         for token in operator_tokens:
1448                                 if mydepstr.startswith(token+" "):
1449                                         myteststr = mydepstr[len(token):]
1450                                 else:
1451                                         myteststr = mydepstr
1452                                 if myteststr.endswith(" "+token):
1453                                         myteststr = myteststr[:-len(token)]
1454                                 while myteststr.find(" "+token+" ") != -1:
1455                                         myteststr = " ".join(myteststr.split(" "+token+" ", 1))
1456                                 if myteststr.find(token) != -1:
1457                                         badsyntax.append("'%s' not separated by space" % (token))
1458
1459                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1460                                 for token in mydepstr.split():
1461                                         if token in operator_tokens or \
1462                                                 token[-1:] == "?":
1463                                                 if token == "test?" and \
1464                                                         mytype in ("RDEPEND", "PDEPEND"):
1465                                                         stats[mytype + '.suspect'] += 1
1466                                                         fails[mytype + '.suspect'].append(relative_path + \
1467                                                                 ": 'test?' USE conditional in %s" % mytype)
1468                                                 continue
1469                                         try:
1470                                                 atom = portage.dep.Atom(token)
1471                                         except portage.exception.InvalidAtom:
1472                                                 badsyntax.append("'%s' not a valid atom" % token)
1473                                         else:
1474                                                 is_blocker = atom.blocker
1475
1476                                                 if mytype == "DEPEND" and \
1477                                                         not is_blocker and \
1478                                                         not inherited_java_eclass and \
1479                                                         portage.dep_getkey(atom) == "virtual/jdk":
1480                                                         stats['java.eclassesnotused'] += 1
1481                                                         fails['java.eclassesnotused'].append(relative_path)
1482                                                 elif mytype in ("PDEPEND", "RDEPEND"):
1483                                                         if not is_blocker and \
1484                                                                 portage.dep_getkey(atom) in suspect_rdepend:
1485                                                                 stats[mytype + '.suspect'] += 1
1486                                                                 fails[mytype + '.suspect'].append(
1487                                                                         relative_path + ": '%s'" % atom)
1488                                                 if eapi == "0":
1489                                                         if portage.dep.dep_getslot(atom):
1490                                                                 stats['EAPI.incompatible'] += 1
1491                                                                 fails['EAPI.incompatible'].append(
1492                                                                         (relative_path + ": %s slot dependency" + \
1493                                                                         " not supported with EAPI='%s':" + \
1494                                                                         " '%s'") % (mytype, eapi, atom))
1495                                                 if atom.use and eapi in ("0", "1"):
1496                                                         stats['EAPI.incompatible'] += 1
1497                                                         fails['EAPI.incompatible'].append(
1498                                                                 (relative_path + ": %s use dependency" + \
1499                                                                 " not supported with EAPI='%s':" + \
1500                                                                 " '%s'") % (mytype, eapi, atom))
1501                                                 if atom.blocker and atom.blocker.overlap.forbid \
1502                                                         and eapi in ("0", "1"):
1503                                                         stats['EAPI.incompatible'] += 1
1504                                                         fails['EAPI.incompatible'].append(
1505                                                                 (relative_path + ": %s new blocker syntax" + \
1506                                                                 " not supported with EAPI='%s':" + \
1507                                                                 " '%s'") % (mytype, eapi, atom))
1508
1509                                                 if atom.operator == "~" and \
1510                                                         portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1511                                                         stats[mytype + '.badtilde'] += 1
1512                                                         fails[mytype + '.badtilde'].append(
1513                                                                 (relative_path + ": %s uses the ~ operator"
1514                                                                  " with a non-zero revision:" + \
1515                                                                  " '%s'") % (mytype, atom))
1516
1517                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1518
1519                 for m,b in zip(type_list, badsyntax):
1520                         stats[m+".syntax"] += 1
1521                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1522
1523                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1524                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1525                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1526                 badlicsyntax = badlicsyntax > 0
1527                 badprovsyntax = badprovsyntax > 0
1528
1529                 # uselist checks - global
1530                 myuse = []
1531                 default_use = []
1532                 for myflag in myaux["IUSE"].split():
1533                         flag_name = myflag.lstrip("+-")
1534                         used_useflags.add(flag_name)
1535                         if myflag != flag_name:
1536                                 default_use.append(myflag)
1537                         if flag_name not in uselist:
1538                                 myuse.append(flag_name)
1539
1540                 # uselist checks - metadata
1541                 for mypos in range(len(myuse)-1,-1,-1):
1542                         if myuse[mypos] and (myuse[mypos] in muselist):
1543                                 del myuse[mypos]
1544
1545                 if default_use and eapi == "0":
1546                         for myflag in default_use:
1547                                 stats['EAPI.incompatible'] += 1
1548                                 fails['EAPI.incompatible'].append(
1549                                         (relative_path + ": IUSE defaults" + \
1550                                         " not supported with EAPI='%s':" + \
1551                                         " '%s'") % (eapi, myflag))
1552
1553                 for mypos in range(len(myuse)):
1554                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1555                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1556
1557                 # license checks
1558                 if not badlicsyntax:
1559                         myuse = myaux["LICENSE"]
1560                         # Parse the LICENSE variable, remove USE conditions and
1561                         # flatten it.
1562                         myuse=portage.dep.use_reduce(portage.dep.paren_reduce(myuse), matchall=1)
1563                         myuse=portage.flatten(myuse)
1564                         # Check each entry to ensure that it exists in PORTDIR's
1565                         # license directory.
1566                         for mypos in range(0,len(myuse)):
1567                                 # Need to check for "||" manually as no portage
1568                                 # function will remove it without removing values.
1569                                 if myuse[mypos] not in liclist and myuse[mypos] != "||":
1570                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1571                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1572
1573                 #keyword checks
1574                 myuse = myaux["KEYWORDS"].split()
1575                 for mykey in myuse:
1576                         myskey=mykey[:]
1577                         if myskey[0]=="-":
1578                                 myskey=myskey[1:]
1579                         if myskey[0]=="~":
1580                                 myskey=myskey[1:]
1581                         if mykey!="-*":
1582                                 if myskey not in kwlist:
1583                                         stats["KEYWORDS.invalid"] += 1
1584                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1585                                 elif myskey not in profiles:
1586                                         stats["KEYWORDS.invalid"] += 1
1587                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1588
1589                 #restrict checks
1590                 myrestrict = None
1591                 try:
1592                         myrestrict = portage.dep.use_reduce(
1593                                 portage.dep.paren_reduce(myaux["RESTRICT"]), matchall=1)
1594                 except portage.exception.InvalidDependString as e:
1595                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1596                         fails["RESTRICT.syntax"].append(
1597                                 "%s: RESTRICT: %s" % (relative_path, e))
1598                         del e
1599                 if myrestrict:
1600                         myrestrict = set(portage.flatten(myrestrict))
1601                         mybadrestrict = myrestrict.difference(valid_restrict)
1602                         if mybadrestrict:
1603                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
1604                                 for mybad in mybadrestrict:
1605                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1606                 # Syntax Checks
1607                 relative_path = os.path.join(x, y + ".ebuild")
1608                 full_path = os.path.join(repodir, relative_path)
1609                 try:
1610                         # All ebuilds should have utf_8 encoding.
1611                         f = codecs.open(_unicode_encode(full_path,
1612                                 encoding=_encodings['fs'], errors='strict'),
1613                                 mode='r', encoding=_encodings['repo.content'])
1614                         try:
1615                                 for check_name, e in run_checks(f, pkg):
1616                                         stats[check_name] += 1
1617                                         fails[check_name].append(relative_path + ': %s' % e)
1618                         finally:
1619                                 f.close()
1620                 except UnicodeDecodeError:
1621                         # A file.UTF8 failure will have already been recorded above.
1622                         pass
1623
1624                 if options.force:
1625                         # The dep_check() calls are the most expensive QA test. If --force
1626                         # is enabled, there's no point in wasting time on these since the
1627                         # user is intent on forcing the commit anyway.
1628                         continue
1629
1630                 for keyword,arch,groups in arches:
1631
1632                         if arch not in profiles:
1633                                 # A missing profile will create an error further down
1634                                 # during the KEYWORDS verification.
1635                                 continue
1636
1637                         for prof in profiles[arch]:
1638
1639                                 if prof.status not in ("stable", "dev") or \
1640                                         prof.status == "dev" and not options.include_dev:
1641                                         continue
1642
1643                                 dep_settings = arch_caches.get(prof.sub_path)
1644                                 if dep_settings is None:
1645                                         dep_settings = portage.config(
1646                                                 config_profile_path=prof.abs_path,
1647                                                 config_incrementals=portage.const.INCREMENTALS,
1648                                                 local_config=False,
1649                                                 env=env)
1650                                         if options.without_mask:
1651                                                 dep_settings.pmaskdict.clear()
1652                                         arch_caches[prof.sub_path] = dep_settings
1653                                         while True:
1654                                                 try:
1655                                                         # Protect ACCEPT_KEYWORDS from config.regenerate()
1656                                                         # (just in case)
1657                                                         dep_settings.incrementals.remove("ACCEPT_KEYWORDS")
1658                                                 except ValueError:
1659                                                         break
1660
1661                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
1662                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
1663                                 if xcache is None:
1664                                         portdb.melt()
1665                                         portdb.freeze()
1666                                         xcache = portdb.xcache
1667                                         xcache.update(shared_xmatch_caches)
1668                                         arch_xmatch_caches[xmatch_cache_key] = xcache
1669
1670                                 trees["/"]["porttree"].settings = dep_settings
1671                                 portdb.mysettings = dep_settings
1672                                 portdb.xcache = xcache
1673                                 # for package.use.mask support inside dep_check
1674                                 dep_settings.setcpv(pkg)
1675                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1676                                 # just in case, prevent config.reset() from nuking these.
1677                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1678
1679                                 for myprovide in myaux["PROVIDE"].split():
1680                                         prov_cp = portage.dep_getkey(myprovide)
1681                                         if prov_cp not in dep_settings.getvirtuals():
1682                                                 stats["virtual.unavailable"]+=1
1683                                                 fails["virtual.unavailable"].append("%s: %s(%s) %s" % \
1684                                                         (relative_path, keyword, prof.sub_path, prov_cp))
1685
1686                                 if not baddepsyntax:
1687                                         ismasked = os.path.join(catdir, y) not in \
1688                                                 portdb.xmatch("list-visible", x)
1689                                         if ismasked:
1690                                                 if not have_pmasked:
1691                                                         have_pmasked = bool(dep_settings._getMaskAtom(
1692                                                                 pkg.cpv, pkg.metadata))
1693                                                 if options.ignore_masked:
1694                                                         continue
1695                                                 #we are testing deps for a masked package; give it some lee-way
1696                                                 suffix="masked"
1697                                                 matchmode = "minimum-all"
1698                                         else:
1699                                                 suffix=""
1700                                                 matchmode = "minimum-visible"
1701
1702                                         if not have_dev_keywords:
1703                                                 have_dev_keywords = \
1704                                                         bool(dev_keywords.intersection(keywords))
1705
1706                                         if prof.status == "dev":
1707                                                 suffix=suffix+"indev"
1708
1709                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1710
1711                                                 mykey=mytype+".bad"+suffix
1712                                                 myvalue = myaux[mytype]
1713                                                 if not myvalue:
1714                                                         continue
1715
1716                                                 success, atoms = portage.dep_check(myvalue, portdb,
1717                                                         dep_settings, use="all", mode=matchmode,
1718                                                         trees=trees)
1719
1720                                                 if success:
1721                                                         if atoms:
1722                                                                 #we have some unsolvable deps
1723                                                                 #remove ! deps, which always show up as unsatisfiable
1724                                                                 atoms = [str(atom) for atom in atoms if not atom.blocker]
1725                                                                 #if we emptied out our list, continue:
1726                                                                 if not atoms:
1727                                                                         continue
1728                                                                 stats[mykey]=stats[mykey]+1
1729                                                                 fails[mykey].append("%s: %s(%s) %s" % \
1730                                                                         (relative_path, keyword,
1731                                                                         prof.sub_path, repr(atoms)))
1732                                                 else:
1733                                                         stats[mykey]=stats[mykey]+1
1734                                                         fails[mykey].append("%s: %s(%s) %s" % \
1735                                                                 (relative_path, keyword,
1736                                                                 prof.sub_path, repr(atoms)))
1737
1738         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1739         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1740         #if not portage.portdb.xmatch("bestmatch-visible",x):
1741         #    stats["ebuild.nostable"]+=1
1742         #    fails["ebuild.nostable"].append(x)
1743         if allmasked and repolevel == 3:
1744                 stats["ebuild.allmasked"]+=1
1745                 fails["ebuild.allmasked"].append(x)
1746
1747         # check if there are unused local USE-descriptions in metadata.xml
1748         for myflag in muselist.difference(used_useflags):
1749                 stats["metadata.warning"] += 1
1750                 fails["metadata.warning"].append(
1751                         "%s/metadata.xml: unused local USE-description: '%s'" % \
1752                         (x, myflag))
1753
1754 if options.mode == "manifest":
1755         sys.exit(dofail)
1756
1757 #dofail will be set to 1 if we have failed in at least one non-warning category
1758 dofail=0
1759 #dowarn will be set to 1 if we tripped any warnings
1760 dowarn=0
1761 #dofull will be set if we should print a "repoman full" informational message
1762 dofull = options.mode != 'full'
1763
1764 for x in qacats:
1765         if not stats[x]:
1766                 continue
1767         dowarn = 1
1768         if x not in qawarnings:
1769                 dofail = 1
1770
1771 if dofail or \
1772         (dowarn and not (options.quiet or options.mode == "scan")):
1773         dofull = 0
1774
1775 # Save QA output so that it can be conveniently displayed
1776 # in $EDITOR while the user creates a commit message.
1777 # Otherwise, the user would not be able to see this output
1778 # once the editor has taken over the screen.
1779 qa_output = StringIO()
1780 style_file = ConsoleStyleFile(sys.stdout)
1781 if options.mode == 'commit' and \
1782         (not commitmessage or not commitmessage.strip()):
1783         style_file.write_listener = qa_output
1784 console_writer = StyleWriter(file=style_file, maxcol=9999)
1785 console_writer.style_listener = style_file.new_styles
1786
1787 f = formatter.AbstractFormatter(console_writer)
1788
1789 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
1790
1791 style_file.flush()
1792 del console_writer, f, style_file
1793 qa_output = qa_output.getvalue()
1794 qa_output = qa_output.splitlines(True)
1795
1796 def grouplist(mylist,seperator="/"):
1797         """(list,seperator="/") -- Takes a list of elements; groups them into
1798         same initial element categories. Returns a dict of {base:[sublist]}
1799         From: ["blah/foo","spork/spatula","blah/weee/splat"]
1800         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
1801         mygroups={}
1802         for x in mylist:
1803                 xs=x.split(seperator)
1804                 if xs[0]==".":
1805                         xs=xs[1:]
1806                 if xs[0] not in mygroups:
1807                         mygroups[xs[0]]=[seperator.join(xs[1:])]
1808                 else:
1809                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
1810         return mygroups
1811
1812 suggest_ignore_masked = False
1813 suggest_include_dev = False
1814
1815 if have_pmasked and not (options.without_mask or options.ignore_masked):
1816         suggest_ignore_masked = True
1817 if have_dev_keywords and not options.include_dev:
1818         suggest_include_dev = True
1819
1820 if suggest_ignore_masked or suggest_include_dev:
1821         print()
1822         if suggest_ignore_masked:
1823                 print(bold("Note: use --without-mask to check " + \
1824                         "KEYWORDS on dependencies of masked packages"))
1825
1826         if suggest_include_dev:
1827                 print(bold("Note: use --include-dev (-d) to check " + \
1828                         "dependencies for 'dev' profiles"))
1829         print()
1830
1831 if options.mode != 'commit':
1832         if dofull:
1833                 print(bold("Note: type \"repoman full\" for a complete listing."))
1834         if dowarn and not dofail:
1835                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
1836         elif not dofail:
1837                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
1838         elif dofail:
1839                 print(turquoise("Please fix these important QA issues first."))
1840                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
1841                 sys.exit(1)
1842 else:
1843         if dofail and can_force and options.force and not options.pretend:
1844                 print(green("RepoMan sez:") + \
1845                         " \"You want to commit even with these QA issues?\n" + \
1846                         "              I'll take it this time, but I'm not happy.\"\n")
1847         elif dofail:
1848                 if options.force and not can_force:
1849                         print(bad("The --force option has been disabled due to extraordinary issues."))
1850                 print(turquoise("Please fix these important QA issues first."))
1851                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
1852                 sys.exit(1)
1853
1854         if options.pretend:
1855                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
1856
1857         myunadded = []
1858         if vcs == "cvs":
1859                 try:
1860                         myvcstree=portage.cvstree.getentries("./",recursive=1)
1861                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
1862                 except SystemExit as e:
1863                         raise  # TODO propogate this
1864                 except:
1865                         err("Error retrieving CVS tree; exiting.")
1866         if vcs == "svn":
1867                 try:
1868                         svnstatus=os.popen("svn status --no-ignore").readlines()
1869                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
1870                 except SystemExit as e:
1871                         raise  # TODO propogate this
1872                 except:
1873                         err("Error retrieving SVN info; exiting.")
1874         if vcs == "git":
1875                 # get list of files not under version control or missing
1876                 myf = os.popen("git ls-files --others")
1877                 myunadded = [ "./" + elem[:-1] for elem in myf ]
1878                 myf.close()
1879
1880         myautoadd=[]
1881         if myunadded:
1882                 for x in range(len(myunadded)-1,-1,-1):
1883                         xs=myunadded[x].split("/")
1884                         if xs[-1]=="files":
1885                                 print("!!! files dir is not added! Please correct this.")
1886                                 sys.exit(-1)
1887                         elif xs[-1]=="Manifest":
1888                                 # It's a manifest... auto add
1889                                 myautoadd+=[myunadded[x]]
1890                                 del myunadded[x]
1891
1892         if myautoadd:
1893                 print(">>> Auto-Adding missing Manifest(s)...")
1894                 if options.pretend:
1895                         if vcs == "cvs":
1896                                 print("(cvs add "+" ".join(myautoadd)+")")
1897                         if vcs == "svn":
1898                                 print("(svn add "+" ".join(myautoadd)+")")
1899                         elif vcs == "git":
1900                                 print("(git add "+" ".join(myautoadd)+")")
1901                         retval=0
1902                 else:
1903                         if vcs == "cvs":
1904                                 retval=os.system("cvs add "+" ".join(myautoadd))
1905                         if vcs == "svn":
1906                                 retval=os.system("svn add "+" ".join(myautoadd))
1907                         elif vcs == "git":
1908                                 retval=os.system("git add "+" ".join(myautoadd))
1909                 if retval:
1910                         writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \
1911                                 (vcs, retval), level=logging.ERROR, noiselevel=-1)
1912                         sys.exit(retval)
1913
1914         if myunadded:
1915                 print(red("!!! The following files are in your local tree but are not added to the master"))
1916                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
1917                 for x in myunadded:
1918                         print("   ",x)
1919                 print()
1920                 print()
1921                 sys.exit(1)
1922
1923         if vcs == "cvs":
1924                 mycvstree = cvstree.getentries("./", recursive=1)
1925                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1926                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1927                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
1928                 bin_blob_pattern = re.compile("^-kb$")
1929                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
1930                         recursive=1, basedir="./"))
1931
1932
1933         if vcs == "svn":
1934                 svnstatus = os.popen("svn status").readlines()
1935                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1936                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1937                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D") ]
1938                 # in contrast to CVS, SVN expands nothing by default.
1939                 # bin_blobs historically
1940                 # were just there to see what files need to be checked for
1941                 # keyword expansion, which is exactly what we do here, so
1942                 # slightly change the semantic meaning of "bin_blob"...  In the
1943                 # future we could store which keyword is expanded.
1944                 props = os.popen("svn propget -R svn:keywords").readlines()
1945
1946                 # For files with multiple props set, props are delimited by newlines,
1947                 # so exclude lines that don't contain " - " since each of those lines
1948                 # only a contain props for a file listed on a previous line.
1949                 expansion = set("./" + prop.split(" - ")[0] \
1950                         for prop in props if " - " in prop)
1951
1952         elif vcs == "git":
1953                 strip_levels = repolevel - 1
1954
1955                 mychanged = os.popen("git diff-index --name-only --diff-filter=M HEAD").readlines()
1956                 if strip_levels:
1957                         mychanged = [elem[repo_subdir_len:] for elem in mychanged \
1958                                 if elem[:repo_subdir_len] == repo_subdir]
1959                 mychanged = ["./" + elem[:-1] for elem in mychanged]
1960
1961                 mynew = os.popen("git diff-index --name-only --diff-filter=A HEAD").readlines()
1962                 if strip_levels:
1963                         mynew = [elem[repo_subdir_len:] for elem in mynew \
1964                                 if elem[:repo_subdir_len] == repo_subdir]
1965                 mynew = ["./" + elem[:-1] for elem in mynew]
1966
1967                 myremoved = os.popen("git diff-index --name-only --diff-filter=D HEAD").readlines()
1968                 if strip_levels:
1969                         myremoved = [elem[repo_subdir_len:] for elem in myremoved \
1970                                 if elem[:repo_subdir_len] == repo_subdir]
1971                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1972
1973         if vcs:
1974                 if not (mychanged or mynew or myremoved):
1975                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
1976                         print()
1977                         print("(Didn't find any changed files...)")
1978                         print()
1979                         sys.exit(1)
1980
1981         # Manifests need to be regenerated after all other commits, so don't commit
1982         # them now even if they have changed.
1983         mymanifests = set()
1984         myupdates = set()
1985         for f in mychanged + mynew:
1986                 if "Manifest" == os.path.basename(f):
1987                         mymanifests.add(f)
1988                 else:
1989                         myupdates.add(f)
1990         if vcs == 'git':
1991                 myupdates.difference_update(myremoved)
1992         myupdates = list(myupdates)
1993         mymanifests = list(mymanifests)
1994         myheaders = []
1995         mydirty = []
1996         headerstring = "'\$(Header|Id)"
1997         headerstring += ".*\$'"
1998         for myfile in myupdates:
1999
2000                 # for CVS, no_expansion contains files that are excluded from expansion
2001                 if vcs == "cvs":
2002                         if myfile in no_expansion:
2003                                 continue
2004
2005                 # for SVN, expansion contains files that are included in expansion
2006                 elif vcs == "svn":
2007                         if myfile not in expansion:
2008                                 continue
2009
2010                 myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile)
2011                 if myout[0] == 0:
2012                         myheaders.append(myfile)
2013
2014         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2015         if vcs == 'git':
2016                 # With git, there's never any keyword expansion, so there's
2017                 # no need to regenerate manifests and all files will be
2018                 # committed in one big commit at the end.
2019                 print()
2020         else:
2021                 print("%s have headers that will change." % green(str(len(myheaders))))
2022                 print("* Files with headers will cause the " + \
2023                         "manifests to be made and recommited.")
2024         logging.info("myupdates: %s", myupdates)
2025         logging.info("myheaders: %s", myheaders)
2026
2027         commitmessage = options.commitmsg
2028         if options.commitmsgfile:
2029                 try:
2030                         f = open(options.commitmsgfile)
2031                         commitmessage = f.read()
2032                         f.close()
2033                         del f
2034                 except (IOError, OSError) as e:
2035                         if e.errno == errno.ENOENT:
2036                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2037                         else:
2038                                 raise
2039                 # We've read the content so the file is no longer needed.
2040                 commitmessagefile = None
2041         if not commitmessage or not commitmessage.strip():
2042                 try:
2043                         editor = os.environ.get("EDITOR")
2044                         if editor and utilities.editor_is_executable(editor):
2045                                 commitmessage = utilities.get_commit_message_with_editor(
2046                                         editor, message=qa_output)
2047                         else:
2048                                 commitmessage = utilities.get_commit_message_with_stdin()
2049                 except KeyboardInterrupt:
2050                         exithandler()
2051                 if not commitmessage or not commitmessage.strip():
2052                         print("* no commit message?  aborting commit.")
2053                         sys.exit(1)
2054         commitmessage = commitmessage.rstrip()
2055         portage_version = getattr(portage, "VERSION", None)
2056         if portage_version is None:
2057                 sys.stderr.write("Failed to insert portage version in message!\n")
2058                 sys.stderr.flush()
2059                 portage_version = "Unknown"
2060         unameout = platform.system() + " "
2061         if platform.system() in ["Darwin", "SunOS"]:
2062                 unameout += platform.processor()
2063         else:
2064                 unameout += platform.machine()
2065         commitmessage += "\n(Portage version: %s/%s/%s" % \
2066                 (portage_version, vcs, unameout)
2067         if options.force:
2068                 commitmessage += ", RepoMan options: --force"
2069         commitmessage += ")"
2070
2071         if vcs != 'git' and (myupdates or myremoved):
2072                 myfiles = myupdates + myremoved
2073                 if not myheaders and "sign" not in repoman_settings.features:
2074                         myfiles += mymanifests
2075                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2076                 mymsg = os.fdopen(fd, "w")
2077                 mymsg.write(commitmessage)
2078                 mymsg.close()
2079
2080                 print()
2081                 print(green("Using commit message:"))
2082                 print(green("------------------------------------------------------------------------------"))
2083                 print(commitmessage)
2084                 print(green("------------------------------------------------------------------------------"))
2085                 print()
2086
2087                 # Having a leading ./ prefix on file paths can trigger a bug in
2088                 # the cvs server when committing files to multiple directories,
2089                 # so strip the prefix.
2090                 myfiles = [f.lstrip("./") for f in myfiles]
2091
2092                 commit_cmd = [vcs]
2093                 commit_cmd.extend(vcs_global_opts)
2094                 commit_cmd.append("commit")
2095                 commit_cmd.extend(vcs_local_opts)
2096                 commit_cmd.extend(["-F", commitmessagefile])
2097                 commit_cmd.extend(myfiles)
2098
2099                 try:
2100                         if options.pretend:
2101                                 print("(%s)" % (" ".join(commit_cmd),))
2102                         else:
2103                                 retval = spawn(commit_cmd, env=os.environ)
2104                                 if retval != os.EX_OK:
2105                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2106                                                 "error code: %s\n") % (vcs, retval),
2107                                                 level=logging.ERROR, noiselevel=-1)
2108                                         sys.exit(retval)
2109                 finally:
2110                         try:
2111                                 os.unlink(commitmessagefile)
2112                         except OSError:
2113                                 pass
2114
2115         # Setup the GPG commands
2116         def gpgsign(filename):
2117                 if "PORTAGE_GPG_KEY" not in repoman_settings:
2118                         raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
2119                 if "PORTAGE_GPG_DIR" not in repoman_settings:
2120                         if "HOME" in os.environ:
2121                                 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(os.environ["HOME"], ".gnupg")
2122                                 logging.info("Automatically setting PORTAGE_GPG_DIR to %s" % repoman_settings["PORTAGE_GPG_DIR"])
2123                         else:
2124                                 raise portage.exception.MissingParameter("PORTAGE_GPG_DIR is unset!")
2125                 gpg_dir = repoman_settings["PORTAGE_GPG_DIR"]
2126                 if gpg_dir.startswith("~") and "HOME" in os.environ:
2127                         repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(
2128                                 os.environ["HOME"], gpg_dir[1:].lstrip(os.path.sep))
2129                 if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2130                         raise portage.exception.InvalidLocation(
2131                                 "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2132                                 repoman_settings["PORTAGE_GPG_DIR"])
2133                 gpgcmd = "gpg --sign --clearsign --yes "
2134                 gpgcmd+= "--default-key "+repoman_settings["PORTAGE_GPG_KEY"]
2135                 if "PORTAGE_GPG_DIR" in repoman_settings:
2136                         gpgcmd += " --homedir "+repoman_settings["PORTAGE_GPG_DIR"]
2137                 if options.pretend:
2138                         print("("+gpgcmd+" "+filename+")")
2139                 else:
2140                         rValue = os.system(gpgcmd+" "+filename)
2141                         if rValue == os.EX_OK:
2142                                 os.rename(filename+".asc", filename)
2143                         else:
2144                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2145
2146         # When files are removed and re-added, the cvs server will put /Attic/
2147         # inside the $Header path. This code detects the problem and corrects it
2148         # so that the Manifest will generate correctly. See bug #169500.
2149         from portage.util import write_atomic
2150         cvs_header = re.compile(r'^#\s*\$Header.*\$$')
2151         for x in myheaders:
2152                 f = open(x)
2153                 mylines = f.readlines()
2154                 f.close()
2155                 modified = False
2156                 for i, line in enumerate(mylines):
2157                         if cvs_header.match(line) and "/Attic/" in line:
2158                                 mylines[i] = line.replace("/Attic/", "/")
2159                                 modified = True
2160                 if modified:
2161                         write_atomic(x, "".join(mylines))
2162
2163         manifest_commit_required = True
2164         if vcs != 'git' and (myupdates or myremoved):
2165                 myfiles = myupdates + myremoved
2166                 for x in range(len(myfiles)-1, -1, -1):
2167                         if myfiles[x].count("/") < 4-repolevel:
2168                                 del myfiles[x]
2169                 mydone=[]
2170                 if repolevel==3:   # In a package dir
2171                         repoman_settings["O"] = startdir
2172                         portage.digestgen([], repoman_settings, manifestonly=1,
2173                                 myportdb=portdb)
2174                 elif repolevel==2: # In a category dir
2175                         for x in myfiles:
2176                                 xs=x.split("/")
2177                                 if len(xs) < 4-repolevel:
2178                                         continue
2179                                 if xs[0]==".":
2180                                         xs=xs[1:]
2181                                 if xs[0] in mydone:
2182                                         continue
2183                                 mydone.append(xs[0])
2184                                 repoman_settings["O"] = os.path.join(startdir, xs[0])
2185                                 if not os.path.isdir(repoman_settings["O"]):
2186                                         continue
2187                                 portage.digestgen([], repoman_settings, manifestonly=1,
2188                                         myportdb=portdb)
2189                 elif repolevel==1: # repo-cvsroot
2190                         print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
2191                         for x in myfiles:
2192                                 xs=x.split("/")
2193                                 if len(xs) < 4-repolevel:
2194                                         continue
2195                                 if xs[0]==".":
2196                                         xs=xs[1:]
2197                                 if "/".join(xs[:2]) in mydone:
2198                                         continue
2199                                 mydone.append("/".join(xs[:2]))
2200                                 repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1])
2201                                 if not os.path.isdir(repoman_settings["O"]):
2202                                         continue
2203                                 portage.digestgen([], repoman_settings, manifestonly=1,
2204                                         myportdb=portdb)
2205                 else:
2206                         print(red("I'm confused... I don't know where I am!"))
2207                         sys.exit(1)
2208
2209                 # Force an unsigned commit when more than one Manifest needs to be signed.
2210                 if repolevel < 3 and "sign" in repoman_settings.features:
2211
2212                         fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2213                         mymsg = os.fdopen(fd, "w")
2214                         mymsg.write(commitmessage)
2215                         mymsg.write("\n (Unsigned Manifest commit)")
2216                         mymsg.close()
2217
2218                         commit_cmd = [vcs]
2219                         commit_cmd.extend(vcs_global_opts)
2220                         commit_cmd.append("commit")
2221                         commit_cmd.extend(vcs_local_opts)
2222                         commit_cmd.extend(["-F", commitmessagefile])
2223                         commit_cmd.extend(f.lstrip("./") for f in mymanifests)
2224
2225                         try:
2226                                 if options.pretend:
2227                                         print("(%s)" % (" ".join(commit_cmd),))
2228                                 else:
2229                                         retval = spawn(commit_cmd, env=os.environ)
2230                                         if retval:
2231                                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2232                                                         "error code: %s\n") % (vcs, retval),
2233                                                         level=logging.ERROR, noiselevel=-1)
2234                                                 sys.exit(retval)
2235                         finally:
2236                                 try:
2237                                         os.unlink(commitmessagefile)
2238                                 except OSError:
2239                                         pass
2240                         manifest_commit_required = False
2241
2242         signed = False
2243         if "sign" in repoman_settings.features:
2244                 signed = True
2245                 myfiles = myupdates + myremoved + mymanifests
2246                 try:
2247                         if repolevel==3:   # In a package dir
2248                                 repoman_settings["O"] = "."
2249                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2250                         elif repolevel==2: # In a category dir
2251                                 mydone=[]
2252                                 for x in myfiles:
2253                                         xs=x.split("/")
2254                                         if len(xs) < 4-repolevel:
2255                                                 continue
2256                                         if xs[0]==".":
2257                                                 xs=xs[1:]
2258                                         if xs[0] in mydone:
2259                                                 continue
2260                                         mydone.append(xs[0])
2261                                         repoman_settings["O"] = os.path.join(".", xs[0])
2262                                         if not os.path.isdir(repoman_settings["O"]):
2263                                                 continue
2264                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2265                         elif repolevel==1: # repo-cvsroot
2266                                 print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
2267                                 mydone=[]
2268                                 for x in myfiles:
2269                                         xs=x.split("/")
2270                                         if len(xs) < 4-repolevel:
2271                                                 continue
2272                                         if xs[0]==".":
2273                                                 xs=xs[1:]
2274                                         if "/".join(xs[:2]) in mydone:
2275                                                 continue
2276                                         mydone.append("/".join(xs[:2]))
2277                                         repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
2278                                         if not os.path.isdir(repoman_settings["O"]):
2279                                                 continue
2280                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2281                 except portage.exception.PortageException as e:
2282                         portage.writemsg("!!! %s\n" % str(e))
2283                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2284                         signed = False
2285
2286         if vcs == 'git':
2287                 # It's not safe to use the git commit -a option since there might
2288                 # be some modified files elsewhere in the working tree that the
2289                 # user doesn't want to commit. Therefore, call git update-index
2290                 # in order to ensure that the index is updated with the latest
2291                 # versions of all new and modified files in the relevant portion
2292                 # of the working tree.
2293                 myfiles = mymanifests + myupdates
2294                 myfiles.sort()
2295                 update_index_cmd = ["git", "update-index"]
2296                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2297                 if options.pretend:
2298                         print("(%s)" % (" ".join(update_index_cmd),))
2299                 else:
2300                         retval = spawn(update_index_cmd, env=os.environ)
2301                         if retval != os.EX_OK:
2302                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2303                                         "error code: %s\n") % (vcs, retval),
2304                                         level=logging.ERROR, noiselevel=-1)
2305                                 sys.exit(retval)
2306
2307         if vcs == 'git' or manifest_commit_required or signed:
2308
2309                 myfiles = mymanifests[:]
2310                 if vcs == 'git':
2311                         myfiles += myupdates
2312                         myfiles += myremoved
2313                 myfiles.sort()
2314
2315                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2316                 mymsg = os.fdopen(fd, "w")
2317                 mymsg.write(commitmessage)
2318                 if signed:
2319                         mymsg.write("\n (Signed Manifest commit)")
2320                 else:
2321                         mymsg.write("\n (Unsigned Manifest commit)")
2322                 mymsg.close()
2323
2324                 commit_cmd = []
2325                 if options.pretend and vcs is None:
2326                         # substitute a bogus value for pretend output
2327                         commit_cmd.append("cvs")
2328                 else:
2329                         commit_cmd.append(vcs)
2330                 commit_cmd.extend(vcs_global_opts)
2331                 commit_cmd.append("commit")
2332                 commit_cmd.extend(vcs_local_opts)
2333                 commit_cmd.extend(["-F", commitmessagefile])
2334                 commit_cmd.extend(f.lstrip("./") for f in myfiles)
2335
2336                 try:
2337                         if options.pretend:
2338                                 print("(%s)" % (" ".join(commit_cmd),))
2339                         else:
2340                                 retval = spawn(commit_cmd, env=os.environ)
2341                                 if retval != os.EX_OK:
2342                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2343                                                 "error code: %s\n") % (vcs, retval),
2344                                                 level=logging.ERROR, noiselevel=-1)
2345                                         sys.exit(retval)
2346                 finally:
2347                         try:
2348                                 os.unlink(commitmessagefile)
2349                         except OSError:
2350                                 pass
2351
2352         print()
2353         if vcs:
2354                 print("Commit complete.")
2355         else:
2356                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2357         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2358 sys.exit(0)
2359