bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2012 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27
  28 try:
  29         from urllib.request import urlopen as urllib_request_urlopen
  30 except ImportError:
  31         from urllib import urlopen as urllib_request_urlopen
  32
  33 from itertools import chain
  34 from stat import S_ISDIR
  35
  36 try:
  37         import portage
  38 except ImportError:
  39         from os import path as osp
  40         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  41         import portage
  42 portage._disable_legacy_globals()
  43 portage.dep._internal_warnings = True
  44
  45 try:
  46         import xml.etree.ElementTree
  47         from xml.parsers.expat import ExpatError
  48 except (ImportError, SystemError):
  49         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  50         from portage.output import EOutput
  51         out = EOutput()
  52         for line in msg:
  53                 out.eerror(line)
  54         sys.exit(1)
  55
  56 from portage import os
  57 from portage import subprocess_getstatusoutput
  58 from portage import _encodings
  59 from portage import _unicode_encode
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 from portage import cvstree, normalize_path
  69 from portage import util
  70 from portage.exception import (FileNotFound, MissingParameter,
  71         ParseError, PermissionDenied)
  72 from portage.manifest import _prohibited_filename_chars_re as \
  73         disallowed_filename_chars_re
  74 from portage.process import find_binary, spawn
  75 from portage.output import bold, create_color_func, \
  76         green, nocolor, red
  77 from portage.output import ConsoleStyleFile, StyleWriter
  78 from portage.util import cmp_sort_key, writemsg_level
  79 from portage.util._desktop_entry import validate_desktop_entry
  80 from portage.package.ebuild.digestgen import digestgen
  81 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  82
  83 if sys.hexversion >= 0x3000000:
  84         basestring = str
  85
  86 util.initialize_logger()
  87
  88 # 14 is the length of DESCRIPTION=""
  89 max_desc_len = 100
  90 allowed_filename_chars="a-zA-Z0-9._-+:"
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 bad = create_color_func("BAD")
  93
  94 # A sane umask is needed for files that portage creates.
  95 os.umask(0o22)
  96 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  97 # behave incrementally.
  98 repoman_incrementals = tuple(x for x in \
  99         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 100 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 101 repoman_settings = portage.config(config_root=config_root, local_config=False)
 102
 103 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 104         repoman_settings.get('TERM') == 'dumb' or \
 105         not sys.stdout.isatty():
 106         nocolor()
 107
 108 def warn(txt):
 109         print("repoman: " + txt)
 110
 111 def err(txt):
 112         warn(txt)
 113         sys.exit(1)
 114
 115 def exithandler(signum=None, frame=None):
 116         logging.fatal("Interrupted; exiting...")
 117         if signum is None:
 118                 sys.exit(1)
 119         else:
 120                 sys.exit(128 + signum)
 121
 122 signal.signal(signal.SIGINT,exithandler)
 123
 124 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 125         """Repoman needs it's own HelpFormatter for now, because the default ones
 126         murder the help text."""
 127
 128         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 129                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 130
 131         def format_description(self, description):
 132                 return description
 133
 134 class RepomanOptionParser(optparse.OptionParser):
 135         """Add the on_tail function, ruby has it, optionParser should too
 136         """
 137
 138         def __init__(self, *args, **kwargs):
 139                 optparse.OptionParser.__init__(self, *args, **kwargs)
 140                 self.tail = ""
 141
 142         def on_tail(self, description):
 143                 self.tail += description
 144
 145         def format_help(self, formatter=None):
 146                 result = optparse.OptionParser.format_help(self, formatter)
 147                 result += self.tail
 148                 return result
 149
 150
 151 def ParseArgs(argv, qahelp):
 152         """This function uses a customized optionParser to parse command line arguments for repoman
 153         Args:
 154           argv - a sequence of command line arguments
 155                 qahelp - a dict of qa warning to help message
 156         Returns:
 157           (opts, args), just like a call to parser.parse_args()
 158         """
 159
 160         if argv and isinstance(argv[0], bytes):
 161                 argv = [portage._unicode_decode(x) for x in argv]
 162
 163         modes = {
 164                 'commit' : 'Run a scan then commit changes',
 165                 'ci' : 'Run a scan then commit changes',
 166                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 167                 'full' : 'Scan directory tree and print all issues (not a summary)',
 168                 'help' : 'Show this screen',
 169                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 170                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 171                 'scan' : 'Scan directory tree for QA issues'
 172         }
 173
 174         mode_keys = list(modes)
 175         mode_keys.sort()
 176
 177         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 178         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 179         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 180         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 181         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 182
 183         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 184                 help='Request a confirmation before commiting')
 185
 186         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 187                 help='specify a commit message on the command line')
 188
 189         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 190                 help='specify a path to a file that contains a commit message')
 191
 192         parser.add_option('--digest',
 193                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 194                 help='Automatically update Manifest digests for modified files')
 195
 196         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 197                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 198
 199         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 200                 help='do not print unnecessary messages')
 201
 202         parser.add_option(
 203                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 204                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 205                 'regardless of modification if \'force\' is specified)')
 206
 207         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 208                 help='Commit with QA violations')
 209
 210         parser.add_option('--vcs', dest='vcs',
 211                 help='Force using specific VCS instead of autodetection')
 212
 213         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 214                 help='be very verbose in output', default=0)
 215
 216         parser.add_option('-V', '--version', dest='version', action='store_true',
 217                 help='show version info')
 218
 219         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 220                 default=False, help='forces the metadata.xml parse check to be carried out')
 221
 222         parser.add_option(
 223                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 224                 metavar="<y|n>",
 225                 help='only check packages that have uncommitted modifications')
 226
 227         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 228                 default=False, help='ignore arch-specific failures (where arch != host)')
 229
 230         parser.add_option("--ignore-default-opts",
 231                 action="store_true",
 232                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 233
 234         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 235                 default=False, help='ignore masked packages (not allowed with commit mode)')
 236
 237         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 238                 default=False, help='include dev profiles in dependency checks')
 239
 240         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 241                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 242
 243         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 244                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 245
 246         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 247                 help='specify which mode repoman will run in (default=full)')
 248
 249         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 250
 251         for k in mode_keys:
 252                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 253
 254         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 255
 256         sorted_qa = list(qahelp)
 257         sorted_qa.sort()
 258         for k in sorted_qa:
 259                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 260
 261         opts, args = parser.parse_args(argv[1:])
 262
 263         if not opts.ignore_default_opts:
 264                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 265                 if default_opts:
 266                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 267
 268         if opts.mode == 'help':
 269                 parser.print_help(short=False)
 270
 271         for arg in args:
 272                 if arg in modes:
 273                         if not opts.mode:
 274                                 opts.mode = arg
 275                                 break
 276                 else:
 277                         parser.error("invalid mode: %s" % arg)
 278
 279         if not opts.mode:
 280                 opts.mode = 'full'
 281
 282         if opts.mode == 'ci':
 283                 opts.mode = 'commit'  # backwards compat shortcut
 284
 285         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 286                 if opts.ignore_masked:
 287                         parser.error('Commit mode and --ignore-masked are not compatible')
 288                 if opts.without_mask:
 289                         parser.error('Commit mode and --without-mask are not compatible')
 290
 291         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 292         for val in range(opts.verbosity):
 293                 logger = logging.getLogger()
 294                 logger.setLevel(logger.getEffectiveLevel() - 10)
 295
 296         for val in range(opts.quiet):
 297                 logger = logging.getLogger()
 298                 logger.setLevel(logger.getEffectiveLevel() + 10)
 299
 300         return (opts, args)
 301
 302 qahelp={
 303         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 304         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 305         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 306         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 307         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 308         "changelog.missing":"Missing ChangeLog files",
 309         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 310         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 311         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 312         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 313         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 314         "file.size":"Files in the files directory must be under 20 KiB",
 315         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 316         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 317         "file.UTF8":"File is not UTF8 compliant",
 318         "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf",
 319         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 320         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 321         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 322         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 323         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 324         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 325         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 326         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 327         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 328         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 329         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 330         "EAPI.definition":"EAPI definition does not conform to PMS section 8.3.1 (first non-comment, non-blank line)",
 331         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 332         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 333         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 334         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 335         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 336         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 337         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 338         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 339         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 340         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 341         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 342         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 343         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 344         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 345         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 346         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 347         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 348         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 349         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 350         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 351         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 352         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 353         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 354         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 355         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 356         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 357         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 358         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 359         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 360         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 361         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 362         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 363         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 364         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 365         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 366         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 367         "variable.readonly":"Assigning a readonly variable",
 368         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 369         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 370         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 371         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 372         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 373         "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
 374         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 375         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 376         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 377         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 378         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 379         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 380         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 381         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 382         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 383         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 384         "ebuild.badheader":"This ebuild has a malformed header",
 385         "eprefixify.defined":"The ebuild uses eprefixify, but does not inherit the prefix eclass",
 386         "manifest.bad":"Manifest has missing or incorrect digests",
 387         "metadata.missing":"Missing metadata.xml files",
 388         "metadata.bad":"Bad metadata.xml files",
 389         "metadata.warning":"Warnings in metadata.xml files",
 390         "portage.internal":"The ebuild uses an internal Portage function",
 391         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 392         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 393         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 394 }
 395
 396 qacats = list(qahelp)
 397 qacats.sort()
 398
 399 qawarnings = set((
 400 "changelog.missing",
 401 "changelog.notadded",
 402 "dependency.unknown",
 403 "digest.assumed",
 404 "digest.unused",
 405 "ebuild.notadded",
 406 "ebuild.nesteddie",
 407 "desktop.invalid",
 408 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 409 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 410 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 411 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 412 "DESCRIPTION.toolong",
 413 "EAPI.deprecated",
 414 "HOMEPAGE.virtual",
 415 "LICENSE.virtual",
 416 "KEYWORDS.dropped",
 417 "KEYWORDS.stupid",
 418 "KEYWORDS.missing",
 419 "IUSE.undefined",
 420 "PDEPEND.suspect",
 421 "RDEPEND.implicit",
 422 "RDEPEND.suspect",
 423 "RESTRICT.invalid",
 424 "ebuild.minorsyn",
 425 "ebuild.badheader",
 426 "ebuild.patches",
 427 "file.size",
 428 "inherit.autotools",
 429 "inherit.deprecated",
 430 "java.eclassesnotused",
 431 "wxwidgets.eclassnotused",
 432 "metadata.warning",
 433 "portage.internal",
 434 "usage.obsolete",
 435 "upstream.workaround",
 436 "LIVEVCS.stable",
 437 "LIVEVCS.unmasked",
 438 ))
 439
 440 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 441
 442 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 443 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 444 allvars.update(Package.metadata_keys)
 445 allvars = sorted(allvars)
 446 commitmessage=None
 447 for x in missingvars:
 448         x += ".missing"
 449         if x not in qacats:
 450                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 451                 qacats.append(x)
 452                 qawarnings.add(x)
 453
 454 valid_restrict = frozenset(["binchecks", "bindist",
 455         "fetch", "installsources", "mirror",
 456         "primaryuri", "strip", "test", "userpriv"])
 457
 458 live_eclasses = frozenset([
 459         "bzr",
 460         "cvs",
 461         "darcs",
 462         "git",
 463         "git-2",
 464         "mercurial",
 465         "subversion",
 466         "tla",
 467 ])
 468
 469 suspect_rdepend = frozenset([
 470         "app-arch/cabextract",
 471         "app-arch/rpm2targz",
 472         "app-doc/doxygen",
 473         "dev-lang/nasm",
 474         "dev-lang/swig",
 475         "dev-lang/yasm",
 476         "dev-perl/extutils-pkgconfig",
 477         "dev-util/byacc",
 478         "dev-util/cmake",
 479         "dev-util/ftjam",
 480         "dev-util/gperf",
 481         "dev-util/gtk-doc",
 482         "dev-util/gtk-doc-am",
 483         "dev-util/intltool",
 484         "dev-util/jam",
 485         "dev-util/pkgconfig",
 486         "dev-util/scons",
 487         "dev-util/unifdef",
 488         "dev-util/yacc",
 489         "media-gfx/ebdftopcf",
 490         "sys-apps/help2man",
 491         "sys-devel/autoconf",
 492         "sys-devel/automake",
 493         "sys-devel/bin86",
 494         "sys-devel/bison",
 495         "sys-devel/dev86",
 496         "sys-devel/flex",
 497         "sys-devel/m4",
 498         "sys-devel/pmake",
 499         "virtual/linux-sources",
 500         "x11-misc/bdftopcf",
 501         "x11-misc/imake",
 502 ])
 503
 504 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 505 # force refetch if the local copy creation time is older than this
 506 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 507
 508 # file.executable
 509 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 510
 511 options, arguments = ParseArgs(sys.argv, qahelp)
 512
 513 if options.version:
 514         print("Portage", portage.VERSION)
 515         sys.exit(0)
 516
 517 # Set this to False when an extraordinary issue (generally
 518 # something other than a QA issue) makes it impossible to
 519 # commit (like if Manifest generation fails).
 520 can_force = True
 521
 522 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 523 if portdir is None:
 524         sys.exit(1)
 525
 526 myreporoot = os.path.basename(portdir_overlay)
 527 myreporoot += mydir[len(portdir_overlay):]
 528
 529 if options.vcs:
 530         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 531                 vcs = options.vcs
 532         else:
 533                 vcs = None
 534 else:
 535         vcses = utilities.FindVCS()
 536         if len(vcses) > 1:
 537                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 538                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 539                 sys.exit(1)
 540         elif vcses:
 541                 vcs = vcses[0]
 542         else:
 543                 vcs = None
 544
 545 if options.if_modified == "y" and vcs is None:
 546         logging.info("Not in a version controlled repository; "
 547                 "disabling --if-modified.")
 548         options.if_modified = "n"
 549
 550 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 551 vcs_preserves_mtime = vcs in ('cvs',)
 552
 553 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 554 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 555 if vcs_global_opts is None:
 556         if vcs in ('cvs', 'svn'):
 557                 vcs_global_opts = "-q"
 558         else:
 559                 vcs_global_opts = ""
 560 vcs_global_opts = vcs_global_opts.split()
 561
 562 if options.mode == 'commit' and not options.pretend and not vcs:
 563         logging.info("Not in a version controlled repository; enabling pretend mode.")
 564         options.pretend = True
 565
 566 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 567 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 568         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 569         portage._shell_quote(portdir_overlay))
 570 # We have to call the config constructor again so
 571 # that config.repositories is initialized correctly.
 572 repoman_settings = portage.config(config_root=config_root, local_config=False,
 573         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 574
 575 root = repoman_settings['EROOT']
 576 trees = {
 577         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 578 }
 579 portdb = trees[root]['porttree'].dbapi
 580
 581 # Constrain dependency resolution to the master(s)
 582 # that are specified in layout.conf.
 583 repodir = os.path.realpath(portdir_overlay)
 584 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 585 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 586 portdir = portdb.porttrees[0]
 587
 588 if repo_config.allow_provide_virtual:
 589         qawarnings.add("virtual.oldstyle")
 590
 591 if repo_config.sign_commit:
 592         if vcs == 'git':
 593                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 594                 # the commit arguments. If key_id is unspecified, then it must be
 595                 # configured by `git config user.signingkey key_id`.
 596                 vcs_local_opts.append("--gpg-sign")
 597
 598 # In order to disable manifest signatures, repos may set
 599 # "sign-manifests = false" in metadata/layout.conf. This
 600 # can be used to prevent merge conflicts like those that
 601 # thin-manifests is designed to prevent.
 602 sign_manifests = "sign" in repoman_settings.features and \
 603         repo_config.sign_manifest
 604
 605 manifest_hashes = repo_config.manifest_hashes
 606 if manifest_hashes is None:
 607         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 608
 609 if options.mode in ("commit", "fix", "manifest"):
 610         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 611                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 612                 "metadata/layout.conf does not contain the '%s' hash which "
 613                 "is required by this portage version. You will have to "
 614                 "upgrade portage if you want to generate valid manifests for "
 615                 "this repository.") % \
 616                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 617                 for line in textwrap.wrap(msg, 70):
 618                         logging.error(line)
 619                 sys.exit(1)
 620
 621         unsupported_hashes = manifest_hashes.difference(
 622                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 623         if unsupported_hashes:
 624                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 625                 "metadata/layout.conf contains one or more hash types '%s' "
 626                 "which are not supported by this portage version. You will "
 627                 "have to upgrade portage if you want to generate valid "
 628                 "manifests for this repository.") % \
 629                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 630                 for line in textwrap.wrap(msg, 70):
 631                         logging.error(line)
 632                 sys.exit(1)
 633
 634 if "commit" == options.mode and \
 635         repo_config.name == "gentoo" and \
 636         "RMD160" in manifest_hashes and \
 637         "RMD160" not in portage.checksum.hashorigin_map:
 638         msg = "Please install " \
 639         "pycrypto or enable python's ssl USE flag in order " \
 640         "to enable RMD160 hash support. See bug #198398 for " \
 641         "more information."
 642         prefix = bad(" * ")
 643         for line in textwrap.wrap(msg, 70):
 644                 print(prefix + line)
 645         sys.exit(1)
 646
 647 if options.echangelog is None and repo_config.update_changelog:
 648         options.echangelog = 'y'
 649
 650 if vcs is None:
 651         options.echangelog = 'n'
 652
 653 # The --echangelog option causes automatic ChangeLog generation,
 654 # which invalidates changelog.ebuildadded and changelog.missing
 655 # checks.
 656 # Note: Some don't use ChangeLogs in distributed SCMs.
 657 # It will be generated on server side from scm log,
 658 # before package moves to the rsync server.
 659 # This is needed because they try to avoid merge collisions.
 660 # Gentoo's Council decided to always use the ChangeLog file.
 661 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 662 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 663
 664 if 'digest' in repoman_settings.features and options.digest != 'n':
 665         options.digest = 'y'
 666
 667 logging.debug("vcs: %s" % (vcs,))
 668 logging.debug("repo config: %s" % (repo_config,))
 669 logging.debug("options: %s" % (options,))
 670
 671 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 672 # profile-specific config constructor calls.
 673 env = os.environ.copy()
 674 env['PORTDIR'] = portdir
 675 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 676
 677 logging.info('Setting paths:')
 678 logging.info('PORTDIR = "' + portdir + '"')
 679 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 680
 681 # It's confusing if these warnings are displayed without the user
 682 # being told which profile they come from, so disable them.
 683 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 684
 685 categories = []
 686 for path in repo_config.eclass_db.porttrees:
 687         categories.extend(portage.util.grabfile(
 688                 os.path.join(path, 'profiles', 'categories')))
 689 repoman_settings.categories = frozenset(
 690         portage.util.stack_lists([categories], incremental=1))
 691 categories = repoman_settings.categories
 692
 693 portdb.settings = repoman_settings
 694 root_config = RootConfig(repoman_settings, trees[root], None)
 695 # We really only need to cache the metadata that's necessary for visibility
 696 # filtering. Anything else can be discarded to reduce memory consumption.
 697 portdb._aux_cache_keys.clear()
 698 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 699
 700 reposplit = myreporoot.split(os.path.sep)
 701 repolevel = len(reposplit)
 702
 703 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 704 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 705 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 706 if options.mode == 'commit' and repolevel not in [1,2,3]:
 707         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 708         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 709         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 710         print(red("***"))
 711         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 712
 713 # Make startdir relative to the canonical repodir, so that we can pass
 714 # it to digestgen and it won't have to be canonicalized again.
 715 if repolevel == 1:
 716         startdir = repodir
 717 else:
 718         startdir = normalize_path(mydir)
 719         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 720
 721 def caterror(mycat):
 722         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 723
 724 class ProfileDesc(object):
 725         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 726         def __init__(self, arch, status, sub_path, tree_path):
 727                 self.arch = arch
 728                 self.status = status
 729                 if sub_path:
 730                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 731                 self.sub_path = sub_path
 732                 self.tree_path = tree_path
 733                 if tree_path:
 734                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 735                 else:
 736                         self.abs_path = tree_path
 737
 738         def __str__(self):
 739                 if self.sub_path:
 740                         return self.sub_path
 741                 return 'empty profile'
 742
 743 profile_list = []
 744 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 745
 746 # get lists of valid keywords, licenses, and use
 747 kwlist = set()
 748 liclist = set()
 749 uselist = set()
 750 global_pmasklines = []
 751
 752 for path in portdb.porttrees:
 753         try:
 754                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 755         except OSError:
 756                 pass
 757         kwlist.update(portage.grabfile(os.path.join(path,
 758                 "profiles", "arch.list")))
 759
 760         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 761         for x in use_desc:
 762                 x = x.split()
 763                 if x:
 764                         uselist.add(x[0])
 765
 766         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 767         try:
 768                 expand_list = os.listdir(expand_desc_dir)
 769         except OSError:
 770                 pass
 771         else:
 772                 for fn in expand_list:
 773                         if not fn[-5:] == '.desc':
 774                                 continue
 775                         use_prefix = fn[:-5].lower() + '_'
 776                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 777                                 x = x.split()
 778                                 if x:
 779                                         uselist.add(use_prefix + x[0])
 780
 781         global_pmasklines.append(portage.util.grabfile_package(
 782                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 783
 784         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 785         try:
 786                 desc_file = io.open(_unicode_encode(desc_path,
 787                         encoding=_encodings['fs'], errors='strict'),
 788                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 789         except EnvironmentError:
 790                 pass
 791         else:
 792                 for i, x in enumerate(desc_file):
 793                         if x[0] == "#":
 794                                 continue
 795                         arch = x.split()
 796                         if len(arch) == 0:
 797                                 continue
 798                         if len(arch) != 3:
 799                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 800                                         desc_path + " line %d" % (i+1, ))
 801                         elif arch[0] not in kwlist:
 802                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 803                                         desc_path + " line %d" % (i+1, ))
 804                         elif arch[2] not in valid_profile_types:
 805                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 806                                         desc_path + " line %d" % (i+1, ))
 807                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 808                         if not os.path.isdir(profile_desc.abs_path):
 809                                 logging.error(
 810                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 811                                         arch[2], arch[1], arch[0], desc_path, i+1)
 812                                 continue
 813                         if os.path.exists(
 814                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 815                                 continue
 816                         profile_list.append(profile_desc)
 817                 desc_file.close()
 818
 819 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 820 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 821
 822 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 823 global_pmaskdict = {}
 824 for x in global_pmasklines:
 825         global_pmaskdict.setdefault(x.cp, []).append(x)
 826 del global_pmasklines
 827
 828 def has_global_mask(pkg):
 829         mask_atoms = global_pmaskdict.get(pkg.cp)
 830         if mask_atoms:
 831                 pkg_list = [pkg]
 832                 for x in mask_atoms:
 833                         if portage.dep.match_from_list(x, pkg_list):
 834                                 return x
 835         return None
 836
 837 # Ensure that profile sub_path attributes are unique. Process in reverse order
 838 # so that profiles with duplicate sub_path from overlays will override
 839 # profiles with the same sub_path from parent repos.
 840 profiles = {}
 841 profile_list.reverse()
 842 profile_sub_paths = set()
 843 for prof in profile_list:
 844         if prof.sub_path in profile_sub_paths:
 845                 continue
 846         profile_sub_paths.add(prof.sub_path)
 847         profiles.setdefault(prof.arch, []).append(prof)
 848
 849 # Use an empty profile for checking dependencies of
 850 # packages that have empty KEYWORDS.
 851 prof = ProfileDesc('**', 'stable', '', '')
 852 profiles.setdefault(prof.arch, []).append(prof)
 853
 854 for x in repoman_settings.archlist():
 855         if x[0] == "~":
 856                 continue
 857         if x not in profiles:
 858                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 859                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 860                 print(red("up with the "+x+" team."))
 861                 print()
 862
 863 if not liclist:
 864         logging.fatal("Couldn't find licenses?")
 865         sys.exit(1)
 866
 867 if not kwlist:
 868         logging.fatal("Couldn't read KEYWORDS from arch.list")
 869         sys.exit(1)
 870
 871 if not uselist:
 872         logging.fatal("Couldn't find use.desc?")
 873         sys.exit(1)
 874
 875 scanlist=[]
 876 if repolevel==2:
 877         #we are inside a category directory
 878         catdir=reposplit[-1]
 879         if catdir not in categories:
 880                 caterror(catdir)
 881         mydirlist=os.listdir(startdir)
 882         for x in mydirlist:
 883                 if x == "CVS" or x.startswith("."):
 884                         continue
 885                 if os.path.isdir(startdir+"/"+x):
 886                         scanlist.append(catdir+"/"+x)
 887         repo_subdir = catdir + os.sep
 888 elif repolevel==1:
 889         for x in categories:
 890                 if not os.path.isdir(startdir+"/"+x):
 891                         continue
 892                 for y in os.listdir(startdir+"/"+x):
 893                         if y == "CVS" or y.startswith("."):
 894                                 continue
 895                         if os.path.isdir(startdir+"/"+x+"/"+y):
 896                                 scanlist.append(x+"/"+y)
 897         repo_subdir = ""
 898 elif repolevel==3:
 899         catdir = reposplit[-2]
 900         if catdir not in categories:
 901                 caterror(catdir)
 902         scanlist.append(catdir+"/"+reposplit[-1])
 903         repo_subdir = scanlist[-1] + os.sep
 904 else:
 905         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 906                 ' from the current working directory'
 907         logging.critical(msg)
 908         sys.exit(1)
 909
 910 repo_subdir_len = len(repo_subdir)
 911 scanlist.sort()
 912
 913 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 914
 915 def vcs_files_to_cps(vcs_file_iter):
 916         """
 917         Iterate over the given modified file paths returned from the vcs,
 918         and return a frozenset containing category/pn strings for each
 919         modified package.
 920         """
 921
 922         modified_cps = []
 923
 924         if repolevel == 3:
 925                 if reposplit[-2] in categories and \
 926                         next(vcs_file_iter, None) is not None:
 927                         modified_cps.append("/".join(reposplit[-2:]))
 928
 929         elif repolevel == 2:
 930                 category = reposplit[-1]
 931                 if category in categories:
 932                         for filename in vcs_file_iter:
 933                                 f_split = filename.split(os.sep)
 934                                 # ['.', pn,...]
 935                                 if len(f_split) > 2:
 936                                         modified_cps.append(category + "/" + f_split[1])
 937
 938         else:
 939                 # repolevel == 1
 940                 for filename in vcs_file_iter:
 941                         f_split = filename.split(os.sep)
 942                         # ['.', category, pn,...]
 943                         if len(f_split) > 3 and f_split[1] in categories:
 944                                 modified_cps.append("/".join(f_split[1:3]))
 945
 946         return frozenset(modified_cps)
 947
 948 def git_supports_gpg_sign():
 949         status, cmd_output = \
 950                 subprocess_getstatusoutput("git --version")
 951         cmd_output = cmd_output.split()
 952         if cmd_output:
 953                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
 954                 if version is not None:
 955                         version = [int(x) for x in version.groups()[1:]]
 956                         if version[0] > 1 or \
 957                                 (version[0] == 1 and version[1] > 7) or \
 958                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
 959                                 return True
 960         return False
 961
 962 def dev_keywords(profiles):
 963         """
 964         Create a set of KEYWORDS values that exist in 'dev'
 965         profiles. These are used
 966         to trigger a message notifying the user when they might
 967         want to add the --include-dev option.
 968         """
 969         type_arch_map = {}
 970         for arch, arch_profiles in profiles.items():
 971                 for prof in arch_profiles:
 972                         arch_set = type_arch_map.get(prof.status)
 973                         if arch_set is None:
 974                                 arch_set = set()
 975                                 type_arch_map[prof.status] = arch_set
 976                         arch_set.add(arch)
 977
 978         dev_keywords = type_arch_map.get('dev', set())
 979         dev_keywords.update(['~' + arch for arch in dev_keywords])
 980         return frozenset(dev_keywords)
 981
 982 dev_keywords = dev_keywords(profiles)
 983
 984 stats={}
 985 fails={}
 986
 987 # provided by the desktop-file-utils package
 988 desktop_file_validate = find_binary("desktop-file-validate")
 989 desktop_pattern = re.compile(r'.*\.desktop$')
 990
 991 for x in qacats:
 992         stats[x]=0
 993         fails[x]=[]
 994
 995 xmllint_capable = False
 996 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
 997
 998 def parsedate(s):
 999         """Parse a RFC 822 date and time string.
1000         This is required for python3 compatibility, since the
1001         rfc822.parsedate() function is not available."""
1002
1003         s_split = []
1004         for x in s.upper().split():
1005                 for y in x.split(','):
1006                         if y:
1007                                 s_split.append(y)
1008
1009         if len(s_split) != 6:
1010                 return None
1011
1012         # %a, %d %b %Y %H:%M:%S %Z
1013         a, d, b, Y, H_M_S, Z = s_split
1014
1015         # Convert month to integer, since strptime %w is locale-dependent.
1016         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1017                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1018         m = month_map.get(b)
1019         if m is None:
1020                 return None
1021         m = str(m).rjust(2, '0')
1022
1023         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1024
1025 def fetch_metadata_dtd():
1026         """
1027         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1028         metadata_dtd_ctime_interval.
1029         @rtype: bool
1030         @return: True if successful, otherwise False
1031         """
1032
1033         must_fetch = True
1034         metadata_dtd_st = None
1035         current_time = int(time.time())
1036         try:
1037                 metadata_dtd_st = os.stat(metadata_dtd)
1038         except EnvironmentError as e:
1039                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1040                         raise
1041                 del e
1042         else:
1043                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1044                 if abs(current_time - metadata_dtd_st.st_ctime) \
1045                         < metadata_dtd_ctime_interval:
1046                         must_fetch = False
1047
1048         if must_fetch:
1049                 print()
1050                 print(green("***") + " the local copy of metadata.dtd " + \
1051                         "needs to be refetched, doing that now")
1052                 print()
1053                 try:
1054                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1055                         msg_info = url_f.info()
1056                         last_modified = msg_info.get('last-modified')
1057                         if last_modified is not None:
1058                                 last_modified = parsedate(last_modified)
1059                                 if last_modified is not None:
1060                                         last_modified = calendar.timegm(last_modified)
1061
1062                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1063                         try:
1064                                 local_f = open(metadata_dtd_tmp, mode='wb')
1065                                 local_f.write(url_f.read())
1066                                 local_f.close()
1067                                 if last_modified is not None:
1068                                         try:
1069                                                 os.utime(metadata_dtd_tmp,
1070                                                         (int(last_modified), int(last_modified)))
1071                                         except OSError:
1072                                                 # This fails on some odd non-unix-like filesystems.
1073                                                 # We don't really need the mtime to be preserved
1074                                                 # anyway here (currently we use ctime to trigger
1075                                                 # fetch), so just ignore it.
1076                                                 pass
1077                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1078                         finally:
1079                                 try:
1080                                         os.unlink(metadata_dtd_tmp)
1081                                 except OSError:
1082                                         pass
1083
1084                         url_f.close()
1085
1086                 except EnvironmentError as e:
1087                         print()
1088                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1089                         print(red("!!!")+" exception '%s' though." % (e,))
1090                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1091                         return False
1092
1093         return True
1094
1095 if options.mode == "manifest":
1096         pass
1097 elif not find_binary('xmllint'):
1098         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1099         if options.xml_parse or repolevel==3:
1100                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1101                 sys.exit(1)
1102 else:
1103         if not fetch_metadata_dtd():
1104                 sys.exit(1)
1105         #this can be problematic if xmllint changes their output
1106         xmllint_capable=True
1107
1108 if options.mode == 'commit' and vcs:
1109         utilities.detect_vcs_conflicts(options, vcs)
1110
1111 if options.mode == "manifest":
1112         pass
1113 elif options.pretend:
1114         print(green("\nRepoMan does a once-over of the neighborhood..."))
1115 else:
1116         print(green("\nRepoMan scours the neighborhood..."))
1117
1118 new_ebuilds = set()
1119 modified_ebuilds = set()
1120 modified_changelogs = set()
1121 mychanged = []
1122 mynew = []
1123 myremoved = []
1124
1125 if vcs == "cvs":
1126         mycvstree = cvstree.getentries("./", recursive=1)
1127         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1128         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1129         if options.if_modified == "y":
1130                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1131
1132 if vcs == "svn":
1133         with os.popen("svn status") as f:
1134                 svnstatus = f.readlines()
1135         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1136         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1137         if options.if_modified == "y":
1138                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1139
1140 elif vcs == "git":
1141         with os.popen("git diff-index --name-only "
1142                 "--relative --diff-filter=M HEAD") as f:
1143                 mychanged = f.readlines()
1144         mychanged = ["./" + elem[:-1] for elem in mychanged]
1145
1146         with os.popen("git diff-index --name-only "
1147                 "--relative --diff-filter=A HEAD") as f:
1148                 mynew = f.readlines()
1149         mynew = ["./" + elem[:-1] for elem in mynew]
1150         if options.if_modified == "y":
1151                 with os.popen("git diff-index --name-only "
1152                         "--relative --diff-filter=D HEAD") as f:
1153                         myremoved = f.readlines()
1154                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1155
1156 elif vcs == "bzr":
1157         with os.popen("bzr status -S .") as f:
1158                 bzrstatus = f.readlines()
1159         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1160         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1161         if options.if_modified == "y":
1162                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1163
1164 elif vcs == "hg":
1165         with os.popen("hg status --no-status --modified .") as f:
1166                 mychanged = f.readlines()
1167         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1168         mynew = os.popen("hg status --no-status --added .").readlines()
1169         mynew = ["./" + elem.rstrip() for elem in mynew]
1170         if options.if_modified == "y":
1171                 with os.popen("hg status --no-status --removed .") as f:
1172                         myremoved = f.readlines()
1173                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1174
1175 if vcs:
1176         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1177         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1178         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1179                 if os.path.basename(x) == "ChangeLog")
1180
1181 def vcs_new_changed(relative_path):
1182         for x in chain(mychanged, mynew):
1183                 if x == relative_path:
1184                         return True
1185         return False
1186
1187 have_pmasked = False
1188 have_dev_keywords = False
1189 dofail = 0
1190
1191 # NOTE: match-all caches are not shared due to potential
1192 # differences between profiles in _get_implicit_iuse.
1193 arch_caches={}
1194 arch_xmatch_caches = {}
1195 shared_xmatch_caches = {"cp-list":{}}
1196
1197 # Disable the "ebuild.notadded" check when not in commit mode and
1198 # running `svn status` in every package dir will be too expensive.
1199
1200 check_ebuild_notadded = not \
1201         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1202
1203 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1204 thirdpartymirrors = []
1205 for v in repoman_settings.thirdpartymirrors().values():
1206         for v in v:
1207                 if not v.endswith("/"):
1208                         v += "/"
1209                 thirdpartymirrors.append(v)
1210
1211 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1212         """
1213         Implements doctype() as required to avoid deprecation warnings with
1214         >=python-2.7.
1215         """
1216         def doctype(self, name, pubid, system):
1217                 pass
1218
1219 try:
1220         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1221 except (EnvironmentError, ParseError, PermissionDenied) as e:
1222         err(str(e))
1223 except FileNotFound:
1224         # TODO: Download as we do for metadata.dtd, but add a way to
1225         # disable for non-gentoo repoman users who may not have herds.
1226         herd_base = None
1227
1228 effective_scanlist = scanlist
1229 if options.if_modified == "y":
1230         effective_scanlist = sorted(vcs_files_to_cps(
1231                 chain(mychanged, mynew, myremoved)))
1232
1233 for x in effective_scanlist:
1234         #ebuilds and digests added to cvs respectively.
1235         logging.info("checking package %s" % x)
1236         # save memory by discarding xmatch caches from previous package(s)
1237         arch_xmatch_caches.clear()
1238         eadded=[]
1239         catdir,pkgdir=x.split("/")
1240         checkdir=repodir+"/"+x
1241         checkdir_relative = ""
1242         if repolevel < 3:
1243                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1244         if repolevel < 2:
1245                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1246         checkdir_relative = os.path.join(".", checkdir_relative)
1247         generated_manifest = False
1248
1249         if options.mode == "manifest" or \
1250           (options.mode != 'manifest-check' and options.digest == 'y') or \
1251           options.mode in ('commit', 'fix') and not options.pretend:
1252                 auto_assumed = set()
1253                 fetchlist_dict = portage.FetchlistDict(checkdir,
1254                         repoman_settings, portdb)
1255                 if options.mode == 'manifest' and options.force:
1256                         portage._doebuild_manifest_exempt_depend += 1
1257                         try:
1258                                 distdir = repoman_settings['DISTDIR']
1259                                 mf = repoman_settings.repositories.get_repo_for_location(
1260                                         os.path.dirname(os.path.dirname(checkdir)))
1261                                 mf = mf.load_manifest(checkdir, distdir,
1262                                         fetchlist_dict=fetchlist_dict)
1263                                 mf.create(requiredDistfiles=None,
1264                                         assumeDistHashesAlways=True)
1265                                 for distfiles in fetchlist_dict.values():
1266                                         for distfile in distfiles:
1267                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1268                                                         mf.fhashdict['DIST'].pop(distfile, None)
1269                                                 else:
1270                                                         auto_assumed.add(distfile)
1271                                 mf.write()
1272                         finally:
1273                                 portage._doebuild_manifest_exempt_depend -= 1
1274
1275                 repoman_settings["O"] = checkdir
1276                 try:
1277                         generated_manifest = digestgen(
1278                                 mysettings=repoman_settings, myportdb=portdb)
1279                 except portage.exception.PermissionDenied as e:
1280                         generated_manifest = False
1281                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1282                                 level=logging.ERROR, noiselevel=-1)
1283
1284                 if not generated_manifest:
1285                         print("Unable to generate manifest.")
1286                         dofail = 1
1287
1288                 if options.mode == "manifest":
1289                         if not dofail and options.force and auto_assumed and \
1290                                 'assume-digests' in repoman_settings.features:
1291                                 # Show which digests were assumed despite the --force option
1292                                 # being given. This output will already have been shown by
1293                                 # digestgen() if assume-digests is not enabled, so only show
1294                                 # it here if assume-digests is enabled.
1295                                 pkgs = list(fetchlist_dict)
1296                                 pkgs.sort()
1297                                 portage.writemsg_stdout("  digest.assumed" + \
1298                                         portage.output.colorize("WARN",
1299                                         str(len(auto_assumed)).rjust(18)) + "\n")
1300                                 for cpv in pkgs:
1301                                         fetchmap = fetchlist_dict[cpv]
1302                                         pf = portage.catsplit(cpv)[1]
1303                                         for distfile in sorted(fetchmap):
1304                                                 if distfile in auto_assumed:
1305                                                         portage.writemsg_stdout(
1306                                                                 "   %s::%s\n" % (pf, distfile))
1307                         continue
1308                 elif dofail:
1309                         sys.exit(1)
1310
1311         if not generated_manifest:
1312                 repoman_settings['O'] = checkdir
1313                 repoman_settings['PORTAGE_QUIET'] = '1'
1314                 if not portage.digestcheck([], repoman_settings, strict=1):
1315                         stats["manifest.bad"] += 1
1316                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1317                 repoman_settings.pop('PORTAGE_QUIET', None)
1318
1319         if options.mode == 'manifest-check':
1320                 continue
1321
1322         checkdirlist=os.listdir(checkdir)
1323         ebuildlist=[]
1324         pkgs = {}
1325         allvalid = True
1326         for y in checkdirlist:
1327                 if (y in no_exec or y.endswith(".ebuild")) and \
1328                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1329                                 stats["file.executable"] += 1
1330                                 fails["file.executable"].append(os.path.join(checkdir, y))
1331                 if y.endswith(".ebuild"):
1332                         pf = y[:-7]
1333                         ebuildlist.append(pf)
1334                         cpv = "%s/%s" % (catdir, pf)
1335                         try:
1336                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1337                         except KeyError:
1338                                 allvalid = False
1339                                 stats["ebuild.syntax"] += 1
1340                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1341                                 continue
1342                         except IOError:
1343                                 allvalid = False
1344                                 stats["ebuild.output"] += 1
1345                                 fails["ebuild.output"].append(os.path.join(x, y))
1346                                 continue
1347                         if not portage.eapi_is_supported(myaux["EAPI"]):
1348                                 allvalid = False
1349                                 stats["EAPI.unsupported"] += 1
1350                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1351                                 continue
1352                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1353                                 root_config=root_config, type_name="ebuild")
1354
1355         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1356         pkgsplits = {}
1357         for i in range(len(ebuildlist)):
1358                 ebuild_split = portage.pkgsplit(ebuildlist[i])
1359                 pkgsplits[ebuild_split] = ebuildlist[i]
1360                 ebuildlist[i] = ebuild_split
1361         ebuildlist.sort(key=cmp_sort_key(portage.pkgcmp))
1362         for i in range(len(ebuildlist)):
1363                 ebuildlist[i] = pkgsplits[ebuildlist[i]]
1364         del pkgsplits
1365
1366         slot_keywords = {}
1367
1368         if len(pkgs) != len(ebuildlist):
1369                 # If we can't access all the metadata then it's totally unsafe to
1370                 # commit since there's no way to generate a correct Manifest.
1371                 # Do not try to do any more QA checks on this package since missing
1372                 # metadata leads to false positives for several checks, and false
1373                 # positives confuse users.
1374                 can_force = False
1375                 continue
1376
1377         for y in checkdirlist:
1378                 m = disallowed_filename_chars_re.search(y.strip(os.sep))
1379                 if m is not None:
1380                         y_relative = os.path.join(checkdir_relative, y)
1381                         if vcs is not None and not vcs_new_changed(y_relative):
1382                                 # If the file isn't in the VCS new or changed set, then
1383                                 # assume that it's an irrelevant temporary file (Manifest
1384                                 # entries are not generated for file names containing
1385                                 # prohibited characters). See bug #406877.
1386                                 m = None
1387                 if m is not None:
1388                         stats["file.name"] += 1
1389                         fails["file.name"].append("%s/%s: char '%s'" % \
1390                                 (checkdir, y, m.group(0)))
1391
1392                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1393                         continue
1394                 f = None
1395                 try:
1396                         line = 1
1397                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1398                                 encoding=_encodings['fs'], errors='strict'),
1399                                 mode='r', encoding=_encodings['repo.content'])
1400                         for l in f:
1401                                 line +=1
1402                 except UnicodeDecodeError as ue:
1403                         stats["file.UTF8"] += 1
1404                         s = ue.object[:ue.start]
1405                         l2 = s.count("\n")
1406                         line += l2
1407                         if l2 != 0:
1408                                 s = s[s.rfind("\n") + 1:]
1409                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1410                 finally:
1411                         if f is not None:
1412                                 f.close()
1413
1414         if vcs in ("git", "hg") and check_ebuild_notadded:
1415                 if vcs == "git":
1416                         myf = os.popen("git ls-files --others %s" % \
1417                                 (portage._shell_quote(checkdir_relative),))
1418                 if vcs == "hg":
1419                         myf = os.popen("hg status --no-status --unknown %s" % \
1420                                 (portage._shell_quote(checkdir_relative),))
1421                 for l in myf:
1422                         if l[:-1][-7:] == ".ebuild":
1423                                 stats["ebuild.notadded"] += 1
1424                                 fails["ebuild.notadded"].append(
1425                                         os.path.join(x, os.path.basename(l[:-1])))
1426                 myf.close()
1427
1428         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1429                 try:
1430                         if vcs == "cvs":
1431                                 myf=open(checkdir+"/CVS/Entries","r")
1432                         if vcs == "svn":
1433                                 myf = os.popen("svn status --depth=files --verbose " + checkdir)
1434                         if vcs == "bzr":
1435                                 myf = os.popen("bzr ls -v --kind=file " + checkdir)
1436                         myl = myf.readlines()
1437                         myf.close()
1438                         for l in myl:
1439                                 if vcs == "cvs":
1440                                         if l[0]!="/":
1441                                                 continue
1442                                         splitl=l[1:].split("/")
1443                                         if not len(splitl):
1444                                                 continue
1445                                         if splitl[0][-7:]==".ebuild":
1446                                                 eadded.append(splitl[0][:-7])
1447                                 if vcs == "svn":
1448                                         if l[:1] == "?":
1449                                                 continue
1450                                         if l[:7] == '      >':
1451                                                 # tree conflict, new in subversion 1.6
1452                                                 continue
1453                                         l = l.split()[-1]
1454                                         if l[-7:] == ".ebuild":
1455                                                 eadded.append(os.path.basename(l[:-7]))
1456                                 if vcs == "bzr":
1457                                         if l[1:2] == "?":
1458                                                 continue
1459                                         l = l.split()[-1]
1460                                         if l[-7:] == ".ebuild":
1461                                                 eadded.append(os.path.basename(l[:-7]))
1462                         if vcs == "svn":
1463                                 myf = os.popen("svn status " + checkdir)
1464                                 myl=myf.readlines()
1465                                 myf.close()
1466                                 for l in myl:
1467                                         if l[0] == "A":
1468                                                 l = l.rstrip().split(' ')[-1]
1469                                                 if l[-7:] == ".ebuild":
1470                                                         eadded.append(os.path.basename(l[:-7]))
1471                 except IOError:
1472                         if vcs == "cvs":
1473                                 stats["CVS/Entries.IO_error"] += 1
1474                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1475                         else:
1476                                 raise
1477                         continue
1478
1479         mf = repoman_settings.repositories.get_repo_for_location(
1480                 os.path.dirname(os.path.dirname(checkdir)))
1481         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1482         mydigests=mf.getTypeDigests("DIST")
1483
1484         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1485         myfiles_all = []
1486         src_uri_error = False
1487         for mykey in fetchlist_dict:
1488                 try:
1489                         myfiles_all.extend(fetchlist_dict[mykey])
1490                 except portage.exception.InvalidDependString as e:
1491                         src_uri_error = True
1492                         try:
1493                                 portdb.aux_get(mykey, ["SRC_URI"])
1494                         except KeyError:
1495                                 # This will be reported as an "ebuild.syntax" error.
1496                                 pass
1497                         else:
1498                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1499                                 fails["SRC_URI.syntax"].append(
1500                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1501         del fetchlist_dict
1502         if not src_uri_error:
1503                 # This test can produce false positives if SRC_URI could not
1504                 # be parsed for one or more ebuilds. There's no point in
1505                 # producing a false error here since the root cause will
1506                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1507                 # or "ebuild.sytax".
1508                 myfiles_all = set(myfiles_all)
1509                 for entry in mydigests:
1510                         if entry not in myfiles_all:
1511                                 stats["digest.unused"] += 1
1512                                 fails["digest.unused"].append(checkdir+"::"+entry)
1513                 for entry in myfiles_all:
1514                         if entry not in mydigests:
1515                                 stats["digest.missing"] += 1
1516                                 fails["digest.missing"].append(checkdir+"::"+entry)
1517         del myfiles_all
1518
1519         if os.path.exists(checkdir+"/files"):
1520                 filesdirlist=os.listdir(checkdir+"/files")
1521
1522                 # recurse through files directory
1523                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1524                 while filesdirlist:
1525                         y = filesdirlist.pop(0)
1526                         relative_path = os.path.join(x, "files", y)
1527                         full_path = os.path.join(repodir, relative_path)
1528                         try:
1529                                 mystat = os.stat(full_path)
1530                         except OSError as oe:
1531                                 if oe.errno == 2:
1532                                         # don't worry about it.  it likely was removed via fix above.
1533                                         continue
1534                                 else:
1535                                         raise oe
1536                         if S_ISDIR(mystat.st_mode):
1537                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1538                                 if y == "CVS" or y == ".svn":
1539                                         continue
1540                                 for z in os.listdir(checkdir+"/files/"+y):
1541                                         if z == "CVS" or z == ".svn":
1542                                                 continue
1543                                         filesdirlist.append(y+"/"+z)
1544                         # Current policy is no files over 20 KiB, these are the checks. File size between
1545                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1546                         elif mystat.st_size > 61440:
1547                                 stats["file.size.fatal"] += 1
1548                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1549                         elif mystat.st_size > 20480:
1550                                 stats["file.size"] += 1
1551                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1552
1553                         m = disallowed_filename_chars_re.search(
1554                                 os.path.basename(y.rstrip(os.sep)))
1555                         if m is not None:
1556                                 y_relative = os.path.join(checkdir_relative, "files", y)
1557                                 if vcs is not None and not vcs_new_changed(y_relative):
1558                                         # If the file isn't in the VCS new or changed set, then
1559                                         # assume that it's an irrelevant temporary file (Manifest
1560                                         # entries are not generated for file names containing
1561                                         # prohibited characters). See bug #406877.
1562                                         m = None
1563                         if m is not None:
1564                                 stats["file.name"] += 1
1565                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1566                                         (checkdir, y, m.group(0)))
1567
1568                         if desktop_file_validate and desktop_pattern.match(y):
1569                                 cmd_output = validate_desktop_entry(full_path)
1570                                 if cmd_output:
1571                                         # Note: in the future we may want to grab the
1572                                         # warnings in addition to the errors. We're
1573                                         # just doing errors now since we don't want
1574                                         # to generate too much noise at first.
1575                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1576                                         for line in cmd_output:
1577                                                 error_match = error_re.match(line)
1578                                                 if error_match is None:
1579                                                         continue
1580                                                 stats["desktop.invalid"] += 1
1581                                                 fails["desktop.invalid"].append(
1582                                                         relative_path + ': %s' % error_match.group(1))
1583
1584         del mydigests
1585
1586         if check_changelog and "ChangeLog" not in checkdirlist:
1587                 stats["changelog.missing"]+=1
1588                 fails["changelog.missing"].append(x+"/ChangeLog")
1589
1590         musedict = {}
1591         #metadata.xml file check
1592         if "metadata.xml" not in checkdirlist:
1593                 stats["metadata.missing"]+=1
1594                 fails["metadata.missing"].append(x+"/metadata.xml")
1595         #metadata.xml parse check
1596         else:
1597                 metadata_bad = False
1598
1599                 # read metadata.xml into memory
1600                 try:
1601                         _metadata_xml = xml.etree.ElementTree.parse(
1602                                 os.path.join(checkdir, "metadata.xml"),
1603                                 parser=xml.etree.ElementTree.XMLParser(
1604                                         target=_MetadataTreeBuilder()))
1605                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1606                         metadata_bad = True
1607                         stats["metadata.bad"] += 1
1608                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1609                         del e
1610                 else:
1611                         # load USE flags from metadata.xml
1612                         try:
1613                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1614                         except portage.exception.ParseError as e:
1615                                 metadata_bad = True
1616                                 stats["metadata.bad"] += 1
1617                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1618
1619                         # Run other metadata.xml checkers
1620                         try:
1621                                 utilities.check_metadata(_metadata_xml, herd_base)
1622                         except (utilities.UnknownHerdsError, ) as e:
1623                                 metadata_bad = True
1624                                 stats["metadata.bad"] += 1
1625                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1626                                 del e
1627
1628                 #Only carry out if in package directory or check forced
1629                 if xmllint_capable and not metadata_bad:
1630                         # xmlint can produce garbage output even on success, so only dump
1631                         # the ouput when it fails.
1632                         st, out = subprocess_getstatusoutput(
1633                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1634                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1635                         if st != os.EX_OK:
1636                                 print(red("!!!") + " metadata.xml is invalid:")
1637                                 for z in out.splitlines():
1638                                         print(red("!!! ")+z)
1639                                 stats["metadata.bad"]+=1
1640                                 fails["metadata.bad"].append(x+"/metadata.xml")
1641
1642                 del metadata_bad
1643         muselist = frozenset(musedict)
1644
1645         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1646         changelog_modified = changelog_path in modified_changelogs
1647
1648         # detect unused local USE-descriptions
1649         used_useflags = set()
1650
1651         for y in ebuildlist:
1652                 relative_path = os.path.join(x, y + ".ebuild")
1653                 full_path = os.path.join(repodir, relative_path)
1654                 ebuild_path = y + ".ebuild"
1655                 if repolevel < 3:
1656                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1657                 if repolevel < 2:
1658                         ebuild_path = os.path.join(catdir, ebuild_path)
1659                 ebuild_path = os.path.join(".", ebuild_path)
1660                 if check_changelog and not changelog_modified \
1661                         and ebuild_path in new_ebuilds:
1662                         stats['changelog.ebuildadded'] += 1
1663                         fails['changelog.ebuildadded'].append(relative_path)
1664
1665                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1666                         #ebuild not added to vcs
1667                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1668                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1669                 myesplit=portage.pkgsplit(y)
1670                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1671                             or pv_toolong_re.search(myesplit[1]) \
1672                             or pv_toolong_re.search(myesplit[2]):
1673                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1674                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1675                         continue
1676                 elif myesplit[0]!=pkgdir:
1677                         print(pkgdir,myesplit[0])
1678                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1679                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1680                         continue
1681
1682                 pkg = pkgs[y]
1683
1684                 if pkg.invalid:
1685                         allvalid = False
1686                         for k, msgs in pkg.invalid.items():
1687                                 for msg in msgs:
1688                                         stats[k] = stats[k] + 1
1689                                         fails[k].append("%s %s" % (relative_path, msg))
1690                         continue
1691
1692                 myaux = pkg.metadata
1693                 eapi = myaux["EAPI"]
1694                 inherited = pkg.inherited
1695                 live_ebuild = live_eclasses.intersection(inherited)
1696
1697                 for k, v in myaux.items():
1698                         if not isinstance(v, basestring):
1699                                 continue
1700                         m = non_ascii_re.search(v)
1701                         if m is not None:
1702                                 stats["variable.invalidchar"] += 1
1703                                 fails["variable.invalidchar"].append(
1704                                         ("%s: %s variable contains non-ASCII " + \
1705                                         "character at position %s") % \
1706                                         (relative_path, k, m.start() + 1))
1707
1708                 if not src_uri_error:
1709                         # Check that URIs don't reference a server from thirdpartymirrors.
1710                         for uri in portage.dep.use_reduce( \
1711                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1712                                 contains_mirror = False
1713                                 for mirror in thirdpartymirrors:
1714                                         if uri.startswith(mirror):
1715                                                 contains_mirror = True
1716                                                 break
1717                                 if not contains_mirror:
1718                                         continue
1719
1720                                 stats["SRC_URI.mirror"] += 1
1721                                 fails["SRC_URI.mirror"].append(
1722                                         "%s: '%s' found in thirdpartymirrors" % \
1723                                         (relative_path, mirror))
1724
1725                 if myaux.get("PROVIDE"):
1726                         stats["virtual.oldstyle"]+=1
1727                         fails["virtual.oldstyle"].append(relative_path)
1728
1729                 for pos, missing_var in enumerate(missingvars):
1730                         if not myaux.get(missing_var):
1731                                 if catdir == "virtual" and \
1732                                         missing_var in ("HOMEPAGE", "LICENSE"):
1733                                         continue
1734                                 if live_ebuild and missing_var == "KEYWORDS":
1735                                         continue
1736                                 myqakey=missingvars[pos]+".missing"
1737                                 stats[myqakey]=stats[myqakey]+1
1738                                 fails[myqakey].append(x+"/"+y+".ebuild")
1739
1740                 if catdir == "virtual":
1741                         for var in ("HOMEPAGE", "LICENSE"):
1742                                 if myaux.get(var):
1743                                         myqakey = var + ".virtual"
1744                                         stats[myqakey] = stats[myqakey] + 1
1745                                         fails[myqakey].append(relative_path)
1746
1747                 # 14 is the length of DESCRIPTION=""
1748                 if len(myaux['DESCRIPTION']) > max_desc_len:
1749                         stats['DESCRIPTION.toolong'] += 1
1750                         fails['DESCRIPTION.toolong'].append(
1751                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1752                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1753
1754                 keywords = myaux["KEYWORDS"].split()
1755                 stable_keywords = []
1756                 for keyword in keywords:
1757                         if not keyword.startswith("~") and \
1758                                 not keyword.startswith("-"):
1759                                 stable_keywords.append(keyword)
1760                 if stable_keywords:
1761                         if ebuild_path in new_ebuilds:
1762                                 stable_keywords.sort()
1763                                 stats["KEYWORDS.stable"] += 1
1764                                 fails["KEYWORDS.stable"].append(
1765                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1766                                                 " ".join(stable_keywords))
1767
1768                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1769                         if not kw.startswith("-"))
1770
1771                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1772                 if previous_keywords is None:
1773                         slot_keywords[myaux["SLOT"]] = set()
1774                 elif ebuild_archs and not live_ebuild:
1775                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1776                         if dropped_keywords:
1777                                 stats["KEYWORDS.dropped"] += 1
1778                                 fails["KEYWORDS.dropped"].append(
1779                                         relative_path + ": %s" % \
1780                                         " ".join(sorted(dropped_keywords)))
1781
1782                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1783
1784                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1785                 if "-*" in keywords:
1786                         haskeyword = False
1787                         for kw in keywords:
1788                                 if kw[0] == "~":
1789                                         kw = kw[1:]
1790                                 if kw in kwlist:
1791                                         haskeyword = True
1792                         if not haskeyword:
1793                                 stats["KEYWORDS.stupid"] += 1
1794                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1795
1796                 """
1797                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1798                 not be allowed to be marked stable
1799                 """
1800                 if live_ebuild and repo_config.name == "gentoo":
1801                         bad_stable_keywords = []
1802                         for keyword in keywords:
1803                                 if not keyword.startswith("~") and \
1804                                         not keyword.startswith("-"):
1805                                         bad_stable_keywords.append(keyword)
1806                                 del keyword
1807                         if bad_stable_keywords:
1808                                 stats["LIVEVCS.stable"] += 1
1809                                 fails["LIVEVCS.stable"].append(
1810                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1811                                                 bad_stable_keywords)
1812                         del bad_stable_keywords
1813
1814                         if keywords and not has_global_mask(pkg):
1815                                 stats["LIVEVCS.unmasked"] += 1
1816                                 fails["LIVEVCS.unmasked"].append(relative_path)
1817
1818                 if options.ignore_arches:
1819                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1820                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1821                 else:
1822                         arches=[]
1823                         for keyword in myaux["KEYWORDS"].split():
1824                                 if (keyword[0]=="-"):
1825                                         continue
1826                                 elif (keyword[0]=="~"):
1827                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1828                                 else:
1829                                         arches.append([keyword, keyword, [keyword]])
1830                         if not arches:
1831                                 # Use an empty profile for checking dependencies of
1832                                 # packages that have empty KEYWORDS.
1833                                 arches.append(['**', '**', ['**']])
1834
1835                 unknown_pkgs = set()
1836                 baddepsyntax = False
1837                 badlicsyntax = False
1838                 badprovsyntax = False
1839                 catpkg = catdir+"/"+y
1840
1841                 inherited_java_eclass = "java-pkg-2" in inherited or \
1842                         "java-pkg-opt-2" in inherited
1843                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1844                 operator_tokens = set(["||", "(", ")"])
1845                 type_list, badsyntax = [], []
1846                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1847                         "LICENSE", "PROPERTIES", "PROVIDE"):
1848                         mydepstr = myaux[mytype]
1849
1850                         token_class = None
1851                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1852                                 token_class=portage.dep.Atom
1853
1854                         try:
1855                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1856                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1857                         except portage.exception.InvalidDependString as e:
1858                                 atoms = None
1859                                 badsyntax.append(str(e))
1860
1861                         if atoms and mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1862                                 if mytype in ("RDEPEND", "PDEPEND") and \
1863                                         "test?" in mydepstr.split():
1864                                         stats[mytype + '.suspect'] += 1
1865                                         fails[mytype + '.suspect'].append(relative_path + \
1866                                                 ": 'test?' USE conditional in %s" % mytype)
1867
1868                                 for atom in atoms:
1869                                         if atom == "||":
1870                                                 continue
1871
1872                                         # Skip dependency.unknown for blockers, so that we
1873                                         # don't encourage people to remove necessary blockers,
1874                                         # as discussed in bug #382407.
1875                                         if atom.blocker is None and \
1876                                                 not portdb.xmatch("match-all", atom) and \
1877                                                 not atom.cp.startswith("virtual/"):
1878                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1879
1880                                         is_blocker = atom.blocker
1881
1882                                         if mytype == "DEPEND" and \
1883                                                 not is_blocker and \
1884                                                 not inherited_java_eclass and \
1885                                                 atom.cp == "virtual/jdk":
1886                                                 stats['java.eclassesnotused'] += 1
1887                                                 fails['java.eclassesnotused'].append(relative_path)
1888                                         elif mytype == "DEPEND" and \
1889                                                 not is_blocker and \
1890                                                 not inherited_wxwidgets_eclass and \
1891                                                 atom.cp == "x11-libs/wxGTK":
1892                                                 stats['wxwidgets.eclassnotused'] += 1
1893                                                 fails['wxwidgets.eclassnotused'].append(
1894                                                         relative_path + ": DEPENDs on x11-libs/wxGTK"
1895                                                         " without inheriting wxwidgets.eclass")
1896                                         elif mytype in ("PDEPEND", "RDEPEND"):
1897                                                 if not is_blocker and \
1898                                                         atom.cp in suspect_rdepend:
1899                                                         stats[mytype + '.suspect'] += 1
1900                                                         fails[mytype + '.suspect'].append(
1901                                                                 relative_path + ": '%s'" % atom)
1902
1903                                         if atom.operator == "~" and \
1904                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1905                                                 stats[mytype + '.badtilde'] += 1
1906                                                 fails[mytype + '.badtilde'].append(
1907                                                         (relative_path + ": %s uses the ~ operator"
1908                                                          " with a non-zero revision:" + \
1909                                                          " '%s'") % (mytype, atom))
1910
1911                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1912
1913                 for m,b in zip(type_list, badsyntax):
1914                         stats[m+".syntax"] += 1
1915                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1916
1917                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1918                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1919                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1920                 badlicsyntax = badlicsyntax > 0
1921                 badprovsyntax = badprovsyntax > 0
1922
1923                 # uselist checks - global
1924                 myuse = []
1925                 default_use = []
1926                 for myflag in myaux["IUSE"].split():
1927                         flag_name = myflag.lstrip("+-")
1928                         used_useflags.add(flag_name)
1929                         if myflag != flag_name:
1930                                 default_use.append(myflag)
1931                         if flag_name not in uselist:
1932                                 myuse.append(flag_name)
1933
1934                 # uselist checks - metadata
1935                 for mypos in range(len(myuse)-1,-1,-1):
1936                         if myuse[mypos] and (myuse[mypos] in muselist):
1937                                 del myuse[mypos]
1938
1939                 if default_use and not eapi_has_iuse_defaults(eapi):
1940                         for myflag in default_use:
1941                                 stats['EAPI.incompatible'] += 1
1942                                 fails['EAPI.incompatible'].append(
1943                                         (relative_path + ": IUSE defaults" + \
1944                                         " not supported with EAPI='%s':" + \
1945                                         " '%s'") % (eapi, myflag))
1946
1947                 for mypos in range(len(myuse)):
1948                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1949                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1950
1951                 # license checks
1952                 if not badlicsyntax:
1953                         # Parse the LICENSE variable, remove USE conditions and
1954                         # flatten it.
1955                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
1956                         # Check each entry to ensure that it exists in PORTDIR's
1957                         # license directory.
1958                         for lic in licenses:
1959                                 # Need to check for "||" manually as no portage
1960                                 # function will remove it without removing values.
1961                                 if lic not in liclist and lic != "||":
1962                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1963                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
1964
1965                 #keyword checks
1966                 myuse = myaux["KEYWORDS"].split()
1967                 for mykey in myuse:
1968                         if mykey not in ("-*", "*", "~*"):
1969                                 myskey = mykey
1970                                 if myskey[:1] == "-":
1971                                         myskey = myskey[1:]
1972                                 if myskey[:1] == "~":
1973                                         myskey = myskey[1:]
1974                                 if myskey not in kwlist:
1975                                         stats["KEYWORDS.invalid"] += 1
1976                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1977                                 elif myskey not in profiles:
1978                                         stats["KEYWORDS.invalid"] += 1
1979                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1980
1981                 #restrict checks
1982                 myrestrict = None
1983                 try:
1984                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
1985                 except portage.exception.InvalidDependString as e:
1986                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1987                         fails["RESTRICT.syntax"].append(
1988                                 "%s: RESTRICT: %s" % (relative_path, e))
1989                         del e
1990                 if myrestrict:
1991                         myrestrict = set(myrestrict)
1992                         mybadrestrict = myrestrict.difference(valid_restrict)
1993                         if mybadrestrict:
1994                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
1995                                 for mybad in mybadrestrict:
1996                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1997                 #REQUIRED_USE check
1998                 required_use = myaux["REQUIRED_USE"]
1999                 if required_use:
2000                         if not eapi_has_required_use(eapi):
2001                                 stats['EAPI.incompatible'] += 1
2002                                 fails['EAPI.incompatible'].append(
2003                                         relative_path + ": REQUIRED_USE" + \
2004                                         " not supported with EAPI='%s'" % (eapi,))
2005                         try:
2006                                 portage.dep.check_required_use(required_use, (),
2007                                         pkg.iuse.is_valid_flag)
2008                         except portage.exception.InvalidDependString as e:
2009                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2010                                 fails["REQUIRED_USE.syntax"].append(
2011                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2012                                 del e
2013
2014                 # Syntax Checks
2015                 relative_path = os.path.join(x, y + ".ebuild")
2016                 full_path = os.path.join(repodir, relative_path)
2017                 if not vcs_preserves_mtime:
2018                         if ebuild_path not in new_ebuilds and \
2019                                 ebuild_path not in modified_ebuilds:
2020                                 pkg.mtime = None
2021                 try:
2022                         # All ebuilds should have utf_8 encoding.
2023                         f = io.open(_unicode_encode(full_path,
2024                                 encoding=_encodings['fs'], errors='strict'),
2025                                 mode='r', encoding=_encodings['repo.content'])
2026                         try:
2027                                 for check_name, e in run_checks(f, pkg):
2028                                         stats[check_name] += 1
2029                                         fails[check_name].append(relative_path + ': %s' % e)
2030                         finally:
2031                                 f.close()
2032                 except UnicodeDecodeError:
2033                         # A file.UTF8 failure will have already been recorded above.
2034                         pass
2035
2036                 if options.force:
2037                         # The dep_check() calls are the most expensive QA test. If --force
2038                         # is enabled, there's no point in wasting time on these since the
2039                         # user is intent on forcing the commit anyway.
2040                         continue
2041
2042                 for keyword,arch,groups in arches:
2043
2044                         if arch not in profiles:
2045                                 # A missing profile will create an error further down
2046                                 # during the KEYWORDS verification.
2047                                 continue
2048
2049                         for prof in profiles[arch]:
2050
2051                                 if prof.status not in ("stable", "dev") or \
2052                                         prof.status == "dev" and not options.include_dev:
2053                                         continue
2054
2055                                 dep_settings = arch_caches.get(prof.sub_path)
2056                                 if dep_settings is None:
2057                                         dep_settings = portage.config(
2058                                                 config_profile_path=prof.abs_path,
2059                                                 config_incrementals=repoman_incrementals,
2060                                                 config_root=config_root,
2061                                                 local_config=False,
2062                                                 _unmatched_removal=options.unmatched_removal,
2063                                                 env=env)
2064                                         dep_settings.categories = repoman_settings.categories
2065                                         if options.without_mask:
2066                                                 dep_settings._mask_manager = \
2067                                                         copy.deepcopy(dep_settings._mask_manager)
2068                                                 dep_settings._mask_manager._pmaskdict.clear()
2069                                         arch_caches[prof.sub_path] = dep_settings
2070
2071                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2072                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2073                                 if xcache is None:
2074                                         portdb.melt()
2075                                         portdb.freeze()
2076                                         xcache = portdb.xcache
2077                                         xcache.update(shared_xmatch_caches)
2078                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2079
2080                                 trees[root]["porttree"].settings = dep_settings
2081                                 portdb.settings = dep_settings
2082                                 portdb.xcache = xcache
2083                                 # for package.use.mask support inside dep_check
2084                                 dep_settings.setcpv(pkg)
2085                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2086                                 # just in case, prevent config.reset() from nuking these.
2087                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2088
2089                                 if not baddepsyntax:
2090                                         ismasked = not ebuild_archs or \
2091                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2092                                         if ismasked:
2093                                                 if not have_pmasked:
2094                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2095                                                                 pkg.cpv, pkg.metadata))
2096                                                 if options.ignore_masked:
2097                                                         continue
2098                                                 #we are testing deps for a masked package; give it some lee-way
2099                                                 suffix="masked"
2100                                                 matchmode = "minimum-all"
2101                                         else:
2102                                                 suffix=""
2103                                                 matchmode = "minimum-visible"
2104
2105                                         if not have_dev_keywords:
2106                                                 have_dev_keywords = \
2107                                                         bool(dev_keywords.intersection(keywords))
2108
2109                                         if prof.status == "dev":
2110                                                 suffix=suffix+"indev"
2111
2112                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
2113
2114                                                 mykey=mytype+".bad"+suffix
2115                                                 myvalue = myaux[mytype]
2116                                                 if not myvalue:
2117                                                         continue
2118
2119                                                 success, atoms = portage.dep_check(myvalue, portdb,
2120                                                         dep_settings, use="all", mode=matchmode,
2121                                                         trees=trees)
2122
2123                                                 if success:
2124                                                         if atoms:
2125
2126                                                                 # Don't bother with dependency.unknown for
2127                                                                 # cases in which *DEPEND.bad is triggered.
2128                                                                 for atom in atoms:
2129                                                                         # dep_check returns all blockers and they
2130                                                                         # aren't counted for *DEPEND.bad, so we
2131                                                                         # ignore them here.
2132                                                                         if not atom.blocker:
2133                                                                                 unknown_pkgs.discard(
2134                                                                                         (mytype, atom.unevaluated_atom))
2135
2136                                                                 if not prof.sub_path:
2137                                                                         # old-style virtuals currently aren't
2138                                                                         # resolvable with empty profile, since
2139                                                                         # 'virtuals' mappings are unavailable
2140                                                                         # (it would be expensive to search
2141                                                                         # for PROVIDE in all ebuilds)
2142                                                                         atoms = [atom for atom in atoms if not \
2143                                                                                 (atom.cp.startswith('virtual/') and \
2144                                                                                 not portdb.cp_list(atom.cp))]
2145
2146                                                                 #we have some unsolvable deps
2147                                                                 #remove ! deps, which always show up as unsatisfiable
2148                                                                 atoms = [str(atom.unevaluated_atom) \
2149                                                                         for atom in atoms if not atom.blocker]
2150
2151                                                                 #if we emptied out our list, continue:
2152                                                                 if not atoms:
2153                                                                         continue
2154                                                                 stats[mykey]=stats[mykey]+1
2155                                                                 fails[mykey].append("%s: %s(%s) %s" % \
2156                                                                         (relative_path, keyword,
2157                                                                         prof, repr(atoms)))
2158                                                 else:
2159                                                         stats[mykey]=stats[mykey]+1
2160                                                         fails[mykey].append("%s: %s(%s) %s" % \
2161                                                                 (relative_path, keyword,
2162                                                                 prof, repr(atoms)))
2163
2164                 if not baddepsyntax and unknown_pkgs:
2165                         type_map = {}
2166                         for mytype, atom in unknown_pkgs:
2167                                 type_map.setdefault(mytype, set()).add(atom)
2168                         for mytype, atoms in type_map.items():
2169                                 stats["dependency.unknown"] += 1
2170                                 fails["dependency.unknown"].append("%s: %s: %s" %
2171                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2172
2173         # check if there are unused local USE-descriptions in metadata.xml
2174         # (unless there are any invalids, to avoid noise)
2175         if allvalid:
2176                 for myflag in muselist.difference(used_useflags):
2177                         stats["metadata.warning"] += 1
2178                         fails["metadata.warning"].append(
2179                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2180                                 (x, myflag))
2181
2182 if options.if_modified == "y" and len(effective_scanlist) < 1:
2183         logging.warn("--if-modified is enabled, but no modified packages were found!")
2184
2185 if options.mode == "manifest":
2186         sys.exit(dofail)
2187
2188 #dofail will be set to 1 if we have failed in at least one non-warning category
2189 dofail=0
2190 #dowarn will be set to 1 if we tripped any warnings
2191 dowarn=0
2192 #dofull will be set if we should print a "repoman full" informational message
2193 dofull = options.mode != 'full'
2194
2195 for x in qacats:
2196         if not stats[x]:
2197                 continue
2198         dowarn = 1
2199         if x not in qawarnings:
2200                 dofail = 1
2201
2202 if dofail or \
2203         (dowarn and not (options.quiet or options.mode == "scan")):
2204         dofull = 0
2205
2206 # Save QA output so that it can be conveniently displayed
2207 # in $EDITOR while the user creates a commit message.
2208 # Otherwise, the user would not be able to see this output
2209 # once the editor has taken over the screen.
2210 qa_output = io.StringIO()
2211 style_file = ConsoleStyleFile(sys.stdout)
2212 if options.mode == 'commit' and \
2213         (not commitmessage or not commitmessage.strip()):
2214         style_file.write_listener = qa_output
2215 console_writer = StyleWriter(file=style_file, maxcol=9999)
2216 console_writer.style_listener = style_file.new_styles
2217
2218 f = formatter.AbstractFormatter(console_writer)
2219
2220 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2221
2222 style_file.flush()
2223 del console_writer, f, style_file
2224 qa_output = qa_output.getvalue()
2225 qa_output = qa_output.splitlines(True)
2226
2227 def grouplist(mylist,seperator="/"):
2228         """(list,seperator="/") -- Takes a list of elements; groups them into
2229         same initial element categories. Returns a dict of {base:[sublist]}
2230         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2231         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2232         mygroups={}
2233         for x in mylist:
2234                 xs=x.split(seperator)
2235                 if xs[0]==".":
2236                         xs=xs[1:]
2237                 if xs[0] not in mygroups:
2238                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2239                 else:
2240                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2241         return mygroups
2242
2243 suggest_ignore_masked = False
2244 suggest_include_dev = False
2245
2246 if have_pmasked and not (options.without_mask or options.ignore_masked):
2247         suggest_ignore_masked = True
2248 if have_dev_keywords and not options.include_dev:
2249         suggest_include_dev = True
2250
2251 if suggest_ignore_masked or suggest_include_dev:
2252         print()
2253         if suggest_ignore_masked:
2254                 print(bold("Note: use --without-mask to check " + \
2255                         "KEYWORDS on dependencies of masked packages"))
2256
2257         if suggest_include_dev:
2258                 print(bold("Note: use --include-dev (-d) to check " + \
2259                         "dependencies for 'dev' profiles"))
2260         print()
2261
2262 if options.mode != 'commit':
2263         if dofull:
2264                 print(bold("Note: type \"repoman full\" for a complete listing."))
2265         if dowarn and not dofail:
2266                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2267         elif not dofail:
2268                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2269         elif dofail:
2270                 print(bad("Please fix these important QA issues first."))
2271                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2272                 sys.exit(1)
2273 else:
2274         if dofail and can_force and options.force and not options.pretend:
2275                 print(green("RepoMan sez:") + \
2276                         " \"You want to commit even with these QA issues?\n" + \
2277                         "              I'll take it this time, but I'm not happy.\"\n")
2278         elif dofail:
2279                 if options.force and not can_force:
2280                         print(bad("The --force option has been disabled due to extraordinary issues."))
2281                 print(bad("Please fix these important QA issues first."))
2282                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2283                 sys.exit(1)
2284
2285         if options.pretend:
2286                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2287
2288         myunadded = []
2289         if vcs == "cvs":
2290                 try:
2291                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2292                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2293                 except SystemExit as e:
2294                         raise  # TODO propagate this
2295                 except:
2296                         err("Error retrieving CVS tree; exiting.")
2297         if vcs == "svn":
2298                 try:
2299                         with os.popen("svn status --no-ignore") as f:
2300                                 svnstatus = f.readlines()
2301                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2302                 except SystemExit as e:
2303                         raise  # TODO propagate this
2304                 except:
2305                         err("Error retrieving SVN info; exiting.")
2306         if vcs == "git":
2307                 # get list of files not under version control or missing
2308                 myf = os.popen("git ls-files --others")
2309                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2310                 myf.close()
2311         if vcs == "bzr":
2312                 try:
2313                         with os.popen("bzr status -S .") as f:
2314                                 bzrstatus = f.readlines()
2315                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2316                 except SystemExit as e:
2317                         raise  # TODO propagate this
2318                 except:
2319                         err("Error retrieving bzr info; exiting.")
2320         if vcs == "hg":
2321                 with os.popen("hg status --no-status --unknown .") as f:
2322                         myunadded = f.readlines()
2323                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2324
2325                 # Mercurial doesn't handle manually deleted files as removed from
2326                 # the repository, so the user need to remove them before commit,
2327                 # using "hg remove [FILES]"
2328                 with os.popen("hg status --no-status --deleted .") as f:
2329                         mydeleted = f.readlines()
2330                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2331
2332
2333         myautoadd=[]
2334         if myunadded:
2335                 for x in range(len(myunadded)-1,-1,-1):
2336                         xs=myunadded[x].split("/")
2337                         if xs[-1]=="files":
2338                                 print("!!! files dir is not added! Please correct this.")
2339                                 sys.exit(-1)
2340                         elif xs[-1]=="Manifest":
2341                                 # It's a manifest... auto add
2342                                 myautoadd+=[myunadded[x]]
2343                                 del myunadded[x]
2344
2345         if myunadded:
2346                 print(red("!!! The following files are in your local tree but are not added to the master"))
2347                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2348                 for x in myunadded:
2349                         print("   ",x)
2350                 print()
2351                 print()
2352                 sys.exit(1)
2353
2354         if vcs == "hg" and mydeleted:
2355                 print(red("!!! The following files are removed manually from your local tree but are not"))
2356                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2357                 for x in mydeleted:
2358                         print("   ",x)
2359                 print()
2360                 print()
2361                 sys.exit(1)
2362
2363         if vcs == "cvs":
2364                 mycvstree = cvstree.getentries("./", recursive=1)
2365                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2366                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2367                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2368                 bin_blob_pattern = re.compile("^-kb$")
2369                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2370                         recursive=1, basedir="./"))
2371
2372
2373         if vcs == "svn":
2374                 with os.popen("svn status") as f:
2375                         svnstatus = f.readlines()
2376                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2377                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2378                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2379
2380                 # Subversion expands keywords specified in svn:keywords properties.
2381                 with os.popen("svn propget -R svn:keywords") as f:
2382                         props = f.readlines()
2383                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2384                         for prop in props if " - " in prop)
2385
2386         elif vcs == "git":
2387                 with os.popen("git diff-index --name-only "
2388                         "--relative --diff-filter=M HEAD") as f:
2389                         mychanged = f.readlines()
2390                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2391
2392                 with os.popen("git diff-index --name-only "
2393                         "--relative --diff-filter=A HEAD") as f:
2394                         mynew = f.readlines()
2395                 mynew = ["./" + elem[:-1] for elem in mynew]
2396
2397                 with os.popen("git diff-index --name-only "
2398                         "--relative --diff-filter=D HEAD") as f:
2399                         myremoved = f.readlines()
2400                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2401
2402         if vcs == "bzr":
2403                 with os.popen("bzr status -S .") as f:
2404                         bzrstatus = f.readlines()
2405                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2406                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2407                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2408                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2409                 # Bazaar expands nothing.
2410
2411         if vcs == "hg":
2412                 with os.popen("hg status --no-status --modified .") as f:
2413                         mychanged = f.readlines()
2414                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2415
2416                 with os.popen("hg status --no-status --added .") as f:
2417                         mynew = f.readlines()
2418                 mynew = ["./" + elem.rstrip() for elem in mynew]
2419
2420                 with os.popen("hg status --no-status --removed .") as f:
2421                         myremoved = f.readlines()
2422                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2423
2424         if vcs:
2425                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2426                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2427                         print()
2428                         print("(Didn't find any changed files...)")
2429                         print()
2430                         sys.exit(1)
2431
2432         # Manifests need to be regenerated after all other commits, so don't commit
2433         # them now even if they have changed.
2434         mymanifests = set()
2435         myupdates = set()
2436         for f in mychanged + mynew:
2437                 if "Manifest" == os.path.basename(f):
2438                         mymanifests.add(f)
2439                 else:
2440                         myupdates.add(f)
2441         myupdates.difference_update(myremoved)
2442         myupdates = list(myupdates)
2443         mymanifests = list(mymanifests)
2444         myheaders = []
2445         mydirty = []
2446
2447         commitmessage = options.commitmsg
2448         if options.commitmsgfile:
2449                 try:
2450                         f = io.open(_unicode_encode(options.commitmsgfile,
2451                         encoding=_encodings['fs'], errors='strict'),
2452                         mode='r', encoding=_encodings['content'], errors='replace')
2453                         commitmessage = f.read()
2454                         f.close()
2455                         del f
2456                 except (IOError, OSError) as e:
2457                         if e.errno == errno.ENOENT:
2458                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2459                         else:
2460                                 raise
2461                 # We've read the content so the file is no longer needed.
2462                 commitmessagefile = None
2463         if not commitmessage or not commitmessage.strip():
2464                 try:
2465                         editor = os.environ.get("EDITOR")
2466                         if editor and utilities.editor_is_executable(editor):
2467                                 commitmessage = utilities.get_commit_message_with_editor(
2468                                         editor, message=qa_output)
2469                         else:
2470                                 commitmessage = utilities.get_commit_message_with_stdin()
2471                 except KeyboardInterrupt:
2472                         exithandler()
2473                 if not commitmessage or not commitmessage.strip():
2474                         print("* no commit message?  aborting commit.")
2475                         sys.exit(1)
2476         commitmessage = commitmessage.rstrip()
2477         changelog_msg = commitmessage
2478         portage_version = getattr(portage, "VERSION", None)
2479         if portage_version is None:
2480                 sys.stderr.write("Failed to insert portage version in message!\n")
2481                 sys.stderr.flush()
2482                 portage_version = "Unknown"
2483         unameout = platform.system() + " "
2484         if platform.system() in ["Darwin", "SunOS"]:
2485                 unameout += platform.processor()
2486         else:
2487                 unameout += platform.machine()
2488         commitmessage += "\n\n(Portage version: %s/%s/%s" % \
2489                 (portage_version, vcs, unameout)
2490         if options.force:
2491                 commitmessage += ", RepoMan options: --force"
2492         commitmessage += ")"
2493
2494         if options.echangelog in ('y', 'force'):
2495                 logging.info("checking for unmodified ChangeLog files")
2496                 committer_name = utilities.get_committer_name(env=repoman_settings)
2497                 for x in sorted(vcs_files_to_cps(
2498                         chain(myupdates, mymanifests, myremoved))):
2499                         catdir, pkgdir = x.split("/")
2500                         checkdir = repodir + "/" + x
2501                         checkdir_relative = ""
2502                         if repolevel < 3:
2503                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2504                         if repolevel < 2:
2505                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2506                         checkdir_relative = os.path.join(".", checkdir_relative)
2507
2508                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2509                         changelog_modified = changelog_path in modified_changelogs
2510                         if changelog_modified and options.echangelog != 'force':
2511                                 continue
2512
2513                         # get changes for this package
2514                         cdrlen = len(checkdir_relative)
2515                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2516                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2517                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2518
2519                         # Skip ChangeLog generation if only the Manifest was modified,
2520                         # as discussed in bug #398009.
2521                         nontrivial_cl_files = set()
2522                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2523                         nontrivial_cl_files.difference_update(['Manifest'])
2524                         if not nontrivial_cl_files and options.echangelog != 'force':
2525                                 continue
2526
2527                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2528                                 committer_name, changelog_msg,
2529                                 os.path.join(repodir, 'skel.ChangeLog'),
2530                                 catdir, pkgdir,
2531                                 new=clnew, removed=clremoved, changed=clchanged,
2532                                 pretend=options.pretend)
2533                         if new_changelog is None:
2534                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2535                                         level=logging.ERROR, noiselevel=-1)
2536                                 sys.exit(1)
2537
2538                         # if the ChangeLog was just created, add it to vcs
2539                         if new_changelog:
2540                                 myautoadd.append(changelog_path)
2541                                 # myautoadd is appended to myupdates below
2542                         else:
2543                                 myupdates.append(changelog_path)
2544
2545         if myautoadd:
2546                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2547                 add_cmd = [vcs, "add"]
2548                 add_cmd += myautoadd
2549                 if options.pretend:
2550                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2551                                 noiselevel=-1)
2552                 else:
2553                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2554                                 # Python 3.1 produces the following TypeError if raw bytes are
2555                                 # passed to subprocess.call():
2556                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2557                                 #     errread, errwrite)
2558                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2559                                 #     raise child_exception
2560                                 # TypeError: expected an object with the buffer interface
2561                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2562                         retcode = subprocess.call(add_cmd)
2563                         if retcode != os.EX_OK:
2564                                 logging.error(
2565                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2566                                 sys.exit(retcode)
2567
2568                 myupdates += myautoadd
2569
2570         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2571
2572         if vcs not in ('cvs', 'svn'):
2573                 # With git, bzr and hg, there's never any keyword expansion, so
2574                 # there's no need to regenerate manifests and all files will be
2575                 # committed in one big commit at the end.
2576                 print()
2577         elif not repo_config.thin_manifest:
2578                 if vcs == 'cvs':
2579                         headerstring = "'\$(Header|Id).*\$'"
2580                 elif vcs == "svn":
2581                         svn_keywords = dict((k.lower(), k) for k in [
2582                                         "Rev",
2583                                         "Revision",
2584                                         "LastChangedRevision",
2585                                         "Date",
2586                                         "LastChangedDate",
2587                                         "Author",
2588                                         "LastChangedBy",
2589                                         "URL",
2590                                         "HeadURL",
2591                                         "Id",
2592                                         "Header",
2593                         ])
2594
2595                 for myfile in myupdates:
2596
2597                         # for CVS, no_expansion contains files that are excluded from expansion
2598                         if vcs == "cvs":
2599                                 if myfile in no_expansion:
2600                                         continue
2601
2602                         # for SVN, expansion contains files that are included in expansion
2603                         elif vcs == "svn":
2604                                 if myfile not in expansion:
2605                                         continue
2606
2607                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2608                                 enabled_keywords = []
2609                                 for k in expansion[myfile]:
2610                                         keyword = svn_keywords.get(k.lower())
2611                                         if keyword is not None:
2612                                                 enabled_keywords.append(keyword)
2613
2614                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2615
2616                         myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile)
2617                         if myout[0] == 0:
2618                                 myheaders.append(myfile)
2619
2620                 print("%s have headers that will change." % green(str(len(myheaders))))
2621                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2622
2623         logging.info("myupdates: %s", myupdates)
2624         logging.info("myheaders: %s", myheaders)
2625
2626         if options.ask and userquery('Commit changes?', True) != 'Yes':
2627                 print("* aborting commit.")
2628                 sys.exit(128 + signal.SIGINT)
2629
2630         # Handle the case where committed files have keywords which
2631         # will change and need a priming commit before the Manifest
2632         # can be committed.
2633         if (myupdates or myremoved) and myheaders:
2634                 myfiles = myupdates + myremoved
2635                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2636                 mymsg = os.fdopen(fd, "wb")
2637                 mymsg.write(_unicode_encode(commitmessage))
2638                 mymsg.close()
2639
2640                 print()
2641                 print(green("Using commit message:"))
2642                 print(green("------------------------------------------------------------------------------"))
2643                 print(commitmessage)
2644                 print(green("------------------------------------------------------------------------------"))
2645                 print()
2646
2647                 # Having a leading ./ prefix on file paths can trigger a bug in
2648                 # the cvs server when committing files to multiple directories,
2649                 # so strip the prefix.
2650                 myfiles = [f.lstrip("./") for f in myfiles]
2651
2652                 commit_cmd = [vcs]
2653                 commit_cmd.extend(vcs_global_opts)
2654                 commit_cmd.append("commit")
2655                 commit_cmd.extend(vcs_local_opts)
2656                 commit_cmd.extend(["-F", commitmessagefile])
2657                 commit_cmd.extend(myfiles)
2658
2659                 try:
2660                         if options.pretend:
2661                                 print("(%s)" % (" ".join(commit_cmd),))
2662                         else:
2663                                 retval = spawn(commit_cmd, env=os.environ)
2664                                 if retval != os.EX_OK:
2665                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2666                                                 "error code: %s\n") % (vcs, retval),
2667                                                 level=logging.ERROR, noiselevel=-1)
2668                                         sys.exit(retval)
2669                 finally:
2670                         try:
2671                                 os.unlink(commitmessagefile)
2672                         except OSError:
2673                                 pass
2674
2675         # Setup the GPG commands
2676         def gpgsign(filename):
2677                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2678                 if gpgcmd is None:
2679                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2680                                 " Is make.globals missing?")
2681                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2682                         "PORTAGE_GPG_KEY" not in repoman_settings:
2683                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2684                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2685                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2686                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2687                                         os.path.expanduser("~/.gnupg")
2688                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2689                                         % repoman_settings["PORTAGE_GPG_DIR"])
2690                         else:
2691                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2692                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2693                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2694                                 raise portage.exception.InvalidLocation(
2695                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2696                                         repoman_settings["PORTAGE_GPG_DIR"])
2697                 gpgvars = {"FILE": filename}
2698                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2699                         v = repoman_settings.get(k)
2700                         if v is not None:
2701                                 gpgvars[k] = v
2702                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2703                 if options.pretend:
2704                         print("("+gpgcmd+")")
2705                 else:
2706                         rValue = os.system(gpgcmd)
2707                         if rValue == os.EX_OK:
2708                                 os.rename(filename+".asc", filename)
2709                         else:
2710                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2711
2712         # When files are removed and re-added, the cvs server will put /Attic/
2713         # inside the $Header path. This code detects the problem and corrects it
2714         # so that the Manifest will generate correctly. See bug #169500.
2715         # Use binary mode in order to avoid potential character encoding issues.
2716         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2717         attic_str = b'/Attic/'
2718         attic_replace = b'/'
2719         for x in myheaders:
2720                 f = open(_unicode_encode(x,
2721                         encoding=_encodings['fs'], errors='strict'),
2722                         mode='rb')
2723                 mylines = f.readlines()
2724                 f.close()
2725                 modified = False
2726                 for i, line in enumerate(mylines):
2727                         if cvs_header_re.match(line) is not None and \
2728                                 attic_str in line:
2729                                 mylines[i] = line.replace(attic_str, attic_replace)
2730                                 modified = True
2731                 if modified:
2732                         portage.util.write_atomic(x, b''.join(mylines),
2733                                 mode='wb')
2734
2735         if repolevel == 1:
2736                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2737                         "doing the entire repository.\"\n")
2738
2739         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2740
2741                 for x in sorted(vcs_files_to_cps(
2742                         chain(myupdates, myremoved, mymanifests))):
2743                         repoman_settings["O"] = os.path.join(repodir, x)
2744                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2745
2746         signed = False
2747         if sign_manifests:
2748                 signed = True
2749                 try:
2750                         for x in sorted(vcs_files_to_cps(
2751                                 chain(myupdates, myremoved, mymanifests))):
2752                                 repoman_settings["O"] = os.path.join(repodir, x)
2753                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2754                 except portage.exception.PortageException as e:
2755                         portage.writemsg("!!! %s\n" % str(e))
2756                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2757                         signed = False
2758
2759         if vcs == 'git':
2760                 # It's not safe to use the git commit -a option since there might
2761                 # be some modified files elsewhere in the working tree that the
2762                 # user doesn't want to commit. Therefore, call git update-index
2763                 # in order to ensure that the index is updated with the latest
2764                 # versions of all new and modified files in the relevant portion
2765                 # of the working tree.
2766                 myfiles = mymanifests + myupdates
2767                 myfiles.sort()
2768                 update_index_cmd = ["git", "update-index"]
2769                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2770                 if options.pretend:
2771                         print("(%s)" % (" ".join(update_index_cmd),))
2772                 else:
2773                         retval = spawn(update_index_cmd, env=os.environ)
2774                         if retval != os.EX_OK:
2775                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2776                                         "error code: %s\n") % (vcs, retval),
2777                                         level=logging.ERROR, noiselevel=-1)
2778                                 sys.exit(retval)
2779
2780         if True:
2781
2782                 myfiles = mymanifests[:]
2783                 # If there are no header (SVN/CVS keywords) changes in
2784                 # the files, this Manifest commit must include the
2785                 # other (yet uncommitted) files.
2786                 if not myheaders:
2787                         myfiles += myupdates
2788                         myfiles += myremoved
2789                 myfiles.sort()
2790
2791                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2792                 mymsg = os.fdopen(fd, "wb")
2793                 # strip the closing parenthesis
2794                 mymsg.write(_unicode_encode(commitmessage[:-1]))
2795                 if signed:
2796                         mymsg.write(_unicode_encode(
2797                                 ", signed Manifest commit with key %s)" % \
2798                                 repoman_settings["PORTAGE_GPG_KEY"]))
2799                 else:
2800                         mymsg.write(b", unsigned Manifest commit)")
2801                 mymsg.close()
2802
2803                 commit_cmd = []
2804                 if options.pretend and vcs is None:
2805                         # substitute a bogus value for pretend output
2806                         commit_cmd.append("cvs")
2807                 else:
2808                         commit_cmd.append(vcs)
2809                 commit_cmd.extend(vcs_global_opts)
2810                 commit_cmd.append("commit")
2811                 commit_cmd.extend(vcs_local_opts)
2812                 if vcs == "hg":
2813                         commit_cmd.extend(["--logfile", commitmessagefile])
2814                         commit_cmd.extend(myfiles)
2815                 else:
2816                         commit_cmd.extend(["-F", commitmessagefile])
2817                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2818
2819                 try:
2820                         if options.pretend:
2821                                 print("(%s)" % (" ".join(commit_cmd),))
2822                         else:
2823                                 retval = spawn(commit_cmd, env=os.environ)
2824                                 if retval != os.EX_OK:
2825
2826                                         if repo_config.sign_commit and vcs == 'git' and \
2827                                                 not git_supports_gpg_sign():
2828                                                 # Inform user that newer git is needed (bug #403323).
2829                                                 logging.error(
2830                                                         "Git >=1.7.9 is required for signed commits!")
2831
2832                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2833                                                 "error code: %s\n") % (vcs, retval),
2834                                                 level=logging.ERROR, noiselevel=-1)
2835                                         sys.exit(retval)
2836                 finally:
2837                         try:
2838                                 os.unlink(commitmessagefile)
2839                         except OSError:
2840                                 pass
2841
2842         print()
2843         if vcs:
2844                 print("Commit complete.")
2845         else:
2846                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2847         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2848 sys.exit(0)
2849