bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2012 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27
  28 try:
  29         from urllib.request import urlopen as urllib_request_urlopen
  30 except ImportError:
  31         from urllib import urlopen as urllib_request_urlopen
  32
  33 from itertools import chain
  34 from stat import S_ISDIR
  35
  36 try:
  37         import portage
  38 except ImportError:
  39         from os import path as osp
  40         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  41         import portage
  42 portage._disable_legacy_globals()
  43 portage.dep._internal_warnings = True
  44
  45 try:
  46         import xml.etree.ElementTree
  47         from xml.parsers.expat import ExpatError
  48 except (SystemExit, KeyboardInterrupt):
  49         raise
  50 except (ImportError, SystemError, RuntimeError, Exception):
  51         # broken or missing xml support
  52         # http://bugs.python.org/issue14988
  53         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  54         from portage.output import EOutput
  55         out = EOutput()
  56         for line in msg:
  57                 out.eerror(line)
  58         sys.exit(1)
  59
  60 from portage import os
  61 from portage import subprocess_getstatusoutput
  62 from portage import _encodings
  63 from portage import _unicode_encode
  64 from repoman.checks import run_checks
  65 from repoman import utilities
  66 from repoman.herdbase import make_herd_base
  67 from _emerge.Package import Package
  68 from _emerge.RootConfig import RootConfig
  69 from _emerge.userquery import userquery
  70 import portage.checksum
  71 import portage.const
  72 from portage import cvstree, normalize_path
  73 from portage import util
  74 from portage.exception import (FileNotFound, MissingParameter,
  75         ParseError, PermissionDenied)
  76 from portage.manifest import _prohibited_filename_chars_re as \
  77         disallowed_filename_chars_re
  78 from portage.process import find_binary, spawn
  79 from portage.output import bold, create_color_func, \
  80         green, nocolor, red
  81 from portage.output import ConsoleStyleFile, StyleWriter
  82 from portage.util import writemsg_level
  83 from portage.util._desktop_entry import validate_desktop_entry
  84 from portage.package.ebuild.digestgen import digestgen
  85 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  86
  87 if sys.hexversion >= 0x3000000:
  88         basestring = str
  89
  90 util.initialize_logger()
  91
  92 # 14 is the length of DESCRIPTION=""
  93 max_desc_len = 100
  94 allowed_filename_chars="a-zA-Z0-9._-+:"
  95 pv_toolong_re = re.compile(r'[0-9]{19,}')
  96 bad = create_color_func("BAD")
  97
  98 # A sane umask is needed for files that portage creates.
  99 os.umask(0o22)
 100 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
 101 # behave incrementally.
 102 repoman_incrementals = tuple(x for x in \
 103         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 104 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 105 repoman_settings = portage.config(config_root=config_root, local_config=False)
 106
 107 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 108         repoman_settings.get('TERM') == 'dumb' or \
 109         not sys.stdout.isatty():
 110         nocolor()
 111
 112 def warn(txt):
 113         print("repoman: " + txt)
 114
 115 def err(txt):
 116         warn(txt)
 117         sys.exit(1)
 118
 119 def exithandler(signum=None, frame=None):
 120         logging.fatal("Interrupted; exiting...")
 121         if signum is None:
 122                 sys.exit(1)
 123         else:
 124                 sys.exit(128 + signum)
 125
 126 signal.signal(signal.SIGINT,exithandler)
 127
 128 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 129         """Repoman needs it's own HelpFormatter for now, because the default ones
 130         murder the help text."""
 131
 132         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 133                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 134
 135         def format_description(self, description):
 136                 return description
 137
 138 class RepomanOptionParser(optparse.OptionParser):
 139         """Add the on_tail function, ruby has it, optionParser should too
 140         """
 141
 142         def __init__(self, *args, **kwargs):
 143                 optparse.OptionParser.__init__(self, *args, **kwargs)
 144                 self.tail = ""
 145
 146         def on_tail(self, description):
 147                 self.tail += description
 148
 149         def format_help(self, formatter=None):
 150                 result = optparse.OptionParser.format_help(self, formatter)
 151                 result += self.tail
 152                 return result
 153
 154
 155 def ParseArgs(argv, qahelp):
 156         """This function uses a customized optionParser to parse command line arguments for repoman
 157         Args:
 158           argv - a sequence of command line arguments
 159                 qahelp - a dict of qa warning to help message
 160         Returns:
 161           (opts, args), just like a call to parser.parse_args()
 162         """
 163
 164         if argv and isinstance(argv[0], bytes):
 165                 argv = [portage._unicode_decode(x) for x in argv]
 166
 167         modes = {
 168                 'commit' : 'Run a scan then commit changes',
 169                 'ci' : 'Run a scan then commit changes',
 170                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 171                 'full' : 'Scan directory tree and print all issues (not a summary)',
 172                 'help' : 'Show this screen',
 173                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 174                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 175                 'scan' : 'Scan directory tree for QA issues'
 176         }
 177
 178         mode_keys = list(modes)
 179         mode_keys.sort()
 180
 181         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 182         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 183         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 184         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 185         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 186
 187         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 188                 help='Request a confirmation before commiting')
 189
 190         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 191                 help='specify a commit message on the command line')
 192
 193         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 194                 help='specify a path to a file that contains a commit message')
 195
 196         parser.add_option('--digest',
 197                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 198                 help='Automatically update Manifest digests for modified files')
 199
 200         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 201                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 202
 203         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 204                 help='do not print unnecessary messages')
 205
 206         parser.add_option(
 207                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 208                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 209                 'regardless of modification if \'force\' is specified)')
 210
 211         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 212                 help='Commit with QA violations')
 213
 214         parser.add_option('--vcs', dest='vcs',
 215                 help='Force using specific VCS instead of autodetection')
 216
 217         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 218                 help='be very verbose in output', default=0)
 219
 220         parser.add_option('-V', '--version', dest='version', action='store_true',
 221                 help='show version info')
 222
 223         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 224                 default=False, help='forces the metadata.xml parse check to be carried out')
 225
 226         parser.add_option(
 227                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 228                 metavar="<y|n>",
 229                 help='only check packages that have uncommitted modifications')
 230
 231         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 232                 default=False, help='ignore arch-specific failures (where arch != host)')
 233
 234         parser.add_option("--ignore-default-opts",
 235                 action="store_true",
 236                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 237
 238         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 239                 default=False, help='ignore masked packages (not allowed with commit mode)')
 240
 241         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 242                 default=False, help='include dev profiles in dependency checks')
 243
 244         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 245                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 246
 247         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 248                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 249
 250         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 251                 help='specify which mode repoman will run in (default=full)')
 252
 253         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 254
 255         for k in mode_keys:
 256                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 257
 258         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 259
 260         sorted_qa = list(qahelp)
 261         sorted_qa.sort()
 262         for k in sorted_qa:
 263                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 264
 265         opts, args = parser.parse_args(argv[1:])
 266
 267         if not opts.ignore_default_opts:
 268                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 269                 if default_opts:
 270                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 271
 272         if opts.mode == 'help':
 273                 parser.print_help(short=False)
 274
 275         for arg in args:
 276                 if arg in modes:
 277                         if not opts.mode:
 278                                 opts.mode = arg
 279                                 break
 280                 else:
 281                         parser.error("invalid mode: %s" % arg)
 282
 283         if not opts.mode:
 284                 opts.mode = 'full'
 285
 286         if opts.mode == 'ci':
 287                 opts.mode = 'commit'  # backwards compat shortcut
 288
 289         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 290                 if opts.ignore_masked:
 291                         parser.error('Commit mode and --ignore-masked are not compatible')
 292                 if opts.without_mask:
 293                         parser.error('Commit mode and --without-mask are not compatible')
 294
 295         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 296         for val in range(opts.verbosity):
 297                 logger = logging.getLogger()
 298                 logger.setLevel(logger.getEffectiveLevel() - 10)
 299
 300         for val in range(opts.quiet):
 301                 logger = logging.getLogger()
 302                 logger.setLevel(logger.getEffectiveLevel() + 10)
 303
 304         return (opts, args)
 305
 306 qahelp={
 307         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 308         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 309         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 310         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 311         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 312         "changelog.missing":"Missing ChangeLog files",
 313         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 314         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 315         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 316         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 317         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 318         "file.size":"Files in the files directory must be under 20 KiB",
 319         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 320         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 321         "file.UTF8":"File is not UTF8 compliant",
 322         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 323         "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it",
 324         "inherit.unused":"Ebuild inherits an eclass but does not use it",
 325         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 326         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 327         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 328         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 329         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 330         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 331         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 332         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 333         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 334         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 335         "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 336         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 337         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 338         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 339         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 340         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 341         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 342         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 343         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 344         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 345         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 346         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 347         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 348         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 349         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 350         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 351         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 352         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 353         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 354         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 355         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 356         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 357         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 358         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 359         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 360         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 361         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 362         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 363         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 364         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 365         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 366         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 367         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 368         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 369         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 370         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 371         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 372         "variable.readonly":"Assigning a readonly variable",
 373         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 374         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 375         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 376         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 377         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 378         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 379         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 380         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 381         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 382         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 383         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 384         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 385         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 386         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 387         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 388         "ebuild.badheader":"This ebuild has a malformed header",
 389         "manifest.bad":"Manifest has missing or incorrect digests",
 390         "metadata.missing":"Missing metadata.xml files",
 391         "metadata.bad":"Bad metadata.xml files",
 392         "metadata.warning":"Warnings in metadata.xml files",
 393         "portage.internal":"The ebuild uses an internal Portage function",
 394         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 395         "virtual.suspect":"Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 396         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 397         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 398 }
 399
 400 qacats = list(qahelp)
 401 qacats.sort()
 402
 403 qawarnings = set((
 404 "changelog.missing",
 405 "changelog.notadded",
 406 "dependency.unknown",
 407 "digest.assumed",
 408 "digest.unused",
 409 "ebuild.notadded",
 410 "ebuild.nesteddie",
 411 "desktop.invalid",
 412 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 413 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 414 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 415 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 416 "DESCRIPTION.toolong",
 417 "EAPI.deprecated",
 418 "HOMEPAGE.virtual",
 419 "LICENSE.virtual",
 420 "KEYWORDS.dropped",
 421 "KEYWORDS.stupid",
 422 "KEYWORDS.missing",
 423 "PDEPEND.suspect",
 424 "RDEPEND.implicit",
 425 "RDEPEND.suspect",
 426 "virtual.suspect",
 427 "RESTRICT.invalid",
 428 "ebuild.minorsyn",
 429 "ebuild.badheader",
 430 "ebuild.patches",
 431 "file.size",
 432 "inherit.unused",
 433 "inherit.deprecated",
 434 "java.eclassesnotused",
 435 "wxwidgets.eclassnotused",
 436 "metadata.warning",
 437 "portage.internal",
 438 "usage.obsolete",
 439 "upstream.workaround",
 440 "LIVEVCS.stable",
 441 "LIVEVCS.unmasked",
 442 ))
 443
 444 if portage.const._ENABLE_INHERIT_CHECK:
 445         # This is experimental, so it's non-fatal.
 446         qawarnings.add("inherit.missing")
 447
 448 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 449
 450 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 451 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 452 allvars.update(Package.metadata_keys)
 453 allvars = sorted(allvars)
 454 commitmessage=None
 455 for x in missingvars:
 456         x += ".missing"
 457         if x not in qacats:
 458                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 459                 qacats.append(x)
 460                 qawarnings.add(x)
 461
 462 valid_restrict = frozenset(["binchecks", "bindist",
 463         "fetch", "installsources", "mirror",
 464         "primaryuri", "strip", "test", "userpriv"])
 465
 466 live_eclasses = frozenset([
 467         "bzr",
 468         "cvs",
 469         "darcs",
 470         "git",
 471         "git-2",
 472         "mercurial",
 473         "subversion",
 474         "tla",
 475 ])
 476
 477 suspect_rdepend = frozenset([
 478         "app-arch/cabextract",
 479         "app-arch/rpm2targz",
 480         "app-doc/doxygen",
 481         "dev-lang/nasm",
 482         "dev-lang/swig",
 483         "dev-lang/yasm",
 484         "dev-perl/extutils-pkgconfig",
 485         "dev-util/byacc",
 486         "dev-util/cmake",
 487         "dev-util/ftjam",
 488         "dev-util/gperf",
 489         "dev-util/gtk-doc",
 490         "dev-util/gtk-doc-am",
 491         "dev-util/intltool",
 492         "dev-util/jam",
 493         "dev-util/pkg-config-lite",
 494         "dev-util/pkgconf",
 495         "dev-util/pkgconfig",
 496         "dev-util/pkgconfig-openbsd",
 497         "dev-util/scons",
 498         "dev-util/unifdef",
 499         "dev-util/yacc",
 500         "media-gfx/ebdftopcf",
 501         "sys-apps/help2man",
 502         "sys-devel/autoconf",
 503         "sys-devel/automake",
 504         "sys-devel/bin86",
 505         "sys-devel/bison",
 506         "sys-devel/dev86",
 507         "sys-devel/flex",
 508         "sys-devel/m4",
 509         "sys-devel/pmake",
 510         "virtual/linux-sources",
 511         "virtual/pkgconfig",
 512         "x11-misc/bdftopcf",
 513         "x11-misc/imake",
 514 ])
 515
 516 suspect_virtual = {
 517         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 518         "dev-util/pkgconf":"virtual/pkgconfig",
 519         "dev-util/pkgconfig":"virtual/pkgconfig",
 520         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 521         "dev-libs/libusb":"virtual/libusb",
 522         "dev-libs/libusbx":"virtual/libusb",
 523         "dev-libs/libusb-compat":"virtual/libusb",
 524 }
 525
 526 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 527 # force refetch if the local copy creation time is older than this
 528 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 529
 530 # file.executable
 531 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 532
 533 options, arguments = ParseArgs(sys.argv, qahelp)
 534
 535 if options.version:
 536         print("Portage", portage.VERSION)
 537         sys.exit(0)
 538
 539 # Set this to False when an extraordinary issue (generally
 540 # something other than a QA issue) makes it impossible to
 541 # commit (like if Manifest generation fails).
 542 can_force = True
 543
 544 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 545 if portdir is None:
 546         sys.exit(1)
 547
 548 myreporoot = os.path.basename(portdir_overlay)
 549 myreporoot += mydir[len(portdir_overlay):]
 550
 551 if options.vcs:
 552         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 553                 vcs = options.vcs
 554         else:
 555                 vcs = None
 556 else:
 557         vcses = utilities.FindVCS()
 558         if len(vcses) > 1:
 559                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 560                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 561                 sys.exit(1)
 562         elif vcses:
 563                 vcs = vcses[0]
 564         else:
 565                 vcs = None
 566
 567 if options.if_modified == "y" and vcs is None:
 568         logging.info("Not in a version controlled repository; "
 569                 "disabling --if-modified.")
 570         options.if_modified = "n"
 571
 572 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 573 vcs_preserves_mtime = vcs in ('cvs',)
 574
 575 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 576 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 577 if vcs_global_opts is None:
 578         if vcs in ('cvs', 'svn'):
 579                 vcs_global_opts = "-q"
 580         else:
 581                 vcs_global_opts = ""
 582 vcs_global_opts = vcs_global_opts.split()
 583
 584 if options.mode == 'commit' and not options.pretend and not vcs:
 585         logging.info("Not in a version controlled repository; enabling pretend mode.")
 586         options.pretend = True
 587
 588 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 589 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 590         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 591         portage._shell_quote(portdir_overlay))
 592 # We have to call the config constructor again so
 593 # that config.repositories is initialized correctly.
 594 repoman_settings = portage.config(config_root=config_root, local_config=False,
 595         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 596
 597 root = repoman_settings['EROOT']
 598 trees = {
 599         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 600 }
 601 portdb = trees[root]['porttree'].dbapi
 602
 603 # Constrain dependency resolution to the master(s)
 604 # that are specified in layout.conf.
 605 repodir = os.path.realpath(portdir_overlay)
 606 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 607 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 608 portdir = portdb.porttrees[0]
 609
 610 if repo_config.allow_provide_virtual:
 611         qawarnings.add("virtual.oldstyle")
 612
 613 if repo_config.sign_commit:
 614         if vcs == 'git':
 615                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 616                 # the commit arguments. If key_id is unspecified, then it must be
 617                 # configured by `git config user.signingkey key_id`.
 618                 vcs_local_opts.append("--gpg-sign")
 619
 620 # In order to disable manifest signatures, repos may set
 621 # "sign-manifests = false" in metadata/layout.conf. This
 622 # can be used to prevent merge conflicts like those that
 623 # thin-manifests is designed to prevent.
 624 sign_manifests = "sign" in repoman_settings.features and \
 625         repo_config.sign_manifest
 626
 627 manifest_hashes = repo_config.manifest_hashes
 628 if manifest_hashes is None:
 629         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 630
 631 if options.mode in ("commit", "fix", "manifest"):
 632         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 633                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 634                 "metadata/layout.conf does not contain the '%s' hash which "
 635                 "is required by this portage version. You will have to "
 636                 "upgrade portage if you want to generate valid manifests for "
 637                 "this repository.") % \
 638                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 639                 for line in textwrap.wrap(msg, 70):
 640                         logging.error(line)
 641                 sys.exit(1)
 642
 643         unsupported_hashes = manifest_hashes.difference(
 644                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 645         if unsupported_hashes:
 646                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 647                 "metadata/layout.conf contains one or more hash types '%s' "
 648                 "which are not supported by this portage version. You will "
 649                 "have to upgrade portage if you want to generate valid "
 650                 "manifests for this repository.") % \
 651                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 652                 for line in textwrap.wrap(msg, 70):
 653                         logging.error(line)
 654                 sys.exit(1)
 655
 656 if "commit" == options.mode and \
 657         repo_config.name == "gentoo" and \
 658         "RMD160" in manifest_hashes and \
 659         "RMD160" not in portage.checksum.hashorigin_map:
 660         msg = "Please install " \
 661         "pycrypto or enable python's ssl USE flag in order " \
 662         "to enable RMD160 hash support. See bug #198398 for " \
 663         "more information."
 664         prefix = bad(" * ")
 665         for line in textwrap.wrap(msg, 70):
 666                 print(prefix + line)
 667         sys.exit(1)
 668
 669 if options.echangelog is None and repo_config.update_changelog:
 670         options.echangelog = 'y'
 671
 672 if vcs is None:
 673         options.echangelog = 'n'
 674
 675 # The --echangelog option causes automatic ChangeLog generation,
 676 # which invalidates changelog.ebuildadded and changelog.missing
 677 # checks.
 678 # Note: Some don't use ChangeLogs in distributed SCMs.
 679 # It will be generated on server side from scm log,
 680 # before package moves to the rsync server.
 681 # This is needed because they try to avoid merge collisions.
 682 # Gentoo's Council decided to always use the ChangeLog file.
 683 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 684 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 685
 686 if 'digest' in repoman_settings.features and options.digest != 'n':
 687         options.digest = 'y'
 688
 689 logging.debug("vcs: %s" % (vcs,))
 690 logging.debug("repo config: %s" % (repo_config,))
 691 logging.debug("options: %s" % (options,))
 692
 693 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 694 # profile-specific config constructor calls.
 695 env = os.environ.copy()
 696 env['PORTDIR'] = portdir
 697 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 698
 699 logging.info('Setting paths:')
 700 logging.info('PORTDIR = "' + portdir + '"')
 701 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 702
 703 # It's confusing if these warnings are displayed without the user
 704 # being told which profile they come from, so disable them.
 705 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 706
 707 categories = []
 708 for path in repo_config.eclass_db.porttrees:
 709         categories.extend(portage.util.grabfile(
 710                 os.path.join(path, 'profiles', 'categories')))
 711 repoman_settings.categories = frozenset(
 712         portage.util.stack_lists([categories], incremental=1))
 713 categories = repoman_settings.categories
 714
 715 portdb.settings = repoman_settings
 716 root_config = RootConfig(repoman_settings, trees[root], None)
 717 # We really only need to cache the metadata that's necessary for visibility
 718 # filtering. Anything else can be discarded to reduce memory consumption.
 719 portdb._aux_cache_keys.clear()
 720 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 721
 722 reposplit = myreporoot.split(os.path.sep)
 723 repolevel = len(reposplit)
 724
 725 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 726 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 727 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 728 if options.mode == 'commit' and repolevel not in [1,2,3]:
 729         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 730         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 731         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 732         print(red("***"))
 733         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 734
 735 # Make startdir relative to the canonical repodir, so that we can pass
 736 # it to digestgen and it won't have to be canonicalized again.
 737 if repolevel == 1:
 738         startdir = repodir
 739 else:
 740         startdir = normalize_path(mydir)
 741         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 742
 743 def caterror(mycat):
 744         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 745
 746 class ProfileDesc(object):
 747         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 748         def __init__(self, arch, status, sub_path, tree_path):
 749                 self.arch = arch
 750                 self.status = status
 751                 if sub_path:
 752                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 753                 self.sub_path = sub_path
 754                 self.tree_path = tree_path
 755                 if tree_path:
 756                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 757                 else:
 758                         self.abs_path = tree_path
 759
 760         def __str__(self):
 761                 if self.sub_path:
 762                         return self.sub_path
 763                 return 'empty profile'
 764
 765 profile_list = []
 766 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 767
 768 # get lists of valid keywords, licenses, and use
 769 kwlist = set()
 770 liclist = set()
 771 uselist = set()
 772 global_pmasklines = []
 773
 774 for path in portdb.porttrees:
 775         try:
 776                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 777         except OSError:
 778                 pass
 779         kwlist.update(portage.grabfile(os.path.join(path,
 780                 "profiles", "arch.list")))
 781
 782         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 783         for x in use_desc:
 784                 x = x.split()
 785                 if x:
 786                         uselist.add(x[0])
 787
 788         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 789         try:
 790                 expand_list = os.listdir(expand_desc_dir)
 791         except OSError:
 792                 pass
 793         else:
 794                 for fn in expand_list:
 795                         if not fn[-5:] == '.desc':
 796                                 continue
 797                         use_prefix = fn[:-5].lower() + '_'
 798                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 799                                 x = x.split()
 800                                 if x:
 801                                         uselist.add(use_prefix + x[0])
 802
 803         global_pmasklines.append(portage.util.grabfile_package(
 804                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 805
 806         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 807         try:
 808                 desc_file = io.open(_unicode_encode(desc_path,
 809                         encoding=_encodings['fs'], errors='strict'),
 810                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 811         except EnvironmentError:
 812                 pass
 813         else:
 814                 for i, x in enumerate(desc_file):
 815                         if x[0] == "#":
 816                                 continue
 817                         arch = x.split()
 818                         if len(arch) == 0:
 819                                 continue
 820                         if len(arch) != 3:
 821                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 822                                         desc_path + " line %d" % (i+1, ))
 823                         elif arch[0] not in kwlist:
 824                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 825                                         desc_path + " line %d" % (i+1, ))
 826                         elif arch[2] not in valid_profile_types:
 827                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 828                                         desc_path + " line %d" % (i+1, ))
 829                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 830                         if not os.path.isdir(profile_desc.abs_path):
 831                                 logging.error(
 832                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 833                                         arch[2], arch[1], arch[0], desc_path, i+1)
 834                                 continue
 835                         if os.path.exists(
 836                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 837                                 continue
 838                         profile_list.append(profile_desc)
 839                 desc_file.close()
 840
 841 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 842 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 843
 844 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 845 global_pmaskdict = {}
 846 for x in global_pmasklines:
 847         global_pmaskdict.setdefault(x.cp, []).append(x)
 848 del global_pmasklines
 849
 850 def has_global_mask(pkg):
 851         mask_atoms = global_pmaskdict.get(pkg.cp)
 852         if mask_atoms:
 853                 pkg_list = [pkg]
 854                 for x in mask_atoms:
 855                         if portage.dep.match_from_list(x, pkg_list):
 856                                 return x
 857         return None
 858
 859 # Ensure that profile sub_path attributes are unique. Process in reverse order
 860 # so that profiles with duplicate sub_path from overlays will override
 861 # profiles with the same sub_path from parent repos.
 862 profiles = {}
 863 profile_list.reverse()
 864 profile_sub_paths = set()
 865 for prof in profile_list:
 866         if prof.sub_path in profile_sub_paths:
 867                 continue
 868         profile_sub_paths.add(prof.sub_path)
 869         profiles.setdefault(prof.arch, []).append(prof)
 870
 871 # Use an empty profile for checking dependencies of
 872 # packages that have empty KEYWORDS.
 873 prof = ProfileDesc('**', 'stable', '', '')
 874 profiles.setdefault(prof.arch, []).append(prof)
 875
 876 for x in repoman_settings.archlist():
 877         if x[0] == "~":
 878                 continue
 879         if x not in profiles:
 880                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 881                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 882                 print(red("up with the "+x+" team."))
 883                 print()
 884
 885 if not liclist:
 886         logging.fatal("Couldn't find licenses?")
 887         sys.exit(1)
 888
 889 if not kwlist:
 890         logging.fatal("Couldn't read KEYWORDS from arch.list")
 891         sys.exit(1)
 892
 893 if not uselist:
 894         logging.fatal("Couldn't find use.desc?")
 895         sys.exit(1)
 896
 897 scanlist=[]
 898 if repolevel==2:
 899         #we are inside a category directory
 900         catdir=reposplit[-1]
 901         if catdir not in categories:
 902                 caterror(catdir)
 903         mydirlist=os.listdir(startdir)
 904         for x in mydirlist:
 905                 if x == "CVS" or x.startswith("."):
 906                         continue
 907                 if os.path.isdir(startdir+"/"+x):
 908                         scanlist.append(catdir+"/"+x)
 909         repo_subdir = catdir + os.sep
 910 elif repolevel==1:
 911         for x in categories:
 912                 if not os.path.isdir(startdir+"/"+x):
 913                         continue
 914                 for y in os.listdir(startdir+"/"+x):
 915                         if y == "CVS" or y.startswith("."):
 916                                 continue
 917                         if os.path.isdir(startdir+"/"+x+"/"+y):
 918                                 scanlist.append(x+"/"+y)
 919         repo_subdir = ""
 920 elif repolevel==3:
 921         catdir = reposplit[-2]
 922         if catdir not in categories:
 923                 caterror(catdir)
 924         scanlist.append(catdir+"/"+reposplit[-1])
 925         repo_subdir = scanlist[-1] + os.sep
 926 else:
 927         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 928                 ' from the current working directory'
 929         logging.critical(msg)
 930         sys.exit(1)
 931
 932 repo_subdir_len = len(repo_subdir)
 933 scanlist.sort()
 934
 935 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 936
 937 def vcs_files_to_cps(vcs_file_iter):
 938         """
 939         Iterate over the given modified file paths returned from the vcs,
 940         and return a frozenset containing category/pn strings for each
 941         modified package.
 942         """
 943
 944         modified_cps = []
 945
 946         if repolevel == 3:
 947                 if reposplit[-2] in categories and \
 948                         next(vcs_file_iter, None) is not None:
 949                         modified_cps.append("/".join(reposplit[-2:]))
 950
 951         elif repolevel == 2:
 952                 category = reposplit[-1]
 953                 if category in categories:
 954                         for filename in vcs_file_iter:
 955                                 f_split = filename.split(os.sep)
 956                                 # ['.', pn,...]
 957                                 if len(f_split) > 2:
 958                                         modified_cps.append(category + "/" + f_split[1])
 959
 960         else:
 961                 # repolevel == 1
 962                 for filename in vcs_file_iter:
 963                         f_split = filename.split(os.sep)
 964                         # ['.', category, pn,...]
 965                         if len(f_split) > 3 and f_split[1] in categories:
 966                                 modified_cps.append("/".join(f_split[1:3]))
 967
 968         return frozenset(modified_cps)
 969
 970 def git_supports_gpg_sign():
 971         status, cmd_output = \
 972                 subprocess_getstatusoutput("git --version")
 973         cmd_output = cmd_output.split()
 974         if cmd_output:
 975                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
 976                 if version is not None:
 977                         version = [int(x) for x in version.groups()[1:]]
 978                         if version[0] > 1 or \
 979                                 (version[0] == 1 and version[1] > 7) or \
 980                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
 981                                 return True
 982         return False
 983
 984 def dev_keywords(profiles):
 985         """
 986         Create a set of KEYWORDS values that exist in 'dev'
 987         profiles. These are used
 988         to trigger a message notifying the user when they might
 989         want to add the --include-dev option.
 990         """
 991         type_arch_map = {}
 992         for arch, arch_profiles in profiles.items():
 993                 for prof in arch_profiles:
 994                         arch_set = type_arch_map.get(prof.status)
 995                         if arch_set is None:
 996                                 arch_set = set()
 997                                 type_arch_map[prof.status] = arch_set
 998                         arch_set.add(arch)
 999
1000         dev_keywords = type_arch_map.get('dev', set())
1001         dev_keywords.update(['~' + arch for arch in dev_keywords])
1002         return frozenset(dev_keywords)
1003
1004 dev_keywords = dev_keywords(profiles)
1005
1006 stats={}
1007 fails={}
1008
1009 # provided by the desktop-file-utils package
1010 desktop_file_validate = find_binary("desktop-file-validate")
1011 desktop_pattern = re.compile(r'.*\.desktop$')
1012
1013 for x in qacats:
1014         stats[x]=0
1015         fails[x]=[]
1016
1017 xmllint_capable = False
1018 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1019
1020 def parsedate(s):
1021         """Parse a RFC 822 date and time string.
1022         This is required for python3 compatibility, since the
1023         rfc822.parsedate() function is not available."""
1024
1025         s_split = []
1026         for x in s.upper().split():
1027                 for y in x.split(','):
1028                         if y:
1029                                 s_split.append(y)
1030
1031         if len(s_split) != 6:
1032                 return None
1033
1034         # %a, %d %b %Y %H:%M:%S %Z
1035         a, d, b, Y, H_M_S, Z = s_split
1036
1037         # Convert month to integer, since strptime %w is locale-dependent.
1038         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1039                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1040         m = month_map.get(b)
1041         if m is None:
1042                 return None
1043         m = str(m).rjust(2, '0')
1044
1045         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1046
1047 def fetch_metadata_dtd():
1048         """
1049         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1050         metadata_dtd_ctime_interval.
1051         @rtype: bool
1052         @return: True if successful, otherwise False
1053         """
1054
1055         must_fetch = True
1056         metadata_dtd_st = None
1057         current_time = int(time.time())
1058         try:
1059                 metadata_dtd_st = os.stat(metadata_dtd)
1060         except EnvironmentError as e:
1061                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1062                         raise
1063                 del e
1064         else:
1065                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1066                 if abs(current_time - metadata_dtd_st.st_ctime) \
1067                         < metadata_dtd_ctime_interval:
1068                         must_fetch = False
1069
1070         if must_fetch:
1071                 print()
1072                 print(green("***") + " the local copy of metadata.dtd " + \
1073                         "needs to be refetched, doing that now")
1074                 print()
1075                 try:
1076                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1077                         msg_info = url_f.info()
1078                         last_modified = msg_info.get('last-modified')
1079                         if last_modified is not None:
1080                                 last_modified = parsedate(last_modified)
1081                                 if last_modified is not None:
1082                                         last_modified = calendar.timegm(last_modified)
1083
1084                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1085                         try:
1086                                 local_f = open(metadata_dtd_tmp, mode='wb')
1087                                 local_f.write(url_f.read())
1088                                 local_f.close()
1089                                 if last_modified is not None:
1090                                         try:
1091                                                 os.utime(metadata_dtd_tmp,
1092                                                         (int(last_modified), int(last_modified)))
1093                                         except OSError:
1094                                                 # This fails on some odd non-unix-like filesystems.
1095                                                 # We don't really need the mtime to be preserved
1096                                                 # anyway here (currently we use ctime to trigger
1097                                                 # fetch), so just ignore it.
1098                                                 pass
1099                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1100                         finally:
1101                                 try:
1102                                         os.unlink(metadata_dtd_tmp)
1103                                 except OSError:
1104                                         pass
1105
1106                         url_f.close()
1107
1108                 except EnvironmentError as e:
1109                         print()
1110                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1111                         print(red("!!!")+" exception '%s' though." % (e,))
1112                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1113                         return False
1114
1115         return True
1116
1117 if options.mode == "manifest":
1118         pass
1119 elif not find_binary('xmllint'):
1120         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1121         if options.xml_parse or repolevel==3:
1122                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1123                 sys.exit(1)
1124 else:
1125         if not fetch_metadata_dtd():
1126                 sys.exit(1)
1127         #this can be problematic if xmllint changes their output
1128         xmllint_capable=True
1129
1130 if options.mode == 'commit' and vcs:
1131         utilities.detect_vcs_conflicts(options, vcs)
1132
1133 if options.mode == "manifest":
1134         pass
1135 elif options.pretend:
1136         print(green("\nRepoMan does a once-over of the neighborhood..."))
1137 else:
1138         print(green("\nRepoMan scours the neighborhood..."))
1139
1140 new_ebuilds = set()
1141 modified_ebuilds = set()
1142 modified_changelogs = set()
1143 mychanged = []
1144 mynew = []
1145 myremoved = []
1146
1147 if vcs == "cvs":
1148         mycvstree = cvstree.getentries("./", recursive=1)
1149         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1150         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1151         if options.if_modified == "y":
1152                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1153
1154 elif vcs == "svn":
1155         with os.popen("svn status") as f:
1156                 svnstatus = f.readlines()
1157         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1158         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1159         if options.if_modified == "y":
1160                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1161
1162 elif vcs == "git":
1163         with os.popen("git diff-index --name-only "
1164                 "--relative --diff-filter=M HEAD") as f:
1165                 mychanged = f.readlines()
1166         mychanged = ["./" + elem[:-1] for elem in mychanged]
1167
1168         with os.popen("git diff-index --name-only "
1169                 "--relative --diff-filter=A HEAD") as f:
1170                 mynew = f.readlines()
1171         mynew = ["./" + elem[:-1] for elem in mynew]
1172         if options.if_modified == "y":
1173                 with os.popen("git diff-index --name-only "
1174                         "--relative --diff-filter=D HEAD") as f:
1175                         myremoved = f.readlines()
1176                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1177
1178 elif vcs == "bzr":
1179         with os.popen("bzr status -S .") as f:
1180                 bzrstatus = f.readlines()
1181         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1182         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1183         if options.if_modified == "y":
1184                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1185
1186 elif vcs == "hg":
1187         with os.popen("hg status --no-status --modified .") as f:
1188                 mychanged = f.readlines()
1189         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1190         mynew = os.popen("hg status --no-status --added .").readlines()
1191         mynew = ["./" + elem.rstrip() for elem in mynew]
1192         if options.if_modified == "y":
1193                 with os.popen("hg status --no-status --removed .") as f:
1194                         myremoved = f.readlines()
1195                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1196
1197 if vcs:
1198         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1199         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1200         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1201                 if os.path.basename(x) == "ChangeLog")
1202
1203 def vcs_new_changed(relative_path):
1204         for x in chain(mychanged, mynew):
1205                 if x == relative_path:
1206                         return True
1207         return False
1208
1209 have_pmasked = False
1210 have_dev_keywords = False
1211 dofail = 0
1212
1213 # NOTE: match-all caches are not shared due to potential
1214 # differences between profiles in _get_implicit_iuse.
1215 arch_caches={}
1216 arch_xmatch_caches = {}
1217 shared_xmatch_caches = {"cp-list":{}}
1218
1219 # Disable the "ebuild.notadded" check when not in commit mode and
1220 # running `svn status` in every package dir will be too expensive.
1221
1222 check_ebuild_notadded = not \
1223         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1224
1225 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1226 thirdpartymirrors = []
1227 for v in repoman_settings.thirdpartymirrors().values():
1228         for v in v:
1229                 if not v.endswith("/"):
1230                         v += "/"
1231                 thirdpartymirrors.append(v)
1232
1233 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1234         """
1235         Implements doctype() as required to avoid deprecation warnings with
1236         >=python-2.7.
1237         """
1238         def doctype(self, name, pubid, system):
1239                 pass
1240
1241 try:
1242         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1243 except (EnvironmentError, ParseError, PermissionDenied) as e:
1244         err(str(e))
1245 except FileNotFound:
1246         # TODO: Download as we do for metadata.dtd, but add a way to
1247         # disable for non-gentoo repoman users who may not have herds.
1248         herd_base = None
1249
1250 effective_scanlist = scanlist
1251 if options.if_modified == "y":
1252         effective_scanlist = sorted(vcs_files_to_cps(
1253                 chain(mychanged, mynew, myremoved)))
1254
1255 for x in effective_scanlist:
1256         #ebuilds and digests added to cvs respectively.
1257         logging.info("checking package %s" % x)
1258         # save memory by discarding xmatch caches from previous package(s)
1259         arch_xmatch_caches.clear()
1260         eadded=[]
1261         catdir,pkgdir=x.split("/")
1262         checkdir=repodir+"/"+x
1263         checkdir_relative = ""
1264         if repolevel < 3:
1265                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1266         if repolevel < 2:
1267                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1268         checkdir_relative = os.path.join(".", checkdir_relative)
1269         generated_manifest = False
1270
1271         if options.mode == "manifest" or \
1272           (options.mode != 'manifest-check' and options.digest == 'y') or \
1273           options.mode in ('commit', 'fix') and not options.pretend:
1274                 auto_assumed = set()
1275                 fetchlist_dict = portage.FetchlistDict(checkdir,
1276                         repoman_settings, portdb)
1277                 if options.mode == 'manifest' and options.force:
1278                         portage._doebuild_manifest_exempt_depend += 1
1279                         try:
1280                                 distdir = repoman_settings['DISTDIR']
1281                                 mf = repoman_settings.repositories.get_repo_for_location(
1282                                         os.path.dirname(os.path.dirname(checkdir)))
1283                                 mf = mf.load_manifest(checkdir, distdir,
1284                                         fetchlist_dict=fetchlist_dict)
1285                                 mf.create(requiredDistfiles=None,
1286                                         assumeDistHashesAlways=True)
1287                                 for distfiles in fetchlist_dict.values():
1288                                         for distfile in distfiles:
1289                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1290                                                         mf.fhashdict['DIST'].pop(distfile, None)
1291                                                 else:
1292                                                         auto_assumed.add(distfile)
1293                                 mf.write()
1294                         finally:
1295                                 portage._doebuild_manifest_exempt_depend -= 1
1296
1297                 repoman_settings["O"] = checkdir
1298                 try:
1299                         generated_manifest = digestgen(
1300                                 mysettings=repoman_settings, myportdb=portdb)
1301                 except portage.exception.PermissionDenied as e:
1302                         generated_manifest = False
1303                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1304                                 level=logging.ERROR, noiselevel=-1)
1305
1306                 if not generated_manifest:
1307                         print("Unable to generate manifest.")
1308                         dofail = 1
1309
1310                 if options.mode == "manifest":
1311                         if not dofail and options.force and auto_assumed and \
1312                                 'assume-digests' in repoman_settings.features:
1313                                 # Show which digests were assumed despite the --force option
1314                                 # being given. This output will already have been shown by
1315                                 # digestgen() if assume-digests is not enabled, so only show
1316                                 # it here if assume-digests is enabled.
1317                                 pkgs = list(fetchlist_dict)
1318                                 pkgs.sort()
1319                                 portage.writemsg_stdout("  digest.assumed" + \
1320                                         portage.output.colorize("WARN",
1321                                         str(len(auto_assumed)).rjust(18)) + "\n")
1322                                 for cpv in pkgs:
1323                                         fetchmap = fetchlist_dict[cpv]
1324                                         pf = portage.catsplit(cpv)[1]
1325                                         for distfile in sorted(fetchmap):
1326                                                 if distfile in auto_assumed:
1327                                                         portage.writemsg_stdout(
1328                                                                 "   %s::%s\n" % (pf, distfile))
1329                         continue
1330                 elif dofail:
1331                         sys.exit(1)
1332
1333         if not generated_manifest:
1334                 repoman_settings['O'] = checkdir
1335                 repoman_settings['PORTAGE_QUIET'] = '1'
1336                 if not portage.digestcheck([], repoman_settings, strict=1):
1337                         stats["manifest.bad"] += 1
1338                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1339                 repoman_settings.pop('PORTAGE_QUIET', None)
1340
1341         if options.mode == 'manifest-check':
1342                 continue
1343
1344         checkdirlist=os.listdir(checkdir)
1345         ebuildlist=[]
1346         pkgs = {}
1347         allvalid = True
1348         for y in checkdirlist:
1349                 if (y in no_exec or y.endswith(".ebuild")) and \
1350                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1351                                 stats["file.executable"] += 1
1352                                 fails["file.executable"].append(os.path.join(checkdir, y))
1353                 if y.endswith(".ebuild"):
1354                         pf = y[:-7]
1355                         ebuildlist.append(pf)
1356                         cpv = "%s/%s" % (catdir, pf)
1357                         try:
1358                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1359                         except KeyError:
1360                                 allvalid = False
1361                                 stats["ebuild.syntax"] += 1
1362                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1363                                 continue
1364                         except IOError:
1365                                 allvalid = False
1366                                 stats["ebuild.output"] += 1
1367                                 fails["ebuild.output"].append(os.path.join(x, y))
1368                                 continue
1369                         if not portage.eapi_is_supported(myaux["EAPI"]):
1370                                 allvalid = False
1371                                 stats["EAPI.unsupported"] += 1
1372                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1373                                 continue
1374                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1375                                 root_config=root_config, type_name="ebuild")
1376
1377         slot_keywords = {}
1378
1379         if len(pkgs) != len(ebuildlist):
1380                 # If we can't access all the metadata then it's totally unsafe to
1381                 # commit since there's no way to generate a correct Manifest.
1382                 # Do not try to do any more QA checks on this package since missing
1383                 # metadata leads to false positives for several checks, and false
1384                 # positives confuse users.
1385                 can_force = False
1386                 continue
1387
1388         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1389         ebuildlist = sorted(pkgs.values())
1390         ebuildlist = [pkg.pf for pkg in ebuildlist]
1391
1392         for y in checkdirlist:
1393                 m = disallowed_filename_chars_re.search(y.strip(os.sep))
1394                 if m is not None:
1395                         y_relative = os.path.join(checkdir_relative, y)
1396                         if vcs is not None and not vcs_new_changed(y_relative):
1397                                 # If the file isn't in the VCS new or changed set, then
1398                                 # assume that it's an irrelevant temporary file (Manifest
1399                                 # entries are not generated for file names containing
1400                                 # prohibited characters). See bug #406877.
1401                                 m = None
1402                 if m is not None:
1403                         stats["file.name"] += 1
1404                         fails["file.name"].append("%s/%s: char '%s'" % \
1405                                 (checkdir, y, m.group(0)))
1406
1407                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1408                         continue
1409                 f = None
1410                 try:
1411                         line = 1
1412                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1413                                 encoding=_encodings['fs'], errors='strict'),
1414                                 mode='r', encoding=_encodings['repo.content'])
1415                         for l in f:
1416                                 line +=1
1417                 except UnicodeDecodeError as ue:
1418                         stats["file.UTF8"] += 1
1419                         s = ue.object[:ue.start]
1420                         l2 = s.count("\n")
1421                         line += l2
1422                         if l2 != 0:
1423                                 s = s[s.rfind("\n") + 1:]
1424                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1425                 finally:
1426                         if f is not None:
1427                                 f.close()
1428
1429         if vcs in ("git", "hg") and check_ebuild_notadded:
1430                 if vcs == "git":
1431                         myf = os.popen("git ls-files --others %s" % \
1432                                 (portage._shell_quote(checkdir_relative),))
1433                 if vcs == "hg":
1434                         myf = os.popen("hg status --no-status --unknown %s" % \
1435                                 (portage._shell_quote(checkdir_relative),))
1436                 for l in myf:
1437                         if l[:-1][-7:] == ".ebuild":
1438                                 stats["ebuild.notadded"] += 1
1439                                 fails["ebuild.notadded"].append(
1440                                         os.path.join(x, os.path.basename(l[:-1])))
1441                 myf.close()
1442
1443         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1444                 try:
1445                         if vcs == "cvs":
1446                                 myf=open(checkdir+"/CVS/Entries","r")
1447                         if vcs == "svn":
1448                                 myf = os.popen("svn status --depth=files --verbose " + checkdir)
1449                         if vcs == "bzr":
1450                                 myf = os.popen("bzr ls -v --kind=file " + checkdir)
1451                         myl = myf.readlines()
1452                         myf.close()
1453                         for l in myl:
1454                                 if vcs == "cvs":
1455                                         if l[0]!="/":
1456                                                 continue
1457                                         splitl=l[1:].split("/")
1458                                         if not len(splitl):
1459                                                 continue
1460                                         if splitl[0][-7:]==".ebuild":
1461                                                 eadded.append(splitl[0][:-7])
1462                                 if vcs == "svn":
1463                                         if l[:1] == "?":
1464                                                 continue
1465                                         if l[:7] == '      >':
1466                                                 # tree conflict, new in subversion 1.6
1467                                                 continue
1468                                         l = l.split()[-1]
1469                                         if l[-7:] == ".ebuild":
1470                                                 eadded.append(os.path.basename(l[:-7]))
1471                                 if vcs == "bzr":
1472                                         if l[1:2] == "?":
1473                                                 continue
1474                                         l = l.split()[-1]
1475                                         if l[-7:] == ".ebuild":
1476                                                 eadded.append(os.path.basename(l[:-7]))
1477                         if vcs == "svn":
1478                                 myf = os.popen("svn status " + checkdir)
1479                                 myl=myf.readlines()
1480                                 myf.close()
1481                                 for l in myl:
1482                                         if l[0] == "A":
1483                                                 l = l.rstrip().split(' ')[-1]
1484                                                 if l[-7:] == ".ebuild":
1485                                                         eadded.append(os.path.basename(l[:-7]))
1486                 except IOError:
1487                         if vcs == "cvs":
1488                                 stats["CVS/Entries.IO_error"] += 1
1489                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1490                         else:
1491                                 raise
1492                         continue
1493
1494         mf = repoman_settings.repositories.get_repo_for_location(
1495                 os.path.dirname(os.path.dirname(checkdir)))
1496         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1497         mydigests=mf.getTypeDigests("DIST")
1498
1499         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1500         myfiles_all = []
1501         src_uri_error = False
1502         for mykey in fetchlist_dict:
1503                 try:
1504                         myfiles_all.extend(fetchlist_dict[mykey])
1505                 except portage.exception.InvalidDependString as e:
1506                         src_uri_error = True
1507                         try:
1508                                 portdb.aux_get(mykey, ["SRC_URI"])
1509                         except KeyError:
1510                                 # This will be reported as an "ebuild.syntax" error.
1511                                 pass
1512                         else:
1513                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1514                                 fails["SRC_URI.syntax"].append(
1515                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1516         del fetchlist_dict
1517         if not src_uri_error:
1518                 # This test can produce false positives if SRC_URI could not
1519                 # be parsed for one or more ebuilds. There's no point in
1520                 # producing a false error here since the root cause will
1521                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1522                 # or "ebuild.sytax".
1523                 myfiles_all = set(myfiles_all)
1524                 for entry in mydigests:
1525                         if entry not in myfiles_all:
1526                                 stats["digest.unused"] += 1
1527                                 fails["digest.unused"].append(checkdir+"::"+entry)
1528                 for entry in myfiles_all:
1529                         if entry not in mydigests:
1530                                 stats["digest.missing"] += 1
1531                                 fails["digest.missing"].append(checkdir+"::"+entry)
1532         del myfiles_all
1533
1534         if os.path.exists(checkdir+"/files"):
1535                 filesdirlist=os.listdir(checkdir+"/files")
1536
1537                 # recurse through files directory
1538                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1539                 while filesdirlist:
1540                         y = filesdirlist.pop(0)
1541                         relative_path = os.path.join(x, "files", y)
1542                         full_path = os.path.join(repodir, relative_path)
1543                         try:
1544                                 mystat = os.stat(full_path)
1545                         except OSError as oe:
1546                                 if oe.errno == 2:
1547                                         # don't worry about it.  it likely was removed via fix above.
1548                                         continue
1549                                 else:
1550                                         raise oe
1551                         if S_ISDIR(mystat.st_mode):
1552                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1553                                 if y == "CVS" or y == ".svn":
1554                                         continue
1555                                 for z in os.listdir(checkdir+"/files/"+y):
1556                                         if z == "CVS" or z == ".svn":
1557                                                 continue
1558                                         filesdirlist.append(y+"/"+z)
1559                         # Current policy is no files over 20 KiB, these are the checks. File size between
1560                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1561                         elif mystat.st_size > 61440:
1562                                 stats["file.size.fatal"] += 1
1563                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1564                         elif mystat.st_size > 20480:
1565                                 stats["file.size"] += 1
1566                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1567
1568                         m = disallowed_filename_chars_re.search(
1569                                 os.path.basename(y.rstrip(os.sep)))
1570                         if m is not None:
1571                                 y_relative = os.path.join(checkdir_relative, "files", y)
1572                                 if vcs is not None and not vcs_new_changed(y_relative):
1573                                         # If the file isn't in the VCS new or changed set, then
1574                                         # assume that it's an irrelevant temporary file (Manifest
1575                                         # entries are not generated for file names containing
1576                                         # prohibited characters). See bug #406877.
1577                                         m = None
1578                         if m is not None:
1579                                 stats["file.name"] += 1
1580                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1581                                         (checkdir, y, m.group(0)))
1582
1583                         if desktop_file_validate and desktop_pattern.match(y):
1584                                 cmd_output = validate_desktop_entry(full_path)
1585                                 if cmd_output:
1586                                         # Note: in the future we may want to grab the
1587                                         # warnings in addition to the errors. We're
1588                                         # just doing errors now since we don't want
1589                                         # to generate too much noise at first.
1590                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1591                                         for line in cmd_output:
1592                                                 error_match = error_re.match(line)
1593                                                 if error_match is None:
1594                                                         continue
1595                                                 stats["desktop.invalid"] += 1
1596                                                 fails["desktop.invalid"].append(
1597                                                         relative_path + ': %s' % error_match.group(1))
1598
1599         del mydigests
1600
1601         if check_changelog and "ChangeLog" not in checkdirlist:
1602                 stats["changelog.missing"]+=1
1603                 fails["changelog.missing"].append(x+"/ChangeLog")
1604
1605         musedict = {}
1606         #metadata.xml file check
1607         if "metadata.xml" not in checkdirlist:
1608                 stats["metadata.missing"]+=1
1609                 fails["metadata.missing"].append(x+"/metadata.xml")
1610         #metadata.xml parse check
1611         else:
1612                 metadata_bad = False
1613
1614                 # read metadata.xml into memory
1615                 try:
1616                         _metadata_xml = xml.etree.ElementTree.parse(
1617                                 os.path.join(checkdir, "metadata.xml"),
1618                                 parser=xml.etree.ElementTree.XMLParser(
1619                                         target=_MetadataTreeBuilder()))
1620                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1621                         metadata_bad = True
1622                         stats["metadata.bad"] += 1
1623                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1624                         del e
1625                 else:
1626                         # load USE flags from metadata.xml
1627                         try:
1628                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1629                         except portage.exception.ParseError as e:
1630                                 metadata_bad = True
1631                                 stats["metadata.bad"] += 1
1632                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1633
1634                         # Run other metadata.xml checkers
1635                         try:
1636                                 utilities.check_metadata(_metadata_xml, herd_base)
1637                         except (utilities.UnknownHerdsError, ) as e:
1638                                 metadata_bad = True
1639                                 stats["metadata.bad"] += 1
1640                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1641                                 del e
1642
1643                 #Only carry out if in package directory or check forced
1644                 if xmllint_capable and not metadata_bad:
1645                         # xmlint can produce garbage output even on success, so only dump
1646                         # the ouput when it fails.
1647                         st, out = subprocess_getstatusoutput(
1648                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1649                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1650                         if st != os.EX_OK:
1651                                 print(red("!!!") + " metadata.xml is invalid:")
1652                                 for z in out.splitlines():
1653                                         print(red("!!! ")+z)
1654                                 stats["metadata.bad"]+=1
1655                                 fails["metadata.bad"].append(x+"/metadata.xml")
1656
1657                 del metadata_bad
1658         muselist = frozenset(musedict)
1659
1660         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1661         changelog_modified = changelog_path in modified_changelogs
1662
1663         # detect unused local USE-descriptions
1664         used_useflags = set()
1665
1666         for y in ebuildlist:
1667                 relative_path = os.path.join(x, y + ".ebuild")
1668                 full_path = os.path.join(repodir, relative_path)
1669                 ebuild_path = y + ".ebuild"
1670                 if repolevel < 3:
1671                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1672                 if repolevel < 2:
1673                         ebuild_path = os.path.join(catdir, ebuild_path)
1674                 ebuild_path = os.path.join(".", ebuild_path)
1675                 if check_changelog and not changelog_modified \
1676                         and ebuild_path in new_ebuilds:
1677                         stats['changelog.ebuildadded'] += 1
1678                         fails['changelog.ebuildadded'].append(relative_path)
1679
1680                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1681                         #ebuild not added to vcs
1682                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1683                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1684                 myesplit=portage.pkgsplit(y)
1685                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1686                             or pv_toolong_re.search(myesplit[1]) \
1687                             or pv_toolong_re.search(myesplit[2]):
1688                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1689                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1690                         continue
1691                 elif myesplit[0]!=pkgdir:
1692                         print(pkgdir,myesplit[0])
1693                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1694                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1695                         continue
1696
1697                 pkg = pkgs[y]
1698
1699                 if pkg.invalid:
1700                         allvalid = False
1701                         for k, msgs in pkg.invalid.items():
1702                                 for msg in msgs:
1703                                         stats[k] = stats[k] + 1
1704                                         fails[k].append("%s %s" % (relative_path, msg))
1705                         continue
1706
1707                 myaux = pkg.metadata
1708                 eapi = myaux["EAPI"]
1709                 inherited = pkg.inherited
1710                 live_ebuild = live_eclasses.intersection(inherited)
1711
1712                 for k, v in myaux.items():
1713                         if not isinstance(v, basestring):
1714                                 continue
1715                         m = non_ascii_re.search(v)
1716                         if m is not None:
1717                                 stats["variable.invalidchar"] += 1
1718                                 fails["variable.invalidchar"].append(
1719                                         ("%s: %s variable contains non-ASCII " + \
1720                                         "character at position %s") % \
1721                                         (relative_path, k, m.start() + 1))
1722
1723                 if not src_uri_error:
1724                         # Check that URIs don't reference a server from thirdpartymirrors.
1725                         for uri in portage.dep.use_reduce( \
1726                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1727                                 contains_mirror = False
1728                                 for mirror in thirdpartymirrors:
1729                                         if uri.startswith(mirror):
1730                                                 contains_mirror = True
1731                                                 break
1732                                 if not contains_mirror:
1733                                         continue
1734
1735                                 stats["SRC_URI.mirror"] += 1
1736                                 fails["SRC_URI.mirror"].append(
1737                                         "%s: '%s' found in thirdpartymirrors" % \
1738                                         (relative_path, mirror))
1739
1740                 if myaux.get("PROVIDE"):
1741                         stats["virtual.oldstyle"]+=1
1742                         fails["virtual.oldstyle"].append(relative_path)
1743
1744                 for pos, missing_var in enumerate(missingvars):
1745                         if not myaux.get(missing_var):
1746                                 if catdir == "virtual" and \
1747                                         missing_var in ("HOMEPAGE", "LICENSE"):
1748                                         continue
1749                                 if live_ebuild and missing_var == "KEYWORDS":
1750                                         continue
1751                                 myqakey=missingvars[pos]+".missing"
1752                                 stats[myqakey]=stats[myqakey]+1
1753                                 fails[myqakey].append(x+"/"+y+".ebuild")
1754
1755                 if catdir == "virtual":
1756                         for var in ("HOMEPAGE", "LICENSE"):
1757                                 if myaux.get(var):
1758                                         myqakey = var + ".virtual"
1759                                         stats[myqakey] = stats[myqakey] + 1
1760                                         fails[myqakey].append(relative_path)
1761
1762                 # 14 is the length of DESCRIPTION=""
1763                 if len(myaux['DESCRIPTION']) > max_desc_len:
1764                         stats['DESCRIPTION.toolong'] += 1
1765                         fails['DESCRIPTION.toolong'].append(
1766                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1767                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1768
1769                 keywords = myaux["KEYWORDS"].split()
1770                 stable_keywords = []
1771                 for keyword in keywords:
1772                         if not keyword.startswith("~") and \
1773                                 not keyword.startswith("-"):
1774                                 stable_keywords.append(keyword)
1775                 if stable_keywords:
1776                         if ebuild_path in new_ebuilds:
1777                                 stable_keywords.sort()
1778                                 stats["KEYWORDS.stable"] += 1
1779                                 fails["KEYWORDS.stable"].append(
1780                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1781                                                 " ".join(stable_keywords))
1782
1783                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1784                         if not kw.startswith("-"))
1785
1786                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1787                 if previous_keywords is None:
1788                         slot_keywords[myaux["SLOT"]] = set()
1789                 elif ebuild_archs and not live_ebuild:
1790                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1791                         if dropped_keywords:
1792                                 stats["KEYWORDS.dropped"] += 1
1793                                 fails["KEYWORDS.dropped"].append(
1794                                         relative_path + ": %s" % \
1795                                         " ".join(sorted(dropped_keywords)))
1796
1797                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1798
1799                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1800                 if "-*" in keywords:
1801                         haskeyword = False
1802                         for kw in keywords:
1803                                 if kw[0] == "~":
1804                                         kw = kw[1:]
1805                                 if kw in kwlist:
1806                                         haskeyword = True
1807                         if not haskeyword:
1808                                 stats["KEYWORDS.stupid"] += 1
1809                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1810
1811                 """
1812                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1813                 not be allowed to be marked stable
1814                 """
1815                 if live_ebuild and repo_config.name == "gentoo":
1816                         bad_stable_keywords = []
1817                         for keyword in keywords:
1818                                 if not keyword.startswith("~") and \
1819                                         not keyword.startswith("-"):
1820                                         bad_stable_keywords.append(keyword)
1821                                 del keyword
1822                         if bad_stable_keywords:
1823                                 stats["LIVEVCS.stable"] += 1
1824                                 fails["LIVEVCS.stable"].append(
1825                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1826                                                 bad_stable_keywords)
1827                         del bad_stable_keywords
1828
1829                         if keywords and not has_global_mask(pkg):
1830                                 stats["LIVEVCS.unmasked"] += 1
1831                                 fails["LIVEVCS.unmasked"].append(relative_path)
1832
1833                 if options.ignore_arches:
1834                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1835                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1836                 else:
1837                         arches=[]
1838                         for keyword in myaux["KEYWORDS"].split():
1839                                 if (keyword[0]=="-"):
1840                                         continue
1841                                 elif (keyword[0]=="~"):
1842                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1843                                 else:
1844                                         arches.append([keyword, keyword, [keyword]])
1845                         if not arches:
1846                                 # Use an empty profile for checking dependencies of
1847                                 # packages that have empty KEYWORDS.
1848                                 arches.append(['**', '**', ['**']])
1849
1850                 unknown_pkgs = set()
1851                 baddepsyntax = False
1852                 badlicsyntax = False
1853                 badprovsyntax = False
1854                 catpkg = catdir+"/"+y
1855
1856                 inherited_java_eclass = "java-pkg-2" in inherited or \
1857                         "java-pkg-opt-2" in inherited
1858                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1859                 operator_tokens = set(["||", "(", ")"])
1860                 type_list, badsyntax = [], []
1861                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1862                         "LICENSE", "PROPERTIES", "PROVIDE"):
1863                         mydepstr = myaux[mytype]
1864
1865                         token_class = None
1866                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1867                                 token_class=portage.dep.Atom
1868
1869                         try:
1870                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1871                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1872                         except portage.exception.InvalidDependString as e:
1873                                 atoms = None
1874                                 badsyntax.append(str(e))
1875
1876                         if atoms and mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1877                                 if mytype in ("RDEPEND", "PDEPEND") and \
1878                                         "test?" in mydepstr.split():
1879                                         stats[mytype + '.suspect'] += 1
1880                                         fails[mytype + '.suspect'].append(relative_path + \
1881                                                 ": 'test?' USE conditional in %s" % mytype)
1882
1883                                 for atom in atoms:
1884                                         if atom == "||":
1885                                                 continue
1886
1887                                         # Skip dependency.unknown for blockers, so that we
1888                                         # don't encourage people to remove necessary blockers,
1889                                         # as discussed in bug #382407.
1890                                         if atom.blocker is None and \
1891                                                 not portdb.xmatch("match-all", atom) and \
1892                                                 not atom.cp.startswith("virtual/"):
1893                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1894
1895                                         is_blocker = atom.blocker
1896
1897                                         if catdir != "virtual":
1898                                                 if not is_blocker and \
1899                                                         atom.cp in suspect_virtual:
1900                                                         stats['virtual.suspect'] += 1
1901                                                         fails['virtual.suspect'].append(
1902                                                                 relative_path +
1903                                                                 ": %s: consider using '%s' instead of '%s'" %
1904                                                                 (mytype, suspect_virtual[atom.cp], atom))
1905
1906                                         if mytype == "DEPEND" and \
1907                                                 not is_blocker and \
1908                                                 not inherited_java_eclass and \
1909                                                 atom.cp == "virtual/jdk":
1910                                                 stats['java.eclassesnotused'] += 1
1911                                                 fails['java.eclassesnotused'].append(relative_path)
1912                                         elif mytype == "DEPEND" and \
1913                                                 not is_blocker and \
1914                                                 not inherited_wxwidgets_eclass and \
1915                                                 atom.cp == "x11-libs/wxGTK":
1916                                                 stats['wxwidgets.eclassnotused'] += 1
1917                                                 fails['wxwidgets.eclassnotused'].append(
1918                                                         relative_path + ": DEPENDs on x11-libs/wxGTK"
1919                                                         " without inheriting wxwidgets.eclass")
1920                                         elif mytype in ("PDEPEND", "RDEPEND"):
1921                                                 if not is_blocker and \
1922                                                         atom.cp in suspect_rdepend:
1923                                                         stats[mytype + '.suspect'] += 1
1924                                                         fails[mytype + '.suspect'].append(
1925                                                                 relative_path + ": '%s'" % atom)
1926
1927                                         if atom.operator == "~" and \
1928                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1929                                                 stats[mytype + '.badtilde'] += 1
1930                                                 fails[mytype + '.badtilde'].append(
1931                                                         (relative_path + ": %s uses the ~ operator"
1932                                                          " with a non-zero revision:" + \
1933                                                          " '%s'") % (mytype, atom))
1934
1935                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1936
1937                 for m,b in zip(type_list, badsyntax):
1938                         stats[m+".syntax"] += 1
1939                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1940
1941                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1942                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1943                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1944                 badlicsyntax = badlicsyntax > 0
1945                 badprovsyntax = badprovsyntax > 0
1946
1947                 # uselist checks - global
1948                 myuse = []
1949                 default_use = []
1950                 for myflag in myaux["IUSE"].split():
1951                         flag_name = myflag.lstrip("+-")
1952                         used_useflags.add(flag_name)
1953                         if myflag != flag_name:
1954                                 default_use.append(myflag)
1955                         if flag_name not in uselist:
1956                                 myuse.append(flag_name)
1957
1958                 # uselist checks - metadata
1959                 for mypos in range(len(myuse)-1,-1,-1):
1960                         if myuse[mypos] and (myuse[mypos] in muselist):
1961                                 del myuse[mypos]
1962
1963                 if default_use and not eapi_has_iuse_defaults(eapi):
1964                         for myflag in default_use:
1965                                 stats['EAPI.incompatible'] += 1
1966                                 fails['EAPI.incompatible'].append(
1967                                         (relative_path + ": IUSE defaults" + \
1968                                         " not supported with EAPI='%s':" + \
1969                                         " '%s'") % (eapi, myflag))
1970
1971                 for mypos in range(len(myuse)):
1972                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1973                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1974
1975                 # license checks
1976                 if not badlicsyntax:
1977                         # Parse the LICENSE variable, remove USE conditions and
1978                         # flatten it.
1979                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
1980                         # Check each entry to ensure that it exists in PORTDIR's
1981                         # license directory.
1982                         for lic in licenses:
1983                                 # Need to check for "||" manually as no portage
1984                                 # function will remove it without removing values.
1985                                 if lic not in liclist and lic != "||":
1986                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1987                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
1988
1989                 #keyword checks
1990                 myuse = myaux["KEYWORDS"].split()
1991                 for mykey in myuse:
1992                         if mykey not in ("-*", "*", "~*"):
1993                                 myskey = mykey
1994                                 if myskey[:1] == "-":
1995                                         myskey = myskey[1:]
1996                                 if myskey[:1] == "~":
1997                                         myskey = myskey[1:]
1998                                 if myskey not in kwlist:
1999                                         stats["KEYWORDS.invalid"] += 1
2000                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
2001                                 elif myskey not in profiles:
2002                                         stats["KEYWORDS.invalid"] += 1
2003                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
2004
2005                 #restrict checks
2006                 myrestrict = None
2007                 try:
2008                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2009                 except portage.exception.InvalidDependString as e:
2010                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
2011                         fails["RESTRICT.syntax"].append(
2012                                 "%s: RESTRICT: %s" % (relative_path, e))
2013                         del e
2014                 if myrestrict:
2015                         myrestrict = set(myrestrict)
2016                         mybadrestrict = myrestrict.difference(valid_restrict)
2017                         if mybadrestrict:
2018                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2019                                 for mybad in mybadrestrict:
2020                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
2021                 #REQUIRED_USE check
2022                 required_use = myaux["REQUIRED_USE"]
2023                 if required_use:
2024                         if not eapi_has_required_use(eapi):
2025                                 stats['EAPI.incompatible'] += 1
2026                                 fails['EAPI.incompatible'].append(
2027                                         relative_path + ": REQUIRED_USE" + \
2028                                         " not supported with EAPI='%s'" % (eapi,))
2029                         try:
2030                                 portage.dep.check_required_use(required_use, (),
2031                                         pkg.iuse.is_valid_flag, eapi=eapi)
2032                         except portage.exception.InvalidDependString as e:
2033                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2034                                 fails["REQUIRED_USE.syntax"].append(
2035                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2036                                 del e
2037
2038                 # Syntax Checks
2039                 relative_path = os.path.join(x, y + ".ebuild")
2040                 full_path = os.path.join(repodir, relative_path)
2041                 if not vcs_preserves_mtime:
2042                         if ebuild_path not in new_ebuilds and \
2043                                 ebuild_path not in modified_ebuilds:
2044                                 pkg.mtime = None
2045                 try:
2046                         # All ebuilds should have utf_8 encoding.
2047                         f = io.open(_unicode_encode(full_path,
2048                                 encoding=_encodings['fs'], errors='strict'),
2049                                 mode='r', encoding=_encodings['repo.content'])
2050                         try:
2051                                 for check_name, e in run_checks(f, pkg):
2052                                         stats[check_name] += 1
2053                                         fails[check_name].append(relative_path + ': %s' % e)
2054                         finally:
2055                                 f.close()
2056                 except UnicodeDecodeError:
2057                         # A file.UTF8 failure will have already been recorded above.
2058                         pass
2059
2060                 if options.force:
2061                         # The dep_check() calls are the most expensive QA test. If --force
2062                         # is enabled, there's no point in wasting time on these since the
2063                         # user is intent on forcing the commit anyway.
2064                         continue
2065
2066                 for keyword,arch,groups in arches:
2067
2068                         if arch not in profiles:
2069                                 # A missing profile will create an error further down
2070                                 # during the KEYWORDS verification.
2071                                 continue
2072
2073                         for prof in profiles[arch]:
2074
2075                                 if prof.status not in ("stable", "dev") or \
2076                                         prof.status == "dev" and not options.include_dev:
2077                                         continue
2078
2079                                 dep_settings = arch_caches.get(prof.sub_path)
2080                                 if dep_settings is None:
2081                                         dep_settings = portage.config(
2082                                                 config_profile_path=prof.abs_path,
2083                                                 config_incrementals=repoman_incrementals,
2084                                                 config_root=config_root,
2085                                                 local_config=False,
2086                                                 _unmatched_removal=options.unmatched_removal,
2087                                                 env=env)
2088                                         dep_settings.categories = repoman_settings.categories
2089                                         if options.without_mask:
2090                                                 dep_settings._mask_manager_obj = \
2091                                                         copy.deepcopy(dep_settings._mask_manager)
2092                                                 dep_settings._mask_manager._pmaskdict.clear()
2093                                         arch_caches[prof.sub_path] = dep_settings
2094
2095                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2096                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2097                                 if xcache is None:
2098                                         portdb.melt()
2099                                         portdb.freeze()
2100                                         xcache = portdb.xcache
2101                                         xcache.update(shared_xmatch_caches)
2102                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2103
2104                                 trees[root]["porttree"].settings = dep_settings
2105                                 portdb.settings = dep_settings
2106                                 portdb.xcache = xcache
2107                                 # for package.use.mask support inside dep_check
2108                                 dep_settings.setcpv(pkg)
2109                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2110                                 # just in case, prevent config.reset() from nuking these.
2111                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2112
2113                                 if not baddepsyntax:
2114                                         ismasked = not ebuild_archs or \
2115                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2116                                         if ismasked:
2117                                                 if not have_pmasked:
2118                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2119                                                                 pkg.cpv, pkg.metadata))
2120                                                 if options.ignore_masked:
2121                                                         continue
2122                                                 #we are testing deps for a masked package; give it some lee-way
2123                                                 suffix="masked"
2124                                                 matchmode = "minimum-all"
2125                                         else:
2126                                                 suffix=""
2127                                                 matchmode = "minimum-visible"
2128
2129                                         if not have_dev_keywords:
2130                                                 have_dev_keywords = \
2131                                                         bool(dev_keywords.intersection(keywords))
2132
2133                                         if prof.status == "dev":
2134                                                 suffix=suffix+"indev"
2135
2136                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
2137
2138                                                 mykey=mytype+".bad"+suffix
2139                                                 myvalue = myaux[mytype]
2140                                                 if not myvalue:
2141                                                         continue
2142
2143                                                 success, atoms = portage.dep_check(myvalue, portdb,
2144                                                         dep_settings, use="all", mode=matchmode,
2145                                                         trees=trees)
2146
2147                                                 if success:
2148                                                         if atoms:
2149
2150                                                                 # Don't bother with dependency.unknown for
2151                                                                 # cases in which *DEPEND.bad is triggered.
2152                                                                 for atom in atoms:
2153                                                                         # dep_check returns all blockers and they
2154                                                                         # aren't counted for *DEPEND.bad, so we
2155                                                                         # ignore them here.
2156                                                                         if not atom.blocker:
2157                                                                                 unknown_pkgs.discard(
2158                                                                                         (mytype, atom.unevaluated_atom))
2159
2160                                                                 if not prof.sub_path:
2161                                                                         # old-style virtuals currently aren't
2162                                                                         # resolvable with empty profile, since
2163                                                                         # 'virtuals' mappings are unavailable
2164                                                                         # (it would be expensive to search
2165                                                                         # for PROVIDE in all ebuilds)
2166                                                                         atoms = [atom for atom in atoms if not \
2167                                                                                 (atom.cp.startswith('virtual/') and \
2168                                                                                 not portdb.cp_list(atom.cp))]
2169
2170                                                                 #we have some unsolvable deps
2171                                                                 #remove ! deps, which always show up as unsatisfiable
2172                                                                 atoms = [str(atom.unevaluated_atom) \
2173                                                                         for atom in atoms if not atom.blocker]
2174
2175                                                                 #if we emptied out our list, continue:
2176                                                                 if not atoms:
2177                                                                         continue
2178                                                                 stats[mykey]=stats[mykey]+1
2179                                                                 fails[mykey].append("%s: %s(%s) %s" % \
2180                                                                         (relative_path, keyword,
2181                                                                         prof, repr(atoms)))
2182                                                 else:
2183                                                         stats[mykey]=stats[mykey]+1
2184                                                         fails[mykey].append("%s: %s(%s) %s" % \
2185                                                                 (relative_path, keyword,
2186                                                                 prof, repr(atoms)))
2187
2188                 if not baddepsyntax and unknown_pkgs:
2189                         type_map = {}
2190                         for mytype, atom in unknown_pkgs:
2191                                 type_map.setdefault(mytype, set()).add(atom)
2192                         for mytype, atoms in type_map.items():
2193                                 stats["dependency.unknown"] += 1
2194                                 fails["dependency.unknown"].append("%s: %s: %s" %
2195                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2196
2197         # check if there are unused local USE-descriptions in metadata.xml
2198         # (unless there are any invalids, to avoid noise)
2199         if allvalid:
2200                 for myflag in muselist.difference(used_useflags):
2201                         stats["metadata.warning"] += 1
2202                         fails["metadata.warning"].append(
2203                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2204                                 (x, myflag))
2205
2206 if options.if_modified == "y" and len(effective_scanlist) < 1:
2207         logging.warn("--if-modified is enabled, but no modified packages were found!")
2208
2209 if options.mode == "manifest":
2210         sys.exit(dofail)
2211
2212 #dofail will be set to 1 if we have failed in at least one non-warning category
2213 dofail=0
2214 #dowarn will be set to 1 if we tripped any warnings
2215 dowarn=0
2216 #dofull will be set if we should print a "repoman full" informational message
2217 dofull = options.mode != 'full'
2218
2219 for x in qacats:
2220         if not stats[x]:
2221                 continue
2222         dowarn = 1
2223         if x not in qawarnings:
2224                 dofail = 1
2225
2226 if dofail or \
2227         (dowarn and not (options.quiet or options.mode == "scan")):
2228         dofull = 0
2229
2230 # Save QA output so that it can be conveniently displayed
2231 # in $EDITOR while the user creates a commit message.
2232 # Otherwise, the user would not be able to see this output
2233 # once the editor has taken over the screen.
2234 qa_output = io.StringIO()
2235 style_file = ConsoleStyleFile(sys.stdout)
2236 if options.mode == 'commit' and \
2237         (not commitmessage or not commitmessage.strip()):
2238         style_file.write_listener = qa_output
2239 console_writer = StyleWriter(file=style_file, maxcol=9999)
2240 console_writer.style_listener = style_file.new_styles
2241
2242 f = formatter.AbstractFormatter(console_writer)
2243
2244 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2245
2246 style_file.flush()
2247 del console_writer, f, style_file
2248 qa_output = qa_output.getvalue()
2249 qa_output = qa_output.splitlines(True)
2250
2251 def grouplist(mylist,seperator="/"):
2252         """(list,seperator="/") -- Takes a list of elements; groups them into
2253         same initial element categories. Returns a dict of {base:[sublist]}
2254         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2255         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2256         mygroups={}
2257         for x in mylist:
2258                 xs=x.split(seperator)
2259                 if xs[0]==".":
2260                         xs=xs[1:]
2261                 if xs[0] not in mygroups:
2262                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2263                 else:
2264                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2265         return mygroups
2266
2267 suggest_ignore_masked = False
2268 suggest_include_dev = False
2269
2270 if have_pmasked and not (options.without_mask or options.ignore_masked):
2271         suggest_ignore_masked = True
2272 if have_dev_keywords and not options.include_dev:
2273         suggest_include_dev = True
2274
2275 if suggest_ignore_masked or suggest_include_dev:
2276         print()
2277         if suggest_ignore_masked:
2278                 print(bold("Note: use --without-mask to check " + \
2279                         "KEYWORDS on dependencies of masked packages"))
2280
2281         if suggest_include_dev:
2282                 print(bold("Note: use --include-dev (-d) to check " + \
2283                         "dependencies for 'dev' profiles"))
2284         print()
2285
2286 if options.mode != 'commit':
2287         if dofull:
2288                 print(bold("Note: type \"repoman full\" for a complete listing."))
2289         if dowarn and not dofail:
2290                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2291         elif not dofail:
2292                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2293         elif dofail:
2294                 print(bad("Please fix these important QA issues first."))
2295                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2296                 sys.exit(1)
2297 else:
2298         if dofail and can_force and options.force and not options.pretend:
2299                 print(green("RepoMan sez:") + \
2300                         " \"You want to commit even with these QA issues?\n" + \
2301                         "              I'll take it this time, but I'm not happy.\"\n")
2302         elif dofail:
2303                 if options.force and not can_force:
2304                         print(bad("The --force option has been disabled due to extraordinary issues."))
2305                 print(bad("Please fix these important QA issues first."))
2306                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2307                 sys.exit(1)
2308
2309         if options.pretend:
2310                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2311
2312         myunadded = []
2313         if vcs == "cvs":
2314                 try:
2315                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2316                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2317                 except SystemExit as e:
2318                         raise  # TODO propagate this
2319                 except:
2320                         err("Error retrieving CVS tree; exiting.")
2321         if vcs == "svn":
2322                 try:
2323                         with os.popen("svn status --no-ignore") as f:
2324                                 svnstatus = f.readlines()
2325                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2326                 except SystemExit as e:
2327                         raise  # TODO propagate this
2328                 except:
2329                         err("Error retrieving SVN info; exiting.")
2330         if vcs == "git":
2331                 # get list of files not under version control or missing
2332                 myf = os.popen("git ls-files --others")
2333                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2334                 myf.close()
2335         if vcs == "bzr":
2336                 try:
2337                         with os.popen("bzr status -S .") as f:
2338                                 bzrstatus = f.readlines()
2339                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2340                 except SystemExit as e:
2341                         raise  # TODO propagate this
2342                 except:
2343                         err("Error retrieving bzr info; exiting.")
2344         if vcs == "hg":
2345                 with os.popen("hg status --no-status --unknown .") as f:
2346                         myunadded = f.readlines()
2347                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2348
2349                 # Mercurial doesn't handle manually deleted files as removed from
2350                 # the repository, so the user need to remove them before commit,
2351                 # using "hg remove [FILES]"
2352                 with os.popen("hg status --no-status --deleted .") as f:
2353                         mydeleted = f.readlines()
2354                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2355
2356
2357         myautoadd=[]
2358         if myunadded:
2359                 for x in range(len(myunadded)-1,-1,-1):
2360                         xs=myunadded[x].split("/")
2361                         if xs[-1]=="files":
2362                                 print("!!! files dir is not added! Please correct this.")
2363                                 sys.exit(-1)
2364                         elif xs[-1]=="Manifest":
2365                                 # It's a manifest... auto add
2366                                 myautoadd+=[myunadded[x]]
2367                                 del myunadded[x]
2368
2369         if myunadded:
2370                 print(red("!!! The following files are in your local tree but are not added to the master"))
2371                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2372                 for x in myunadded:
2373                         print("   ",x)
2374                 print()
2375                 print()
2376                 sys.exit(1)
2377
2378         if vcs == "hg" and mydeleted:
2379                 print(red("!!! The following files are removed manually from your local tree but are not"))
2380                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2381                 for x in mydeleted:
2382                         print("   ",x)
2383                 print()
2384                 print()
2385                 sys.exit(1)
2386
2387         if vcs == "cvs":
2388                 mycvstree = cvstree.getentries("./", recursive=1)
2389                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2390                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2391                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2392                 bin_blob_pattern = re.compile("^-kb$")
2393                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2394                         recursive=1, basedir="./"))
2395
2396
2397         if vcs == "svn":
2398                 with os.popen("svn status") as f:
2399                         svnstatus = f.readlines()
2400                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2401                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2402                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2403
2404                 # Subversion expands keywords specified in svn:keywords properties.
2405                 with os.popen("svn propget -R svn:keywords") as f:
2406                         props = f.readlines()
2407                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2408                         for prop in props if " - " in prop)
2409
2410         elif vcs == "git":
2411                 with os.popen("git diff-index --name-only "
2412                         "--relative --diff-filter=M HEAD") as f:
2413                         mychanged = f.readlines()
2414                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2415
2416                 with os.popen("git diff-index --name-only "
2417                         "--relative --diff-filter=A HEAD") as f:
2418                         mynew = f.readlines()
2419                 mynew = ["./" + elem[:-1] for elem in mynew]
2420
2421                 with os.popen("git diff-index --name-only "
2422                         "--relative --diff-filter=D HEAD") as f:
2423                         myremoved = f.readlines()
2424                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2425
2426         if vcs == "bzr":
2427                 with os.popen("bzr status -S .") as f:
2428                         bzrstatus = f.readlines()
2429                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2430                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2431                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2432                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2433                 # Bazaar expands nothing.
2434
2435         if vcs == "hg":
2436                 with os.popen("hg status --no-status --modified .") as f:
2437                         mychanged = f.readlines()
2438                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2439
2440                 with os.popen("hg status --no-status --added .") as f:
2441                         mynew = f.readlines()
2442                 mynew = ["./" + elem.rstrip() for elem in mynew]
2443
2444                 with os.popen("hg status --no-status --removed .") as f:
2445                         myremoved = f.readlines()
2446                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2447
2448         if vcs:
2449                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2450                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2451                         print()
2452                         print("(Didn't find any changed files...)")
2453                         print()
2454                         sys.exit(1)
2455
2456         # Manifests need to be regenerated after all other commits, so don't commit
2457         # them now even if they have changed.
2458         mymanifests = set()
2459         myupdates = set()
2460         for f in mychanged + mynew:
2461                 if "Manifest" == os.path.basename(f):
2462                         mymanifests.add(f)
2463                 else:
2464                         myupdates.add(f)
2465         myupdates.difference_update(myremoved)
2466         myupdates = list(myupdates)
2467         mymanifests = list(mymanifests)
2468         myheaders = []
2469         mydirty = []
2470
2471         commitmessage = options.commitmsg
2472         if options.commitmsgfile:
2473                 try:
2474                         f = io.open(_unicode_encode(options.commitmsgfile,
2475                         encoding=_encodings['fs'], errors='strict'),
2476                         mode='r', encoding=_encodings['content'], errors='replace')
2477                         commitmessage = f.read()
2478                         f.close()
2479                         del f
2480                 except (IOError, OSError) as e:
2481                         if e.errno == errno.ENOENT:
2482                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2483                         else:
2484                                 raise
2485                 # We've read the content so the file is no longer needed.
2486                 commitmessagefile = None
2487         if not commitmessage or not commitmessage.strip():
2488                 try:
2489                         editor = os.environ.get("EDITOR")
2490                         if editor and utilities.editor_is_executable(editor):
2491                                 commitmessage = utilities.get_commit_message_with_editor(
2492                                         editor, message=qa_output)
2493                         else:
2494                                 commitmessage = utilities.get_commit_message_with_stdin()
2495                 except KeyboardInterrupt:
2496                         exithandler()
2497                 if not commitmessage or not commitmessage.strip():
2498                         print("* no commit message?  aborting commit.")
2499                         sys.exit(1)
2500         commitmessage = commitmessage.rstrip()
2501         changelog_msg = commitmessage
2502         portage_version = getattr(portage, "VERSION", None)
2503         if portage_version is None:
2504                 sys.stderr.write("Failed to insert portage version in message!\n")
2505                 sys.stderr.flush()
2506                 portage_version = "Unknown"
2507         unameout = platform.system() + " "
2508         if platform.system() in ["Darwin", "SunOS"]:
2509                 unameout += platform.processor()
2510         else:
2511                 unameout += platform.machine()
2512         commitmessage += "\n\n(Portage version: %s/%s/%s" % \
2513                 (portage_version, vcs, unameout)
2514         if options.force:
2515                 commitmessage += ", RepoMan options: --force"
2516         commitmessage += ")"
2517
2518         if options.echangelog in ('y', 'force'):
2519                 logging.info("checking for unmodified ChangeLog files")
2520                 committer_name = utilities.get_committer_name(env=repoman_settings)
2521                 for x in sorted(vcs_files_to_cps(
2522                         chain(myupdates, mymanifests, myremoved))):
2523                         catdir, pkgdir = x.split("/")
2524                         checkdir = repodir + "/" + x
2525                         checkdir_relative = ""
2526                         if repolevel < 3:
2527                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2528                         if repolevel < 2:
2529                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2530                         checkdir_relative = os.path.join(".", checkdir_relative)
2531
2532                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2533                         changelog_modified = changelog_path in modified_changelogs
2534                         if changelog_modified and options.echangelog != 'force':
2535                                 continue
2536
2537                         # get changes for this package
2538                         cdrlen = len(checkdir_relative)
2539                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2540                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2541                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2542
2543                         # Skip ChangeLog generation if only the Manifest was modified,
2544                         # as discussed in bug #398009.
2545                         nontrivial_cl_files = set()
2546                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2547                         nontrivial_cl_files.difference_update(['Manifest'])
2548                         if not nontrivial_cl_files and options.echangelog != 'force':
2549                                 continue
2550
2551                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2552                                 committer_name, changelog_msg,
2553                                 os.path.join(repodir, 'skel.ChangeLog'),
2554                                 catdir, pkgdir,
2555                                 new=clnew, removed=clremoved, changed=clchanged,
2556                                 pretend=options.pretend)
2557                         if new_changelog is None:
2558                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2559                                         level=logging.ERROR, noiselevel=-1)
2560                                 sys.exit(1)
2561
2562                         # if the ChangeLog was just created, add it to vcs
2563                         if new_changelog:
2564                                 myautoadd.append(changelog_path)
2565                                 # myautoadd is appended to myupdates below
2566                         else:
2567                                 myupdates.append(changelog_path)
2568
2569                         if options.ask and not options.pretend:
2570                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2571                                 repoman_settings["O"] = checkdir
2572                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2573
2574         if myautoadd:
2575                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2576                 add_cmd = [vcs, "add"]
2577                 add_cmd += myautoadd
2578                 if options.pretend:
2579                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2580                                 noiselevel=-1)
2581                 else:
2582                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2583                                 # Python 3.1 produces the following TypeError if raw bytes are
2584                                 # passed to subprocess.call():
2585                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2586                                 #     errread, errwrite)
2587                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2588                                 #     raise child_exception
2589                                 # TypeError: expected an object with the buffer interface
2590                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2591                         retcode = subprocess.call(add_cmd)
2592                         if retcode != os.EX_OK:
2593                                 logging.error(
2594                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2595                                 sys.exit(retcode)
2596
2597                 myupdates += myautoadd
2598
2599         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2600
2601         if vcs not in ('cvs', 'svn'):
2602                 # With git, bzr and hg, there's never any keyword expansion, so
2603                 # there's no need to regenerate manifests and all files will be
2604                 # committed in one big commit at the end.
2605                 print()
2606         elif not repo_config.thin_manifest:
2607                 if vcs == 'cvs':
2608                         headerstring = "'\$(Header|Id).*\$'"
2609                 elif vcs == "svn":
2610                         svn_keywords = dict((k.lower(), k) for k in [
2611                                         "Rev",
2612                                         "Revision",
2613                                         "LastChangedRevision",
2614                                         "Date",
2615                                         "LastChangedDate",
2616                                         "Author",
2617                                         "LastChangedBy",
2618                                         "URL",
2619                                         "HeadURL",
2620                                         "Id",
2621                                         "Header",
2622                         ])
2623
2624                 for myfile in myupdates:
2625
2626                         # for CVS, no_expansion contains files that are excluded from expansion
2627                         if vcs == "cvs":
2628                                 if myfile in no_expansion:
2629                                         continue
2630
2631                         # for SVN, expansion contains files that are included in expansion
2632                         elif vcs == "svn":
2633                                 if myfile not in expansion:
2634                                         continue
2635
2636                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2637                                 enabled_keywords = []
2638                                 for k in expansion[myfile]:
2639                                         keyword = svn_keywords.get(k.lower())
2640                                         if keyword is not None:
2641                                                 enabled_keywords.append(keyword)
2642
2643                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2644
2645                         myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile)
2646                         if myout[0] == 0:
2647                                 myheaders.append(myfile)
2648
2649                 print("%s have headers that will change." % green(str(len(myheaders))))
2650                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2651
2652         logging.info("myupdates: %s", myupdates)
2653         logging.info("myheaders: %s", myheaders)
2654
2655         if options.ask and userquery('Commit changes?', True) != 'Yes':
2656                 print("* aborting commit.")
2657                 sys.exit(128 + signal.SIGINT)
2658
2659         # Handle the case where committed files have keywords which
2660         # will change and need a priming commit before the Manifest
2661         # can be committed.
2662         if (myupdates or myremoved) and myheaders:
2663                 myfiles = myupdates + myremoved
2664                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2665                 mymsg = os.fdopen(fd, "wb")
2666                 mymsg.write(_unicode_encode(commitmessage))
2667                 mymsg.close()
2668
2669                 print()
2670                 print(green("Using commit message:"))
2671                 print(green("------------------------------------------------------------------------------"))
2672                 print(commitmessage)
2673                 print(green("------------------------------------------------------------------------------"))
2674                 print()
2675
2676                 # Having a leading ./ prefix on file paths can trigger a bug in
2677                 # the cvs server when committing files to multiple directories,
2678                 # so strip the prefix.
2679                 myfiles = [f.lstrip("./") for f in myfiles]
2680
2681                 commit_cmd = [vcs]
2682                 commit_cmd.extend(vcs_global_opts)
2683                 commit_cmd.append("commit")
2684                 commit_cmd.extend(vcs_local_opts)
2685                 commit_cmd.extend(["-F", commitmessagefile])
2686                 commit_cmd.extend(myfiles)
2687
2688                 try:
2689                         if options.pretend:
2690                                 print("(%s)" % (" ".join(commit_cmd),))
2691                         else:
2692                                 retval = spawn(commit_cmd, env=os.environ)
2693                                 if retval != os.EX_OK:
2694                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2695                                                 "error code: %s\n") % (vcs, retval),
2696                                                 level=logging.ERROR, noiselevel=-1)
2697                                         sys.exit(retval)
2698                 finally:
2699                         try:
2700                                 os.unlink(commitmessagefile)
2701                         except OSError:
2702                                 pass
2703
2704         # Setup the GPG commands
2705         def gpgsign(filename):
2706                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2707                 if gpgcmd is None:
2708                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2709                                 " Is make.globals missing?")
2710                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2711                         "PORTAGE_GPG_KEY" not in repoman_settings:
2712                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2713                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2714                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2715                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2716                                         os.path.expanduser("~/.gnupg")
2717                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2718                                         % repoman_settings["PORTAGE_GPG_DIR"])
2719                         else:
2720                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2721                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2722                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2723                                 raise portage.exception.InvalidLocation(
2724                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2725                                         repoman_settings["PORTAGE_GPG_DIR"])
2726                 gpgvars = {"FILE": filename}
2727                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2728                         v = repoman_settings.get(k)
2729                         if v is not None:
2730                                 gpgvars[k] = v
2731                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2732                 if options.pretend:
2733                         print("("+gpgcmd+")")
2734                 else:
2735                         rValue = os.system(gpgcmd)
2736                         if rValue == os.EX_OK:
2737                                 os.rename(filename+".asc", filename)
2738                         else:
2739                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2740
2741         # When files are removed and re-added, the cvs server will put /Attic/
2742         # inside the $Header path. This code detects the problem and corrects it
2743         # so that the Manifest will generate correctly. See bug #169500.
2744         # Use binary mode in order to avoid potential character encoding issues.
2745         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2746         attic_str = b'/Attic/'
2747         attic_replace = b'/'
2748         for x in myheaders:
2749                 f = open(_unicode_encode(x,
2750                         encoding=_encodings['fs'], errors='strict'),
2751                         mode='rb')
2752                 mylines = f.readlines()
2753                 f.close()
2754                 modified = False
2755                 for i, line in enumerate(mylines):
2756                         if cvs_header_re.match(line) is not None and \
2757                                 attic_str in line:
2758                                 mylines[i] = line.replace(attic_str, attic_replace)
2759                                 modified = True
2760                 if modified:
2761                         portage.util.write_atomic(x, b''.join(mylines),
2762                                 mode='wb')
2763
2764         if repolevel == 1:
2765                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2766                         "doing the entire repository.\"\n")
2767
2768         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2769
2770                 for x in sorted(vcs_files_to_cps(
2771                         chain(myupdates, myremoved, mymanifests))):
2772                         repoman_settings["O"] = os.path.join(repodir, x)
2773                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2774
2775         signed = False
2776         if sign_manifests:
2777                 signed = True
2778                 try:
2779                         for x in sorted(vcs_files_to_cps(
2780                                 chain(myupdates, myremoved, mymanifests))):
2781                                 repoman_settings["O"] = os.path.join(repodir, x)
2782                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2783                                 if not os.path.exists(manifest_path):
2784                                         continue
2785                                 gpgsign(manifest_path)
2786                 except portage.exception.PortageException as e:
2787                         portage.writemsg("!!! %s\n" % str(e))
2788                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2789                         signed = False
2790
2791         if vcs == 'git':
2792                 # It's not safe to use the git commit -a option since there might
2793                 # be some modified files elsewhere in the working tree that the
2794                 # user doesn't want to commit. Therefore, call git update-index
2795                 # in order to ensure that the index is updated with the latest
2796                 # versions of all new and modified files in the relevant portion
2797                 # of the working tree.
2798                 myfiles = mymanifests + myupdates
2799                 myfiles.sort()
2800                 update_index_cmd = ["git", "update-index"]
2801                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2802                 if options.pretend:
2803                         print("(%s)" % (" ".join(update_index_cmd),))
2804                 else:
2805                         retval = spawn(update_index_cmd, env=os.environ)
2806                         if retval != os.EX_OK:
2807                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2808                                         "error code: %s\n") % (vcs, retval),
2809                                         level=logging.ERROR, noiselevel=-1)
2810                                 sys.exit(retval)
2811
2812         if True:
2813
2814                 myfiles = mymanifests[:]
2815                 # If there are no header (SVN/CVS keywords) changes in
2816                 # the files, this Manifest commit must include the
2817                 # other (yet uncommitted) files.
2818                 if not myheaders:
2819                         myfiles += myupdates
2820                         myfiles += myremoved
2821                 myfiles.sort()
2822
2823                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2824                 mymsg = os.fdopen(fd, "wb")
2825                 # strip the closing parenthesis
2826                 mymsg.write(_unicode_encode(commitmessage[:-1]))
2827                 if signed:
2828                         mymsg.write(_unicode_encode(
2829                                 ", signed Manifest commit with key %s)" % \
2830                                 repoman_settings["PORTAGE_GPG_KEY"]))
2831                 else:
2832                         mymsg.write(b", unsigned Manifest commit)")
2833                 mymsg.close()
2834
2835                 commit_cmd = []
2836                 if options.pretend and vcs is None:
2837                         # substitute a bogus value for pretend output
2838                         commit_cmd.append("cvs")
2839                 else:
2840                         commit_cmd.append(vcs)
2841                 commit_cmd.extend(vcs_global_opts)
2842                 commit_cmd.append("commit")
2843                 commit_cmd.extend(vcs_local_opts)
2844                 if vcs == "hg":
2845                         commit_cmd.extend(["--logfile", commitmessagefile])
2846                         commit_cmd.extend(myfiles)
2847                 else:
2848                         commit_cmd.extend(["-F", commitmessagefile])
2849                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2850
2851                 try:
2852                         if options.pretend:
2853                                 print("(%s)" % (" ".join(commit_cmd),))
2854                         else:
2855                                 retval = spawn(commit_cmd, env=os.environ)
2856                                 if retval != os.EX_OK:
2857
2858                                         if repo_config.sign_commit and vcs == 'git' and \
2859                                                 not git_supports_gpg_sign():
2860                                                 # Inform user that newer git is needed (bug #403323).
2861                                                 logging.error(
2862                                                         "Git >=1.7.9 is required for signed commits!")
2863
2864                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2865                                                 "error code: %s\n") % (vcs, retval),
2866                                                 level=logging.ERROR, noiselevel=-1)
2867                                         sys.exit(retval)
2868                 finally:
2869                         try:
2870                                 os.unlink(commitmessagefile)
2871                         except OSError:
2872                                 pass
2873
2874         print()
2875         if vcs:
2876                 print("Commit complete.")
2877         else:
2878                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2879         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2880 sys.exit(0)
2881