bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function, unicode_literals
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import io
  17 import logging
  18 import optparse
  19 import re
  20 import signal
  21 import stat
  22 import subprocess
  23 import sys
  24 import tempfile
  25 import textwrap
  26 import time
  27 import platform
  28
  29 try:
  30         from urllib.request import urlopen as urllib_request_urlopen
  31 except ImportError:
  32         from urllib import urlopen as urllib_request_urlopen
  33
  34 from itertools import chain
  35 from stat import S_ISDIR
  36
  37 from os import path as osp
  38 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  39 sys.path.insert(0, pym_path)
  40 import portage
  41 portage._internal_caller = True
  42 portage._disable_legacy_globals()
  43
  44 try:
  45         import xml.etree.ElementTree
  46         from xml.parsers.expat import ExpatError
  47 except (SystemExit, KeyboardInterrupt):
  48         raise
  49 except (ImportError, SystemError, RuntimeError, Exception):
  50         # broken or missing xml support
  51         # http://bugs.python.org/issue14988
  52         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  53         from portage.output import EOutput
  54         out = EOutput()
  55         for line in msg:
  56                 out.eerror(line)
  57         sys.exit(1)
  58
  59 from portage import os
  60 from portage import _encodings
  61 from portage import _unicode_encode
  62 from repoman.checks import run_checks
  63 from repoman import utilities
  64 from repoman.herdbase import make_herd_base
  65 from _emerge.Package import Package
  66 from _emerge.RootConfig import RootConfig
  67 from _emerge.userquery import userquery
  68 import portage.checksum
  69 import portage.const
  70 from portage import cvstree, normalize_path
  71 from portage import util
  72 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  73         ParseError, PermissionDenied)
  74 from portage.dep import Atom
  75 from portage.process import find_binary, spawn
  76 from portage.output import bold, create_color_func, \
  77         green, nocolor, red
  78 from portage.output import ConsoleStyleFile, StyleWriter
  79 from portage.util import writemsg_level
  80 from portage.package.ebuild.digestgen import digestgen
  81 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  82
  83 if sys.hexversion >= 0x3000000:
  84         basestring = str
  85
  86 util.initialize_logger()
  87
  88 # 14 is the length of DESCRIPTION=""
  89 max_desc_len = 100
  90 allowed_filename_chars="a-zA-Z0-9._-+:"
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  93 bad = create_color_func("BAD")
  94
  95 # A sane umask is needed for files that portage creates.
  96 os.umask(0o22)
  97 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  98 # behave incrementally.
  99 repoman_incrementals = tuple(x for x in \
 100         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 101 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 102 repoman_settings = portage.config(config_root=config_root, local_config=False)
 103
 104 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 105         repoman_settings.get('TERM') == 'dumb' or \
 106         not sys.stdout.isatty():
 107         nocolor()
 108
 109 def warn(txt):
 110         print("repoman: " + txt)
 111
 112 def err(txt):
 113         warn(txt)
 114         sys.exit(1)
 115
 116 def exithandler(signum=None, frame=None):
 117         logging.fatal("Interrupted; exiting...")
 118         if signum is None:
 119                 sys.exit(1)
 120         else:
 121                 sys.exit(128 + signum)
 122
 123 signal.signal(signal.SIGINT,exithandler)
 124
 125 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 126         """Repoman needs it's own HelpFormatter for now, because the default ones
 127         murder the help text."""
 128
 129         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 130                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 131
 132         def format_description(self, description):
 133                 return description
 134
 135 class RepomanOptionParser(optparse.OptionParser):
 136         """Add the on_tail function, ruby has it, optionParser should too
 137         """
 138
 139         def __init__(self, *args, **kwargs):
 140                 optparse.OptionParser.__init__(self, *args, **kwargs)
 141                 self.tail = ""
 142
 143         def on_tail(self, description):
 144                 self.tail += description
 145
 146         def format_help(self, formatter=None):
 147                 result = optparse.OptionParser.format_help(self, formatter)
 148                 result += self.tail
 149                 return result
 150
 151
 152 def ParseArgs(argv, qahelp):
 153         """This function uses a customized optionParser to parse command line arguments for repoman
 154         Args:
 155           argv - a sequence of command line arguments
 156                 qahelp - a dict of qa warning to help message
 157         Returns:
 158           (opts, args), just like a call to parser.parse_args()
 159         """
 160
 161         if argv and isinstance(argv[0], bytes):
 162                 argv = [portage._unicode_decode(x) for x in argv]
 163
 164         modes = {
 165                 'commit' : 'Run a scan then commit changes',
 166                 'ci' : 'Run a scan then commit changes',
 167                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 168                 'full' : 'Scan directory tree and print all issues (not a summary)',
 169                 'help' : 'Show this screen',
 170                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 171                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 172                 'scan' : 'Scan directory tree for QA issues'
 173         }
 174
 175         mode_keys = list(modes)
 176         mode_keys.sort()
 177
 178         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 179         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 180         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 181         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 182         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 183
 184         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 185                 help='Request a confirmation before commiting')
 186
 187         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 188                 help='specify a commit message on the command line')
 189
 190         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 191                 help='specify a path to a file that contains a commit message')
 192
 193         parser.add_option('--digest',
 194                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 195                 help='Automatically update Manifest digests for modified files')
 196
 197         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 198                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 199
 200         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 201                 help='do not print unnecessary messages')
 202
 203         parser.add_option(
 204                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 205                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 206                 'regardless of modification if \'force\' is specified)')
 207
 208         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 209                 help='Commit with QA violations')
 210
 211         parser.add_option('--vcs', dest='vcs',
 212                 help='Force using specific VCS instead of autodetection')
 213
 214         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 215                 help='be very verbose in output', default=0)
 216
 217         parser.add_option('-V', '--version', dest='version', action='store_true',
 218                 help='show version info')
 219
 220         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 221                 default=False, help='forces the metadata.xml parse check to be carried out')
 222
 223         parser.add_option(
 224                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 225                 metavar="<y|n>",
 226                 help='only check packages that have uncommitted modifications')
 227
 228         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 229                 default=False, help='ignore arch-specific failures (where arch != host)')
 230
 231         parser.add_option("--ignore-default-opts",
 232                 action="store_true",
 233                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 234
 235         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 236                 default=False, help='ignore masked packages (not allowed with commit mode)')
 237
 238         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 239                 default=False, help='include dev profiles in dependency checks')
 240
 241         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 242                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 243
 244         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 245                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 246
 247         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 248                 help='specify which mode repoman will run in (default=full)')
 249
 250         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 251
 252         for k in mode_keys:
 253                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 254
 255         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 256
 257         sorted_qa = list(qahelp)
 258         sorted_qa.sort()
 259         for k in sorted_qa:
 260                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 261
 262         opts, args = parser.parse_args(argv[1:])
 263
 264         if not opts.ignore_default_opts:
 265                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 266                 if default_opts:
 267                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 268
 269         if opts.mode == 'help':
 270                 parser.print_help(short=False)
 271
 272         for arg in args:
 273                 if arg in modes:
 274                         if not opts.mode:
 275                                 opts.mode = arg
 276                                 break
 277                 else:
 278                         parser.error("invalid mode: %s" % arg)
 279
 280         if not opts.mode:
 281                 opts.mode = 'full'
 282
 283         if opts.mode == 'ci':
 284                 opts.mode = 'commit'  # backwards compat shortcut
 285
 286         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 287                 if opts.ignore_masked:
 288                         parser.error('Commit mode and --ignore-masked are not compatible')
 289                 if opts.without_mask:
 290                         parser.error('Commit mode and --without-mask are not compatible')
 291
 292         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 293         for val in range(opts.verbosity):
 294                 logger = logging.getLogger()
 295                 logger.setLevel(logger.getEffectiveLevel() - 10)
 296
 297         for val in range(opts.quiet):
 298                 logger = logging.getLogger()
 299                 logger.setLevel(logger.getEffectiveLevel() + 10)
 300
 301         return (opts, args)
 302
 303 qahelp={
 304         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 305         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 306         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 307         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 308         "changelog.missing":"Missing ChangeLog files",
 309         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 310         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 311         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 312         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 313         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 314         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 315         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 316         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 317         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 318         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 319         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 320         "file.size":"Files in the files directory must be under 20 KiB",
 321         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 322         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 323         "file.UTF8":"File is not UTF8 compliant",
 324         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 325         "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it",
 326         "inherit.unused":"Ebuild inherits an eclass but does not use it",
 327         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 328         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 329         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 330         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 331         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 332         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 333         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 334         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 335         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 336         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 337         "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 338         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 339         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 340         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 341         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 342         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 343         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 344         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 345         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 346         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 347         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 348         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 349         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 350         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 351         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 352         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 353         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 354         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 355         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 356         "variable.readonly":"Assigning a readonly variable",
 357         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 358         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 359         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 360         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 361         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 362         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 363         "LICENSE.deprecated":"This ebuild is listing a deprecated license.",
 364         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 365         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 366         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 367         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 368         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 369         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 370         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 371         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 372         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 373         "ebuild.badheader":"This ebuild has a malformed header",
 374         "manifest.bad":"Manifest has missing or incorrect digests",
 375         "metadata.missing":"Missing metadata.xml files",
 376         "metadata.bad":"Bad metadata.xml files",
 377         "metadata.warning":"Warnings in metadata.xml files",
 378         "portage.internal":"The ebuild uses an internal Portage function or variable",
 379         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 380         "virtual.suspect":"Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 381         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 382         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 383 }
 384
 385 qacats = list(qahelp)
 386 qacats.sort()
 387
 388 qawarnings = set((
 389 "changelog.missing",
 390 "changelog.notadded",
 391 "dependency.unknown",
 392 "digest.assumed",
 393 "digest.unused",
 394 "ebuild.notadded",
 395 "ebuild.nesteddie",
 396 "dependency.badmasked",
 397 "dependency.badindev",
 398 "dependency.badmaskedindev",
 399 "dependency.badtilde",
 400 "DESCRIPTION.toolong",
 401 "EAPI.deprecated",
 402 "HOMEPAGE.virtual",
 403 "LICENSE.deprecated",
 404 "LICENSE.virtual",
 405 "KEYWORDS.dropped",
 406 "KEYWORDS.stupid",
 407 "KEYWORDS.missing",
 408 "PDEPEND.suspect",
 409 "RDEPEND.implicit",
 410 "RDEPEND.suspect",
 411 "virtual.suspect",
 412 "RESTRICT.invalid",
 413 "ebuild.minorsyn",
 414 "ebuild.badheader",
 415 "ebuild.patches",
 416 "file.size",
 417 "inherit.unused",
 418 "inherit.deprecated",
 419 "java.eclassesnotused",
 420 "wxwidgets.eclassnotused",
 421 "metadata.warning",
 422 "portage.internal",
 423 "usage.obsolete",
 424 "upstream.workaround",
 425 "LIVEVCS.stable",
 426 "LIVEVCS.unmasked",
 427 ))
 428
 429 if portage.const._ENABLE_INHERIT_CHECK:
 430         # This is experimental, so it's non-fatal.
 431         qawarnings.add("inherit.missing")
 432
 433 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 434
 435 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 436 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 437 allvars.update(Package.metadata_keys)
 438 allvars = sorted(allvars)
 439 commitmessage=None
 440 for x in missingvars:
 441         x += ".missing"
 442         if x not in qacats:
 443                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 444                 qacats.append(x)
 445                 qawarnings.add(x)
 446
 447 valid_restrict = frozenset(["binchecks", "bindist",
 448         "fetch", "installsources", "mirror",
 449         "primaryuri", "strip", "test", "userpriv"])
 450
 451 live_eclasses = frozenset([
 452         "bzr",
 453         "cvs",
 454         "darcs",
 455         "git",
 456         "git-2",
 457         "mercurial",
 458         "subversion",
 459         "tla",
 460 ])
 461
 462 suspect_rdepend = frozenset([
 463         "app-arch/cabextract",
 464         "app-arch/rpm2targz",
 465         "app-doc/doxygen",
 466         "dev-lang/nasm",
 467         "dev-lang/swig",
 468         "dev-lang/yasm",
 469         "dev-perl/extutils-pkgconfig",
 470         "dev-util/byacc",
 471         "dev-util/cmake",
 472         "dev-util/ftjam",
 473         "dev-util/gperf",
 474         "dev-util/gtk-doc",
 475         "dev-util/gtk-doc-am",
 476         "dev-util/intltool",
 477         "dev-util/jam",
 478         "dev-util/pkg-config-lite",
 479         "dev-util/pkgconf",
 480         "dev-util/pkgconfig",
 481         "dev-util/pkgconfig-openbsd",
 482         "dev-util/scons",
 483         "dev-util/unifdef",
 484         "dev-util/yacc",
 485         "media-gfx/ebdftopcf",
 486         "sys-apps/help2man",
 487         "sys-devel/autoconf",
 488         "sys-devel/automake",
 489         "sys-devel/bin86",
 490         "sys-devel/bison",
 491         "sys-devel/dev86",
 492         "sys-devel/flex",
 493         "sys-devel/m4",
 494         "sys-devel/pmake",
 495         "virtual/linux-sources",
 496         "virtual/pkgconfig",
 497         "x11-misc/bdftopcf",
 498         "x11-misc/imake",
 499 ])
 500
 501 suspect_virtual = {
 502         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 503         "dev-util/pkgconf":"virtual/pkgconfig",
 504         "dev-util/pkgconfig":"virtual/pkgconfig",
 505         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 506         "dev-libs/libusb":"virtual/libusb",
 507         "dev-libs/libusbx":"virtual/libusb",
 508         "dev-libs/libusb-compat":"virtual/libusb",
 509 }
 510
 511 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 512 # force refetch if the local copy creation time is older than this
 513 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 514
 515 # file.executable
 516 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 517
 518 options, arguments = ParseArgs(sys.argv, qahelp)
 519
 520 if options.version:
 521         print("Portage", portage.VERSION)
 522         sys.exit(0)
 523
 524 # Set this to False when an extraordinary issue (generally
 525 # something other than a QA issue) makes it impossible to
 526 # commit (like if Manifest generation fails).
 527 can_force = True
 528
 529 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 530 if portdir is None:
 531         sys.exit(1)
 532
 533 myreporoot = os.path.basename(portdir_overlay)
 534 myreporoot += mydir[len(portdir_overlay):]
 535
 536 if options.vcs:
 537         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 538                 vcs = options.vcs
 539         else:
 540                 vcs = None
 541 else:
 542         vcses = utilities.FindVCS()
 543         if len(vcses) > 1:
 544                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 545                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 546                 sys.exit(1)
 547         elif vcses:
 548                 vcs = vcses[0]
 549         else:
 550                 vcs = None
 551
 552 if options.if_modified == "y" and vcs is None:
 553         logging.info("Not in a version controlled repository; "
 554                 "disabling --if-modified.")
 555         options.if_modified = "n"
 556
 557 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 558 vcs_preserves_mtime = vcs in ('cvs',)
 559
 560 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 561 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 562 if vcs_global_opts is None:
 563         if vcs in ('cvs', 'svn'):
 564                 vcs_global_opts = "-q"
 565         else:
 566                 vcs_global_opts = ""
 567 vcs_global_opts = vcs_global_opts.split()
 568
 569 if options.mode == 'commit' and not options.pretend and not vcs:
 570         logging.info("Not in a version controlled repository; enabling pretend mode.")
 571         options.pretend = True
 572
 573 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 574 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 575         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 576         portage._shell_quote(portdir_overlay))
 577 # We have to call the config constructor again so
 578 # that config.repositories is initialized correctly.
 579 repoman_settings = portage.config(config_root=config_root, local_config=False,
 580         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 581
 582 root = repoman_settings['EROOT']
 583 trees = {
 584         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 585 }
 586 portdb = trees[root]['porttree'].dbapi
 587
 588 # Constrain dependency resolution to the master(s)
 589 # that are specified in layout.conf.
 590 repodir = os.path.realpath(portdir_overlay)
 591 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 592 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 593 portdir = portdb.porttrees[0]
 594
 595 if repo_config.allow_provide_virtual:
 596         qawarnings.add("virtual.oldstyle")
 597
 598 if repo_config.sign_commit:
 599         if vcs == 'git':
 600                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 601                 # the commit arguments. If key_id is unspecified, then it must be
 602                 # configured by `git config user.signingkey key_id`.
 603                 vcs_local_opts.append("--gpg-sign")
 604
 605 # In order to disable manifest signatures, repos may set
 606 # "sign-manifests = false" in metadata/layout.conf. This
 607 # can be used to prevent merge conflicts like those that
 608 # thin-manifests is designed to prevent.
 609 sign_manifests = "sign" in repoman_settings.features and \
 610         repo_config.sign_manifest
 611
 612 if sign_manifests and options.mode in ("commit",) and \
 613         repoman_settings.get("PORTAGE_GPG_KEY") and \
 614         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 615         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 616         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 617                 repoman_settings["PORTAGE_GPG_KEY"])
 618         sys.exit(1)
 619
 620 manifest_hashes = repo_config.manifest_hashes
 621 if manifest_hashes is None:
 622         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 623
 624 if options.mode in ("commit", "fix", "manifest"):
 625         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 626                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 627                 "metadata/layout.conf does not contain the '%s' hash which "
 628                 "is required by this portage version. You will have to "
 629                 "upgrade portage if you want to generate valid manifests for "
 630                 "this repository.") % \
 631                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 632                 for line in textwrap.wrap(msg, 70):
 633                         logging.error(line)
 634                 sys.exit(1)
 635
 636         unsupported_hashes = manifest_hashes.difference(
 637                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 638         if unsupported_hashes:
 639                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 640                 "metadata/layout.conf contains one or more hash types '%s' "
 641                 "which are not supported by this portage version. You will "
 642                 "have to upgrade portage if you want to generate valid "
 643                 "manifests for this repository.") % \
 644                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 645                 for line in textwrap.wrap(msg, 70):
 646                         logging.error(line)
 647                 sys.exit(1)
 648
 649 if options.echangelog is None and repo_config.update_changelog:
 650         options.echangelog = 'y'
 651
 652 if vcs is None:
 653         options.echangelog = 'n'
 654
 655 # The --echangelog option causes automatic ChangeLog generation,
 656 # which invalidates changelog.ebuildadded and changelog.missing
 657 # checks.
 658 # Note: Some don't use ChangeLogs in distributed SCMs.
 659 # It will be generated on server side from scm log,
 660 # before package moves to the rsync server.
 661 # This is needed because they try to avoid merge collisions.
 662 # Gentoo's Council decided to always use the ChangeLog file.
 663 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 664 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 665
 666 if 'digest' in repoman_settings.features and options.digest != 'n':
 667         options.digest = 'y'
 668
 669 logging.debug("vcs: %s" % (vcs,))
 670 logging.debug("repo config: %s" % (repo_config,))
 671 logging.debug("options: %s" % (options,))
 672
 673 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 674 # profile-specific config constructor calls.
 675 env = os.environ.copy()
 676 env['PORTDIR'] = portdir
 677 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 678
 679 logging.info('Setting paths:')
 680 logging.info('PORTDIR = "' + portdir + '"')
 681 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 682
 683 # It's confusing if these warnings are displayed without the user
 684 # being told which profile they come from, so disable them.
 685 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 686
 687 categories = []
 688 for path in repo_config.eclass_db.porttrees:
 689         categories.extend(portage.util.grabfile(
 690                 os.path.join(path, 'profiles', 'categories')))
 691 repoman_settings.categories = frozenset(
 692         portage.util.stack_lists([categories], incremental=1))
 693 categories = repoman_settings.categories
 694
 695 portdb.settings = repoman_settings
 696 root_config = RootConfig(repoman_settings, trees[root], None)
 697 # We really only need to cache the metadata that's necessary for visibility
 698 # filtering. Anything else can be discarded to reduce memory consumption.
 699 portdb._aux_cache_keys.clear()
 700 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 701
 702 reposplit = myreporoot.split(os.path.sep)
 703 repolevel = len(reposplit)
 704
 705 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 706 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 707 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 708 if options.mode == 'commit' and repolevel not in [1,2,3]:
 709         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 710         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 711         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 712         print(red("***"))
 713         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 714
 715 # Make startdir relative to the canonical repodir, so that we can pass
 716 # it to digestgen and it won't have to be canonicalized again.
 717 if repolevel == 1:
 718         startdir = repodir
 719 else:
 720         startdir = normalize_path(mydir)
 721         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 722
 723 def caterror(mycat):
 724         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 725
 726 def repoman_getstatusoutput(cmd):
 727         """
 728         Implements an interface similar to getstatusoutput(), but with
 729         customized unicode handling (see bug #310789) and without the shell.
 730         """
 731         args = portage.util.shlex_split(cmd)
 732         encoding = _encodings['fs']
 733         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 734                 # Python 3.1 does not support bytes in Popen args.
 735                 args = [_unicode_encode(x,
 736                         encoding=encoding, errors='strict') for x in args]
 737         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 738                 stderr=subprocess.STDOUT)
 739         output = portage._unicode_decode(proc.communicate()[0],
 740                 encoding=encoding, errors='strict')
 741         if output and output[-1] == "\n":
 742                 # getstatusoutput strips one newline
 743                 output = output[:-1]
 744         return (proc.wait(), output)
 745
 746 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 747         """
 748         Implements an interface similar to os.popen(), but with customized
 749         unicode handling (see bug #310789) and without the shell.
 750         """
 751
 752         __slots__ = ('_proc', '_stdout')
 753
 754         def __init__(self, cmd):
 755                 args = portage.util.shlex_split(cmd)
 756                 encoding = _encodings['fs']
 757                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 758                         # Python 3.1 does not support bytes in Popen args.
 759                         args = [_unicode_encode(x,
 760                                 encoding=encoding, errors='strict') for x in args]
 761                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 762                 object.__setattr__(self, '_proc', proc)
 763                 object.__setattr__(self, '_stdout',
 764                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 765
 766         def _get_target(self):
 767                 return object.__getattribute__(self, '_stdout')
 768
 769         __enter__ = _get_target
 770
 771         def __exit__(self, exc_type, exc_value, traceback):
 772                 proc = object.__getattribute__(self, '_proc')
 773                 proc.wait()
 774                 proc.stdout.close()
 775
 776 class ProfileDesc(object):
 777         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 778         def __init__(self, arch, status, sub_path, tree_path):
 779                 self.arch = arch
 780                 self.status = status
 781                 if sub_path:
 782                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 783                 self.sub_path = sub_path
 784                 self.tree_path = tree_path
 785                 if tree_path:
 786                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 787                 else:
 788                         self.abs_path = tree_path
 789
 790         def __str__(self):
 791                 if self.sub_path:
 792                         return self.sub_path
 793                 return 'empty profile'
 794
 795 profile_list = []
 796 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 797
 798 # get lists of valid keywords, licenses, and use
 799 kwlist = set()
 800 liclist = set()
 801 uselist = set()
 802 global_pmasklines = []
 803
 804 for path in portdb.porttrees:
 805         try:
 806                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 807         except OSError:
 808                 pass
 809         kwlist.update(portage.grabfile(os.path.join(path,
 810                 "profiles", "arch.list")))
 811
 812         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 813         for x in use_desc:
 814                 x = x.split()
 815                 if x:
 816                         uselist.add(x[0])
 817
 818         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 819         try:
 820                 expand_list = os.listdir(expand_desc_dir)
 821         except OSError:
 822                 pass
 823         else:
 824                 for fn in expand_list:
 825                         if not fn[-5:] == '.desc':
 826                                 continue
 827                         use_prefix = fn[:-5].lower() + '_'
 828                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 829                                 x = x.split()
 830                                 if x:
 831                                         uselist.add(use_prefix + x[0])
 832
 833         global_pmasklines.append(portage.util.grabfile_package(
 834                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 835
 836         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 837         try:
 838                 desc_file = io.open(_unicode_encode(desc_path,
 839                         encoding=_encodings['fs'], errors='strict'),
 840                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 841         except EnvironmentError:
 842                 pass
 843         else:
 844                 for i, x in enumerate(desc_file):
 845                         if x[0] == "#":
 846                                 continue
 847                         arch = x.split()
 848                         if len(arch) == 0:
 849                                 continue
 850                         if len(arch) != 3:
 851                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 852                                         desc_path + " line %d" % (i+1, ))
 853                         elif arch[0] not in kwlist:
 854                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 855                                         desc_path + " line %d" % (i+1, ))
 856                         elif arch[2] not in valid_profile_types:
 857                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 858                                         desc_path + " line %d" % (i+1, ))
 859                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 860                         if not os.path.isdir(profile_desc.abs_path):
 861                                 logging.error(
 862                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 863                                         arch[2], arch[1], arch[0], desc_path, i+1)
 864                                 continue
 865                         if os.path.exists(
 866                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 867                                 continue
 868                         profile_list.append(profile_desc)
 869                 desc_file.close()
 870
 871 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 872 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 873
 874 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 875 global_pmaskdict = {}
 876 for x in global_pmasklines:
 877         global_pmaskdict.setdefault(x.cp, []).append(x)
 878 del global_pmasklines
 879
 880 def has_global_mask(pkg):
 881         mask_atoms = global_pmaskdict.get(pkg.cp)
 882         if mask_atoms:
 883                 pkg_list = [pkg]
 884                 for x in mask_atoms:
 885                         if portage.dep.match_from_list(x, pkg_list):
 886                                 return x
 887         return None
 888
 889 # Ensure that profile sub_path attributes are unique. Process in reverse order
 890 # so that profiles with duplicate sub_path from overlays will override
 891 # profiles with the same sub_path from parent repos.
 892 profiles = {}
 893 profile_list.reverse()
 894 profile_sub_paths = set()
 895 for prof in profile_list:
 896         if prof.sub_path in profile_sub_paths:
 897                 continue
 898         profile_sub_paths.add(prof.sub_path)
 899         profiles.setdefault(prof.arch, []).append(prof)
 900
 901 # Use an empty profile for checking dependencies of
 902 # packages that have empty KEYWORDS.
 903 prof = ProfileDesc('**', 'stable', '', '')
 904 profiles.setdefault(prof.arch, []).append(prof)
 905
 906 for x in repoman_settings.archlist():
 907         if x[0] == "~":
 908                 continue
 909         if x not in profiles:
 910                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 911                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 912                 print(red("up with the "+x+" team."))
 913                 print()
 914
 915 liclist_deprecated = set()
 916 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 917         liclist_deprecated.update(
 918                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 919
 920 if not liclist:
 921         logging.fatal("Couldn't find licenses?")
 922         sys.exit(1)
 923
 924 if not kwlist:
 925         logging.fatal("Couldn't read KEYWORDS from arch.list")
 926         sys.exit(1)
 927
 928 if not uselist:
 929         logging.fatal("Couldn't find use.desc?")
 930         sys.exit(1)
 931
 932 scanlist=[]
 933 if repolevel==2:
 934         #we are inside a category directory
 935         catdir=reposplit[-1]
 936         if catdir not in categories:
 937                 caterror(catdir)
 938         mydirlist=os.listdir(startdir)
 939         for x in mydirlist:
 940                 if x == "CVS" or x.startswith("."):
 941                         continue
 942                 if os.path.isdir(startdir+"/"+x):
 943                         scanlist.append(catdir+"/"+x)
 944         repo_subdir = catdir + os.sep
 945 elif repolevel==1:
 946         for x in categories:
 947                 if not os.path.isdir(startdir+"/"+x):
 948                         continue
 949                 for y in os.listdir(startdir+"/"+x):
 950                         if y == "CVS" or y.startswith("."):
 951                                 continue
 952                         if os.path.isdir(startdir+"/"+x+"/"+y):
 953                                 scanlist.append(x+"/"+y)
 954         repo_subdir = ""
 955 elif repolevel==3:
 956         catdir = reposplit[-2]
 957         if catdir not in categories:
 958                 caterror(catdir)
 959         scanlist.append(catdir+"/"+reposplit[-1])
 960         repo_subdir = scanlist[-1] + os.sep
 961 else:
 962         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 963                 ' from the current working directory'
 964         logging.critical(msg)
 965         sys.exit(1)
 966
 967 repo_subdir_len = len(repo_subdir)
 968 scanlist.sort()
 969
 970 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 971
 972 def vcs_files_to_cps(vcs_file_iter):
 973         """
 974         Iterate over the given modified file paths returned from the vcs,
 975         and return a frozenset containing category/pn strings for each
 976         modified package.
 977         """
 978
 979         modified_cps = []
 980
 981         if repolevel == 3:
 982                 if reposplit[-2] in categories and \
 983                         next(vcs_file_iter, None) is not None:
 984                         modified_cps.append("/".join(reposplit[-2:]))
 985
 986         elif repolevel == 2:
 987                 category = reposplit[-1]
 988                 if category in categories:
 989                         for filename in vcs_file_iter:
 990                                 f_split = filename.split(os.sep)
 991                                 # ['.', pn,...]
 992                                 if len(f_split) > 2:
 993                                         modified_cps.append(category + "/" + f_split[1])
 994
 995         else:
 996                 # repolevel == 1
 997                 for filename in vcs_file_iter:
 998                         f_split = filename.split(os.sep)
 999                         # ['.', category, pn,...]
1000                         if len(f_split) > 3 and f_split[1] in categories:
1001                                 modified_cps.append("/".join(f_split[1:3]))
1002
1003         return frozenset(modified_cps)
1004
1005 def git_supports_gpg_sign():
1006         status, cmd_output = \
1007                 repoman_getstatusoutput("git --version")
1008         cmd_output = cmd_output.split()
1009         if cmd_output:
1010                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1011                 if version is not None:
1012                         version = [int(x) for x in version.groups()]
1013                         if version[0] > 1 or \
1014                                 (version[0] == 1 and version[1] > 7) or \
1015                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1016                                 return True
1017         return False
1018
1019 def dev_keywords(profiles):
1020         """
1021         Create a set of KEYWORDS values that exist in 'dev'
1022         profiles. These are used
1023         to trigger a message notifying the user when they might
1024         want to add the --include-dev option.
1025         """
1026         type_arch_map = {}
1027         for arch, arch_profiles in profiles.items():
1028                 for prof in arch_profiles:
1029                         arch_set = type_arch_map.get(prof.status)
1030                         if arch_set is None:
1031                                 arch_set = set()
1032                                 type_arch_map[prof.status] = arch_set
1033                         arch_set.add(arch)
1034
1035         dev_keywords = type_arch_map.get('dev', set())
1036         dev_keywords.update(['~' + arch for arch in dev_keywords])
1037         return frozenset(dev_keywords)
1038
1039 dev_keywords = dev_keywords(profiles)
1040
1041 stats={}
1042 fails={}
1043
1044 for x in qacats:
1045         stats[x]=0
1046         fails[x]=[]
1047
1048 xmllint_capable = False
1049 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1050
1051 def parsedate(s):
1052         """Parse a RFC 822 date and time string.
1053         This is required for python3 compatibility, since the
1054         rfc822.parsedate() function is not available."""
1055
1056         s_split = []
1057         for x in s.upper().split():
1058                 for y in x.split(','):
1059                         if y:
1060                                 s_split.append(y)
1061
1062         if len(s_split) != 6:
1063                 return None
1064
1065         # %a, %d %b %Y %H:%M:%S %Z
1066         a, d, b, Y, H_M_S, Z = s_split
1067
1068         # Convert month to integer, since strptime %w is locale-dependent.
1069         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1070                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1071         m = month_map.get(b)
1072         if m is None:
1073                 return None
1074         m = "%02d" % m
1075
1076         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1077
1078 def fetch_metadata_dtd():
1079         """
1080         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1081         metadata_dtd_ctime_interval.
1082         @rtype: bool
1083         @return: True if successful, otherwise False
1084         """
1085
1086         must_fetch = True
1087         metadata_dtd_st = None
1088         current_time = int(time.time())
1089         try:
1090                 metadata_dtd_st = os.stat(metadata_dtd)
1091         except EnvironmentError as e:
1092                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1093                         raise
1094                 del e
1095         else:
1096                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1097                 if abs(current_time - metadata_dtd_st.st_ctime) \
1098                         < metadata_dtd_ctime_interval:
1099                         must_fetch = False
1100
1101         if must_fetch:
1102                 print()
1103                 print(green("***") + " the local copy of metadata.dtd " + \
1104                         "needs to be refetched, doing that now")
1105                 print()
1106                 try:
1107                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1108                         msg_info = url_f.info()
1109                         last_modified = msg_info.get('last-modified')
1110                         if last_modified is not None:
1111                                 last_modified = parsedate(last_modified)
1112                                 if last_modified is not None:
1113                                         last_modified = calendar.timegm(last_modified)
1114
1115                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1116                         try:
1117                                 local_f = open(metadata_dtd_tmp, mode='wb')
1118                                 local_f.write(url_f.read())
1119                                 local_f.close()
1120                                 if last_modified is not None:
1121                                         try:
1122                                                 os.utime(metadata_dtd_tmp,
1123                                                         (int(last_modified), int(last_modified)))
1124                                         except OSError:
1125                                                 # This fails on some odd non-unix-like filesystems.
1126                                                 # We don't really need the mtime to be preserved
1127                                                 # anyway here (currently we use ctime to trigger
1128                                                 # fetch), so just ignore it.
1129                                                 pass
1130                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1131                         finally:
1132                                 try:
1133                                         os.unlink(metadata_dtd_tmp)
1134                                 except OSError:
1135                                         pass
1136
1137                         url_f.close()
1138
1139                 except EnvironmentError as e:
1140                         print()
1141                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1142                         print(red("!!!")+" exception '%s' though." % (e,))
1143                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1144                         return False
1145
1146         return True
1147
1148 if options.mode == "manifest":
1149         pass
1150 elif not find_binary('xmllint'):
1151         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1152         if options.xml_parse or repolevel==3:
1153                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1154                 sys.exit(1)
1155 else:
1156         if not fetch_metadata_dtd():
1157                 sys.exit(1)
1158         #this can be problematic if xmllint changes their output
1159         xmllint_capable=True
1160
1161 if options.mode == 'commit' and vcs:
1162         utilities.detect_vcs_conflicts(options, vcs)
1163
1164 if options.mode == "manifest":
1165         pass
1166 elif options.pretend:
1167         print(green("\nRepoMan does a once-over of the neighborhood..."))
1168 else:
1169         print(green("\nRepoMan scours the neighborhood..."))
1170
1171 new_ebuilds = set()
1172 modified_ebuilds = set()
1173 modified_changelogs = set()
1174 mychanged = []
1175 mynew = []
1176 myremoved = []
1177
1178 if vcs == "cvs":
1179         mycvstree = cvstree.getentries("./", recursive=1)
1180         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1181         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1182         if options.if_modified == "y":
1183                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1184
1185 elif vcs == "svn":
1186         with repoman_popen("svn status") as f:
1187                 svnstatus = f.readlines()
1188         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1189         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1190         if options.if_modified == "y":
1191                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1192
1193 elif vcs == "git":
1194         with repoman_popen("git diff-index --name-only "
1195                 "--relative --diff-filter=M HEAD") as f:
1196                 mychanged = f.readlines()
1197         mychanged = ["./" + elem[:-1] for elem in mychanged]
1198
1199         with repoman_popen("git diff-index --name-only "
1200                 "--relative --diff-filter=A HEAD") as f:
1201                 mynew = f.readlines()
1202         mynew = ["./" + elem[:-1] for elem in mynew]
1203         if options.if_modified == "y":
1204                 with repoman_popen("git diff-index --name-only "
1205                         "--relative --diff-filter=D HEAD") as f:
1206                         myremoved = f.readlines()
1207                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1208
1209 elif vcs == "bzr":
1210         with repoman_popen("bzr status -S .") as f:
1211                 bzrstatus = f.readlines()
1212         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1213         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1214         if options.if_modified == "y":
1215                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1216
1217 elif vcs == "hg":
1218         with repoman_popen("hg status --no-status --modified .") as f:
1219                 mychanged = f.readlines()
1220         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1221         with repoman_popen("hg status --no-status --added .") as f:
1222                 mynew = f.readlines()
1223         mynew = ["./" + elem.rstrip() for elem in mynew]
1224         if options.if_modified == "y":
1225                 with repoman_popen("hg status --no-status --removed .") as f:
1226                         myremoved = f.readlines()
1227                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1228
1229 if vcs:
1230         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1231         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1232         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1233                 if os.path.basename(x) == "ChangeLog")
1234
1235 def vcs_new_changed(relative_path):
1236         for x in chain(mychanged, mynew):
1237                 if x == relative_path:
1238                         return True
1239         return False
1240
1241 have_pmasked = False
1242 have_dev_keywords = False
1243 dofail = 0
1244
1245 # NOTE: match-all caches are not shared due to potential
1246 # differences between profiles in _get_implicit_iuse.
1247 arch_caches={}
1248 arch_xmatch_caches = {}
1249 shared_xmatch_caches = {"cp-list":{}}
1250
1251 # Disable the "ebuild.notadded" check when not in commit mode and
1252 # running `svn status` in every package dir will be too expensive.
1253
1254 check_ebuild_notadded = not \
1255         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1256
1257 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1258 thirdpartymirrors = {}
1259 for k, v in repoman_settings.thirdpartymirrors().items():
1260         for v in v:
1261                 if not v.endswith("/"):
1262                         v += "/"
1263                 thirdpartymirrors[v] = k
1264
1265 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1266         """
1267         Implements doctype() as required to avoid deprecation warnings with
1268         >=python-2.7.
1269         """
1270         def doctype(self, name, pubid, system):
1271                 pass
1272
1273 try:
1274         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1275 except (EnvironmentError, ParseError, PermissionDenied) as e:
1276         err(str(e))
1277 except FileNotFound:
1278         # TODO: Download as we do for metadata.dtd, but add a way to
1279         # disable for non-gentoo repoman users who may not have herds.
1280         herd_base = None
1281
1282 effective_scanlist = scanlist
1283 if options.if_modified == "y":
1284         effective_scanlist = sorted(vcs_files_to_cps(
1285                 chain(mychanged, mynew, myremoved)))
1286
1287 for x in effective_scanlist:
1288         #ebuilds and digests added to cvs respectively.
1289         logging.info("checking package %s" % x)
1290         # save memory by discarding xmatch caches from previous package(s)
1291         arch_xmatch_caches.clear()
1292         eadded=[]
1293         catdir,pkgdir=x.split("/")
1294         checkdir=repodir+"/"+x
1295         checkdir_relative = ""
1296         if repolevel < 3:
1297                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1298         if repolevel < 2:
1299                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1300         checkdir_relative = os.path.join(".", checkdir_relative)
1301         generated_manifest = False
1302
1303         if options.mode == "manifest" or \
1304           (options.mode != 'manifest-check' and options.digest == 'y') or \
1305           options.mode in ('commit', 'fix') and not options.pretend:
1306                 auto_assumed = set()
1307                 fetchlist_dict = portage.FetchlistDict(checkdir,
1308                         repoman_settings, portdb)
1309                 if options.mode == 'manifest' and options.force:
1310                         portage._doebuild_manifest_exempt_depend += 1
1311                         try:
1312                                 distdir = repoman_settings['DISTDIR']
1313                                 mf = repoman_settings.repositories.get_repo_for_location(
1314                                         os.path.dirname(os.path.dirname(checkdir)))
1315                                 mf = mf.load_manifest(checkdir, distdir,
1316                                         fetchlist_dict=fetchlist_dict)
1317                                 mf.create(requiredDistfiles=None,
1318                                         assumeDistHashesAlways=True)
1319                                 for distfiles in fetchlist_dict.values():
1320                                         for distfile in distfiles:
1321                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1322                                                         mf.fhashdict['DIST'].pop(distfile, None)
1323                                                 else:
1324                                                         auto_assumed.add(distfile)
1325                                 mf.write()
1326                         finally:
1327                                 portage._doebuild_manifest_exempt_depend -= 1
1328
1329                 repoman_settings["O"] = checkdir
1330                 try:
1331                         generated_manifest = digestgen(
1332                                 mysettings=repoman_settings, myportdb=portdb)
1333                 except portage.exception.PermissionDenied as e:
1334                         generated_manifest = False
1335                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1336                                 level=logging.ERROR, noiselevel=-1)
1337
1338                 if not generated_manifest:
1339                         print("Unable to generate manifest.")
1340                         dofail = 1
1341
1342                 if options.mode == "manifest":
1343                         if not dofail and options.force and auto_assumed and \
1344                                 'assume-digests' in repoman_settings.features:
1345                                 # Show which digests were assumed despite the --force option
1346                                 # being given. This output will already have been shown by
1347                                 # digestgen() if assume-digests is not enabled, so only show
1348                                 # it here if assume-digests is enabled.
1349                                 pkgs = list(fetchlist_dict)
1350                                 pkgs.sort()
1351                                 portage.writemsg_stdout("  digest.assumed" + \
1352                                         portage.output.colorize("WARN",
1353                                         str(len(auto_assumed)).rjust(18)) + "\n")
1354                                 for cpv in pkgs:
1355                                         fetchmap = fetchlist_dict[cpv]
1356                                         pf = portage.catsplit(cpv)[1]
1357                                         for distfile in sorted(fetchmap):
1358                                                 if distfile in auto_assumed:
1359                                                         portage.writemsg_stdout(
1360                                                                 "   %s::%s\n" % (pf, distfile))
1361                         continue
1362                 elif dofail:
1363                         sys.exit(1)
1364
1365         if not generated_manifest:
1366                 repoman_settings['O'] = checkdir
1367                 repoman_settings['PORTAGE_QUIET'] = '1'
1368                 if not portage.digestcheck([], repoman_settings, strict=1):
1369                         stats["manifest.bad"] += 1
1370                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1371                 repoman_settings.pop('PORTAGE_QUIET', None)
1372
1373         if options.mode == 'manifest-check':
1374                 continue
1375
1376         checkdirlist=os.listdir(checkdir)
1377         ebuildlist=[]
1378         pkgs = {}
1379         allvalid = True
1380         for y in checkdirlist:
1381                 if (y in no_exec or y.endswith(".ebuild")) and \
1382                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1383                                 stats["file.executable"] += 1
1384                                 fails["file.executable"].append(os.path.join(checkdir, y))
1385                 if y.endswith(".ebuild"):
1386                         pf = y[:-7]
1387                         ebuildlist.append(pf)
1388                         cpv = "%s/%s" % (catdir, pf)
1389                         try:
1390                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1391                         except KeyError:
1392                                 allvalid = False
1393                                 stats["ebuild.syntax"] += 1
1394                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1395                                 continue
1396                         except IOError:
1397                                 allvalid = False
1398                                 stats["ebuild.output"] += 1
1399                                 fails["ebuild.output"].append(os.path.join(x, y))
1400                                 continue
1401                         if not portage.eapi_is_supported(myaux["EAPI"]):
1402                                 allvalid = False
1403                                 stats["EAPI.unsupported"] += 1
1404                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1405                                 continue
1406                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1407                                 root_config=root_config, type_name="ebuild")
1408
1409         slot_keywords = {}
1410
1411         if len(pkgs) != len(ebuildlist):
1412                 # If we can't access all the metadata then it's totally unsafe to
1413                 # commit since there's no way to generate a correct Manifest.
1414                 # Do not try to do any more QA checks on this package since missing
1415                 # metadata leads to false positives for several checks, and false
1416                 # positives confuse users.
1417                 can_force = False
1418                 continue
1419
1420         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1421         ebuildlist = sorted(pkgs.values())
1422         ebuildlist = [pkg.pf for pkg in ebuildlist]
1423
1424         for y in checkdirlist:
1425                 index = repo_config.find_invalid_path_char(y)
1426                 if index != -1:
1427                         y_relative = os.path.join(checkdir_relative, y)
1428                         if vcs is not None and not vcs_new_changed(y_relative):
1429                                 # If the file isn't in the VCS new or changed set, then
1430                                 # assume that it's an irrelevant temporary file (Manifest
1431                                 # entries are not generated for file names containing
1432                                 # prohibited characters). See bug #406877.
1433                                 index = -1
1434                 if index != -1:
1435                         stats["file.name"] += 1
1436                         fails["file.name"].append("%s/%s: char '%s'" % \
1437                                 (checkdir, y, y[index]))
1438
1439                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1440                         continue
1441                 f = None
1442                 try:
1443                         line = 1
1444                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1445                                 encoding=_encodings['fs'], errors='strict'),
1446                                 mode='r', encoding=_encodings['repo.content'])
1447                         for l in f:
1448                                 line +=1
1449                 except UnicodeDecodeError as ue:
1450                         stats["file.UTF8"] += 1
1451                         s = ue.object[:ue.start]
1452                         l2 = s.count("\n")
1453                         line += l2
1454                         if l2 != 0:
1455                                 s = s[s.rfind("\n") + 1:]
1456                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1457                 finally:
1458                         if f is not None:
1459                                 f.close()
1460
1461         if vcs in ("git", "hg") and check_ebuild_notadded:
1462                 if vcs == "git":
1463                         myf = repoman_popen("git ls-files --others %s" % \
1464                                 (portage._shell_quote(checkdir_relative),))
1465                 if vcs == "hg":
1466                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1467                                 (portage._shell_quote(checkdir_relative),))
1468                 for l in myf:
1469                         if l[:-1][-7:] == ".ebuild":
1470                                 stats["ebuild.notadded"] += 1
1471                                 fails["ebuild.notadded"].append(
1472                                         os.path.join(x, os.path.basename(l[:-1])))
1473                 myf.close()
1474
1475         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1476                 try:
1477                         if vcs == "cvs":
1478                                 myf=open(checkdir+"/CVS/Entries","r")
1479                         if vcs == "svn":
1480                                 myf = repoman_popen("svn status --depth=files --verbose " +
1481                                         portage._shell_quote(checkdir))
1482                         if vcs == "bzr":
1483                                 myf = repoman_popen("bzr ls -v --kind=file " +
1484                                         portage._shell_quote(checkdir))
1485                         myl = myf.readlines()
1486                         myf.close()
1487                         for l in myl:
1488                                 if vcs == "cvs":
1489                                         if l[0]!="/":
1490                                                 continue
1491                                         splitl=l[1:].split("/")
1492                                         if not len(splitl):
1493                                                 continue
1494                                         if splitl[0][-7:]==".ebuild":
1495                                                 eadded.append(splitl[0][:-7])
1496                                 if vcs == "svn":
1497                                         if l[:1] == "?":
1498                                                 continue
1499                                         if l[:7] == '      >':
1500                                                 # tree conflict, new in subversion 1.6
1501                                                 continue
1502                                         l = l.split()[-1]
1503                                         if l[-7:] == ".ebuild":
1504                                                 eadded.append(os.path.basename(l[:-7]))
1505                                 if vcs == "bzr":
1506                                         if l[1:2] == "?":
1507                                                 continue
1508                                         l = l.split()[-1]
1509                                         if l[-7:] == ".ebuild":
1510                                                 eadded.append(os.path.basename(l[:-7]))
1511                         if vcs == "svn":
1512                                 myf = repoman_popen("svn status " +
1513                                         portage._shell_quote(checkdir))
1514                                 myl=myf.readlines()
1515                                 myf.close()
1516                                 for l in myl:
1517                                         if l[0] == "A":
1518                                                 l = l.rstrip().split(' ')[-1]
1519                                                 if l[-7:] == ".ebuild":
1520                                                         eadded.append(os.path.basename(l[:-7]))
1521                 except IOError:
1522                         if vcs == "cvs":
1523                                 stats["CVS/Entries.IO_error"] += 1
1524                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1525                         else:
1526                                 raise
1527                         continue
1528
1529         mf = repoman_settings.repositories.get_repo_for_location(
1530                 os.path.dirname(os.path.dirname(checkdir)))
1531         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1532         mydigests=mf.getTypeDigests("DIST")
1533
1534         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1535         myfiles_all = []
1536         src_uri_error = False
1537         for mykey in fetchlist_dict:
1538                 try:
1539                         myfiles_all.extend(fetchlist_dict[mykey])
1540                 except portage.exception.InvalidDependString as e:
1541                         src_uri_error = True
1542                         try:
1543                                 portdb.aux_get(mykey, ["SRC_URI"])
1544                         except KeyError:
1545                                 # This will be reported as an "ebuild.syntax" error.
1546                                 pass
1547                         else:
1548                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1549                                 fails["SRC_URI.syntax"].append(
1550                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1551         del fetchlist_dict
1552         if not src_uri_error:
1553                 # This test can produce false positives if SRC_URI could not
1554                 # be parsed for one or more ebuilds. There's no point in
1555                 # producing a false error here since the root cause will
1556                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1557                 # or "ebuild.sytax".
1558                 myfiles_all = set(myfiles_all)
1559                 for entry in mydigests:
1560                         if entry not in myfiles_all:
1561                                 stats["digest.unused"] += 1
1562                                 fails["digest.unused"].append(checkdir+"::"+entry)
1563                 for entry in myfiles_all:
1564                         if entry not in mydigests:
1565                                 stats["digest.missing"] += 1
1566                                 fails["digest.missing"].append(checkdir+"::"+entry)
1567         del myfiles_all
1568
1569         if os.path.exists(checkdir+"/files"):
1570                 filesdirlist=os.listdir(checkdir+"/files")
1571
1572                 # recurse through files directory
1573                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1574                 while filesdirlist:
1575                         y = filesdirlist.pop(0)
1576                         relative_path = os.path.join(x, "files", y)
1577                         full_path = os.path.join(repodir, relative_path)
1578                         try:
1579                                 mystat = os.stat(full_path)
1580                         except OSError as oe:
1581                                 if oe.errno == 2:
1582                                         # don't worry about it.  it likely was removed via fix above.
1583                                         continue
1584                                 else:
1585                                         raise oe
1586                         if S_ISDIR(mystat.st_mode):
1587                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1588                                 if y == "CVS" or y == ".svn":
1589                                         continue
1590                                 for z in os.listdir(checkdir+"/files/"+y):
1591                                         if z == "CVS" or z == ".svn":
1592                                                 continue
1593                                         filesdirlist.append(y+"/"+z)
1594                         # Current policy is no files over 20 KiB, these are the checks. File size between
1595                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1596                         elif mystat.st_size > 61440:
1597                                 stats["file.size.fatal"] += 1
1598                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1599                         elif mystat.st_size > 20480:
1600                                 stats["file.size"] += 1
1601                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1602
1603                         index = repo_config.find_invalid_path_char(y)
1604                         if index != -1:
1605                                 y_relative = os.path.join(checkdir_relative, "files", y)
1606                                 if vcs is not None and not vcs_new_changed(y_relative):
1607                                         # If the file isn't in the VCS new or changed set, then
1608                                         # assume that it's an irrelevant temporary file (Manifest
1609                                         # entries are not generated for file names containing
1610                                         # prohibited characters). See bug #406877.
1611                                         index = -1
1612                         if index != -1:
1613                                 stats["file.name"] += 1
1614                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1615                                         (checkdir, y, y[index]))
1616         del mydigests
1617
1618         if check_changelog and "ChangeLog" not in checkdirlist:
1619                 stats["changelog.missing"]+=1
1620                 fails["changelog.missing"].append(x+"/ChangeLog")
1621
1622         musedict = {}
1623         #metadata.xml file check
1624         if "metadata.xml" not in checkdirlist:
1625                 stats["metadata.missing"]+=1
1626                 fails["metadata.missing"].append(x+"/metadata.xml")
1627         #metadata.xml parse check
1628         else:
1629                 metadata_bad = False
1630
1631                 # read metadata.xml into memory
1632                 try:
1633                         _metadata_xml = xml.etree.ElementTree.parse(
1634                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1635                                 encoding=_encodings['fs'], errors='strict'),
1636                                 parser=xml.etree.ElementTree.XMLParser(
1637                                         target=_MetadataTreeBuilder()))
1638                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1639                         metadata_bad = True
1640                         stats["metadata.bad"] += 1
1641                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1642                         del e
1643                 else:
1644                         # load USE flags from metadata.xml
1645                         try:
1646                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1647                         except portage.exception.ParseError as e:
1648                                 metadata_bad = True
1649                                 stats["metadata.bad"] += 1
1650                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1651                         else:
1652                                 for atom in chain(*musedict.values()):
1653                                         if atom is None:
1654                                                 continue
1655                                         try:
1656                                                 atom = Atom(atom)
1657                                         except InvalidAtom as e:
1658                                                 stats["metadata.bad"] += 1
1659                                                 fails["metadata.bad"].append(
1660                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1661                                         else:
1662                                                 if atom.cp != x:
1663                                                         stats["metadata.bad"] += 1
1664                                                         fails["metadata.bad"].append(
1665                                                                 ("%s/metadata.xml: Atom contains "
1666                                                                 "unexpected cat/pn: %s") % (x, atom))
1667
1668                         # Run other metadata.xml checkers
1669                         try:
1670                                 utilities.check_metadata(_metadata_xml, herd_base)
1671                         except (utilities.UnknownHerdsError, ) as e:
1672                                 metadata_bad = True
1673                                 stats["metadata.bad"] += 1
1674                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1675                                 del e
1676
1677                 #Only carry out if in package directory or check forced
1678                 if xmllint_capable and not metadata_bad:
1679                         # xmlint can produce garbage output even on success, so only dump
1680                         # the ouput when it fails.
1681                         st, out = repoman_getstatusoutput(
1682                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1683                                  (portage._shell_quote(metadata_dtd),
1684                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1685                         if st != os.EX_OK:
1686                                 print(red("!!!") + " metadata.xml is invalid:")
1687                                 for z in out.splitlines():
1688                                         print(red("!!! ")+z)
1689                                 stats["metadata.bad"]+=1
1690                                 fails["metadata.bad"].append(x+"/metadata.xml")
1691
1692                 del metadata_bad
1693         muselist = frozenset(musedict)
1694
1695         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1696         changelog_modified = changelog_path in modified_changelogs
1697
1698         # detect unused local USE-descriptions
1699         used_useflags = set()
1700
1701         for y in ebuildlist:
1702                 relative_path = os.path.join(x, y + ".ebuild")
1703                 full_path = os.path.join(repodir, relative_path)
1704                 ebuild_path = y + ".ebuild"
1705                 if repolevel < 3:
1706                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1707                 if repolevel < 2:
1708                         ebuild_path = os.path.join(catdir, ebuild_path)
1709                 ebuild_path = os.path.join(".", ebuild_path)
1710                 if check_changelog and not changelog_modified \
1711                         and ebuild_path in new_ebuilds:
1712                         stats['changelog.ebuildadded'] += 1
1713                         fails['changelog.ebuildadded'].append(relative_path)
1714
1715                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1716                         #ebuild not added to vcs
1717                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1718                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1719                 myesplit=portage.pkgsplit(y)
1720                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1721                             or pv_toolong_re.search(myesplit[1]) \
1722                             or pv_toolong_re.search(myesplit[2]):
1723                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1724                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1725                         continue
1726                 elif myesplit[0]!=pkgdir:
1727                         print(pkgdir,myesplit[0])
1728                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1729                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1730                         continue
1731
1732                 pkg = pkgs[y]
1733
1734                 if pkg.invalid:
1735                         allvalid = False
1736                         for k, msgs in pkg.invalid.items():
1737                                 for msg in msgs:
1738                                         stats[k] = stats[k] + 1
1739                                         fails[k].append("%s: %s" % (relative_path, msg))
1740                         continue
1741
1742                 myaux = pkg._metadata
1743                 eapi = myaux["EAPI"]
1744                 inherited = pkg.inherited
1745                 live_ebuild = live_eclasses.intersection(inherited)
1746
1747                 for k, v in myaux.items():
1748                         if not isinstance(v, basestring):
1749                                 continue
1750                         m = non_ascii_re.search(v)
1751                         if m is not None:
1752                                 stats["variable.invalidchar"] += 1
1753                                 fails["variable.invalidchar"].append(
1754                                         ("%s: %s variable contains non-ASCII " + \
1755                                         "character at position %s") % \
1756                                         (relative_path, k, m.start() + 1))
1757
1758                 if not src_uri_error:
1759                         # Check that URIs don't reference a server from thirdpartymirrors.
1760                         for uri in portage.dep.use_reduce( \
1761                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1762                                 contains_mirror = False
1763                                 for mirror, mirror_alias in thirdpartymirrors.items():
1764                                         if uri.startswith(mirror):
1765                                                 contains_mirror = True
1766                                                 break
1767                                 if not contains_mirror:
1768                                         continue
1769
1770                                 new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):])
1771                                 stats["SRC_URI.mirror"] += 1
1772                                 fails["SRC_URI.mirror"].append(
1773                                         "%s: '%s' found in thirdpartymirrors, use '%s'" % \
1774                                         (relative_path, mirror, new_uri))
1775
1776                 if myaux.get("PROVIDE"):
1777                         stats["virtual.oldstyle"]+=1
1778                         fails["virtual.oldstyle"].append(relative_path)
1779
1780                 for pos, missing_var in enumerate(missingvars):
1781                         if not myaux.get(missing_var):
1782                                 if catdir == "virtual" and \
1783                                         missing_var in ("HOMEPAGE", "LICENSE"):
1784                                         continue
1785                                 if live_ebuild and missing_var == "KEYWORDS":
1786                                         continue
1787                                 myqakey=missingvars[pos]+".missing"
1788                                 stats[myqakey]=stats[myqakey]+1
1789                                 fails[myqakey].append(x+"/"+y+".ebuild")
1790
1791                 if catdir == "virtual":
1792                         for var in ("HOMEPAGE", "LICENSE"):
1793                                 if myaux.get(var):
1794                                         myqakey = var + ".virtual"
1795                                         stats[myqakey] = stats[myqakey] + 1
1796                                         fails[myqakey].append(relative_path)
1797
1798                 # 14 is the length of DESCRIPTION=""
1799                 if len(myaux['DESCRIPTION']) > max_desc_len:
1800                         stats['DESCRIPTION.toolong'] += 1
1801                         fails['DESCRIPTION.toolong'].append(
1802                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1803                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1804
1805                 keywords = myaux["KEYWORDS"].split()
1806                 stable_keywords = []
1807                 for keyword in keywords:
1808                         if not keyword.startswith("~") and \
1809                                 not keyword.startswith("-"):
1810                                 stable_keywords.append(keyword)
1811                 if stable_keywords:
1812                         if ebuild_path in new_ebuilds and catdir != "virtual":
1813                                 stable_keywords.sort()
1814                                 stats["KEYWORDS.stable"] += 1
1815                                 fails["KEYWORDS.stable"].append(
1816                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1817                                                 " ".join(stable_keywords))
1818
1819                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1820                         if not kw.startswith("-"))
1821
1822                 previous_keywords = slot_keywords.get(pkg.slot)
1823                 if previous_keywords is None:
1824                         slot_keywords[pkg.slot] = set()
1825                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1826                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1827                         if dropped_keywords:
1828                                 stats["KEYWORDS.dropped"] += 1
1829                                 fails["KEYWORDS.dropped"].append(
1830                                         relative_path + ": %s" % \
1831                                         " ".join(sorted(dropped_keywords)))
1832
1833                 slot_keywords[pkg.slot].update(ebuild_archs)
1834
1835                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1836                 if "-*" in keywords:
1837                         haskeyword = False
1838                         for kw in keywords:
1839                                 if kw[0] == "~":
1840                                         kw = kw[1:]
1841                                 if kw in kwlist:
1842                                         haskeyword = True
1843                         if not haskeyword:
1844                                 stats["KEYWORDS.stupid"] += 1
1845                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1846
1847                 """
1848                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1849                 not be allowed to be marked stable
1850                 """
1851                 if live_ebuild and repo_config.name == "gentoo":
1852                         bad_stable_keywords = []
1853                         for keyword in keywords:
1854                                 if not keyword.startswith("~") and \
1855                                         not keyword.startswith("-"):
1856                                         bad_stable_keywords.append(keyword)
1857                                 del keyword
1858                         if bad_stable_keywords:
1859                                 stats["LIVEVCS.stable"] += 1
1860                                 fails["LIVEVCS.stable"].append(
1861                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1862                                                 bad_stable_keywords)
1863                         del bad_stable_keywords
1864
1865                         if keywords and not has_global_mask(pkg):
1866                                 stats["LIVEVCS.unmasked"] += 1
1867                                 fails["LIVEVCS.unmasked"].append(relative_path)
1868
1869                 if options.ignore_arches:
1870                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1871                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1872                 else:
1873                         arches = set()
1874                         for keyword in keywords:
1875                                 if (keyword[0]=="-"):
1876                                         continue
1877                                 elif (keyword[0]=="~"):
1878                                         arch = keyword[1:]
1879                                         if arch == "*":
1880                                                 for expanded_arch in profiles:
1881                                                         if expanded_arch == "**":
1882                                                                 continue
1883                                                         arches.add((keyword, expanded_arch,
1884                                                                 (expanded_arch, "~" + expanded_arch)))
1885                                         else:
1886                                                 arches.add((keyword, arch, (arch, keyword)))
1887                                 else:
1888                                         if keyword == "*":
1889                                                 for expanded_arch in profiles:
1890                                                         if expanded_arch == "**":
1891                                                                 continue
1892                                                         arches.add((keyword, expanded_arch,
1893                                                                 (expanded_arch,)))
1894                                         else:
1895                                                 arches.add((keyword, keyword, (keyword,)))
1896                         if not arches:
1897                                 # Use an empty profile for checking dependencies of
1898                                 # packages that have empty KEYWORDS.
1899                                 arches.add(('**', '**', ('**',)))
1900
1901                 unknown_pkgs = set()
1902                 baddepsyntax = False
1903                 badlicsyntax = False
1904                 badprovsyntax = False
1905                 catpkg = catdir+"/"+y
1906
1907                 inherited_java_eclass = "java-pkg-2" in inherited or \
1908                         "java-pkg-opt-2" in inherited
1909                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1910                 operator_tokens = set(["||", "(", ")"])
1911                 type_list, badsyntax = [], []
1912                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
1913                         mydepstr = myaux[mytype]
1914
1915                         buildtime = mytype in Package._buildtime_keys
1916                         runtime = mytype in Package._runtime_keys
1917                         token_class = None
1918                         if mytype.endswith("DEPEND"):
1919                                 token_class=portage.dep.Atom
1920
1921                         try:
1922                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1923                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1924                         except portage.exception.InvalidDependString as e:
1925                                 atoms = None
1926                                 badsyntax.append(str(e))
1927
1928                         if atoms and mytype.endswith("DEPEND"):
1929                                 if runtime and \
1930                                         "test?" in mydepstr.split():
1931                                         stats[mytype + '.suspect'] += 1
1932                                         fails[mytype + '.suspect'].append(relative_path + \
1933                                                 ": 'test?' USE conditional in %s" % mytype)
1934
1935                                 for atom in atoms:
1936                                         if atom == "||":
1937                                                 continue
1938
1939                                         # Skip dependency.unknown for blockers, so that we
1940                                         # don't encourage people to remove necessary blockers,
1941                                         # as discussed in bug #382407.
1942                                         if atom.blocker is None and \
1943                                                 not portdb.xmatch("match-all", atom) and \
1944                                                 not atom.cp.startswith("virtual/"):
1945                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1946
1947                                         is_blocker = atom.blocker
1948
1949                                         if catdir != "virtual":
1950                                                 if not is_blocker and \
1951                                                         atom.cp in suspect_virtual:
1952                                                         stats['virtual.suspect'] += 1
1953                                                         fails['virtual.suspect'].append(
1954                                                                 relative_path +
1955                                                                 ": %s: consider using '%s' instead of '%s'" %
1956                                                                 (mytype, suspect_virtual[atom.cp], atom))
1957
1958                                         if buildtime and \
1959                                                 not is_blocker and \
1960                                                 not inherited_java_eclass and \
1961                                                 atom.cp == "virtual/jdk":
1962                                                 stats['java.eclassesnotused'] += 1
1963                                                 fails['java.eclassesnotused'].append(relative_path)
1964                                         elif buildtime and \
1965                                                 not is_blocker and \
1966                                                 not inherited_wxwidgets_eclass and \
1967                                                 atom.cp == "x11-libs/wxGTK":
1968                                                 stats['wxwidgets.eclassnotused'] += 1
1969                                                 fails['wxwidgets.eclassnotused'].append(
1970                                                         (relative_path + ": %ss on x11-libs/wxGTK"
1971                                                         " without inheriting wxwidgets.eclass") % mytype)
1972                                         elif runtime:
1973                                                 if not is_blocker and \
1974                                                         atom.cp in suspect_rdepend:
1975                                                         stats[mytype + '.suspect'] += 1
1976                                                         fails[mytype + '.suspect'].append(
1977                                                                 relative_path + ": '%s'" % atom)
1978
1979                                         if atom.operator == "~" and \
1980                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1981                                                 qacat = 'dependency.badtilde'
1982                                                 stats[qacat] += 1
1983                                                 fails[qacat].append(
1984                                                         (relative_path + ": %s uses the ~ operator"
1985                                                          " with a non-zero revision:" + \
1986                                                          " '%s'") % (mytype, atom))
1987
1988                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1989
1990                 for m,b in zip(type_list, badsyntax):
1991                         if m.endswith("DEPEND"):
1992                                 qacat = "dependency.syntax"
1993                         else:
1994                                 qacat = m + ".syntax"
1995                         stats[qacat] += 1
1996                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
1997
1998                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1999                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
2000                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
2001                 badlicsyntax = badlicsyntax > 0
2002                 badprovsyntax = badprovsyntax > 0
2003
2004                 # uselist checks - global
2005                 myuse = []
2006                 default_use = []
2007                 for myflag in myaux["IUSE"].split():
2008                         flag_name = myflag.lstrip("+-")
2009                         used_useflags.add(flag_name)
2010                         if myflag != flag_name:
2011                                 default_use.append(myflag)
2012                         if flag_name not in uselist:
2013                                 myuse.append(flag_name)
2014
2015                 # uselist checks - metadata
2016                 for mypos in range(len(myuse)-1,-1,-1):
2017                         if myuse[mypos] and (myuse[mypos] in muselist):
2018                                 del myuse[mypos]
2019
2020                 if default_use and not eapi_has_iuse_defaults(eapi):
2021                         for myflag in default_use:
2022                                 stats['EAPI.incompatible'] += 1
2023                                 fails['EAPI.incompatible'].append(
2024                                         (relative_path + ": IUSE defaults" + \
2025                                         " not supported with EAPI='%s':" + \
2026                                         " '%s'") % (eapi, myflag))
2027
2028                 for mypos in range(len(myuse)):
2029                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
2030                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
2031
2032                 # license checks
2033                 if not badlicsyntax:
2034                         # Parse the LICENSE variable, remove USE conditions and
2035                         # flatten it.
2036                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2037                         # Check each entry to ensure that it exists in PORTDIR's
2038                         # license directory.
2039                         for lic in licenses:
2040                                 # Need to check for "||" manually as no portage
2041                                 # function will remove it without removing values.
2042                                 if lic not in liclist and lic != "||":
2043                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
2044                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
2045                                 elif lic in liclist_deprecated:
2046                                         stats["LICENSE.deprecated"] = stats["LICENSE.deprecated"] + 1
2047                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2048
2049                 #keyword checks
2050                 myuse = myaux["KEYWORDS"].split()
2051                 for mykey in myuse:
2052                         if mykey not in ("-*", "*", "~*"):
2053                                 myskey = mykey
2054                                 if myskey[:1] == "-":
2055                                         myskey = myskey[1:]
2056                                 if myskey[:1] == "~":
2057                                         myskey = myskey[1:]
2058                                 if myskey not in kwlist:
2059                                         stats["KEYWORDS.invalid"] += 1
2060                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
2061                                 elif myskey not in profiles:
2062                                         stats["KEYWORDS.invalid"] += 1
2063                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
2064
2065                 #restrict checks
2066                 myrestrict = None
2067                 try:
2068                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2069                 except portage.exception.InvalidDependString as e:
2070                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
2071                         fails["RESTRICT.syntax"].append(
2072                                 "%s: RESTRICT: %s" % (relative_path, e))
2073                         del e
2074                 if myrestrict:
2075                         myrestrict = set(myrestrict)
2076                         mybadrestrict = myrestrict.difference(valid_restrict)
2077                         if mybadrestrict:
2078                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2079                                 for mybad in mybadrestrict:
2080                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
2081                 #REQUIRED_USE check
2082                 required_use = myaux["REQUIRED_USE"]
2083                 if required_use:
2084                         if not eapi_has_required_use(eapi):
2085                                 stats['EAPI.incompatible'] += 1
2086                                 fails['EAPI.incompatible'].append(
2087                                         relative_path + ": REQUIRED_USE" + \
2088                                         " not supported with EAPI='%s'" % (eapi,))
2089                         try:
2090                                 portage.dep.check_required_use(required_use, (),
2091                                         pkg.iuse.is_valid_flag, eapi=eapi)
2092                         except portage.exception.InvalidDependString as e:
2093                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2094                                 fails["REQUIRED_USE.syntax"].append(
2095                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2096                                 del e
2097
2098                 # Syntax Checks
2099                 relative_path = os.path.join(x, y + ".ebuild")
2100                 full_path = os.path.join(repodir, relative_path)
2101                 if not vcs_preserves_mtime:
2102                         if ebuild_path not in new_ebuilds and \
2103                                 ebuild_path not in modified_ebuilds:
2104                                 pkg.mtime = None
2105                 try:
2106                         # All ebuilds should have utf_8 encoding.
2107                         f = io.open(_unicode_encode(full_path,
2108                                 encoding=_encodings['fs'], errors='strict'),
2109                                 mode='r', encoding=_encodings['repo.content'])
2110                         try:
2111                                 for check_name, e in run_checks(f, pkg):
2112                                         stats[check_name] += 1
2113                                         fails[check_name].append(relative_path + ': %s' % e)
2114                         finally:
2115                                 f.close()
2116                 except UnicodeDecodeError:
2117                         # A file.UTF8 failure will have already been recorded above.
2118                         pass
2119
2120                 if options.force:
2121                         # The dep_check() calls are the most expensive QA test. If --force
2122                         # is enabled, there's no point in wasting time on these since the
2123                         # user is intent on forcing the commit anyway.
2124                         continue
2125
2126                 relevant_profiles = []
2127                 for keyword,arch,groups in arches:
2128
2129                         if arch not in profiles:
2130                                 # A missing profile will create an error further down
2131                                 # during the KEYWORDS verification.
2132                                 continue
2133                         relevant_profiles.extend((keyword, groups, prof)
2134                                 for prof in profiles[arch])
2135
2136                 def sort_key(item):
2137                         return item[2].sub_path
2138
2139                 relevant_profiles.sort(key=sort_key)
2140
2141                 for keyword, groups, prof in relevant_profiles:
2142
2143                                 if prof.status not in ("stable", "dev") or \
2144                                         prof.status == "dev" and not options.include_dev:
2145                                         continue
2146
2147                                 dep_settings = arch_caches.get(prof.sub_path)
2148                                 if dep_settings is None:
2149                                         dep_settings = portage.config(
2150                                                 config_profile_path=prof.abs_path,
2151                                                 config_incrementals=repoman_incrementals,
2152                                                 config_root=config_root,
2153                                                 local_config=False,
2154                                                 _unmatched_removal=options.unmatched_removal,
2155                                                 env=env)
2156                                         dep_settings.categories = repoman_settings.categories
2157                                         if options.without_mask:
2158                                                 dep_settings._mask_manager_obj = \
2159                                                         copy.deepcopy(dep_settings._mask_manager)
2160                                                 dep_settings._mask_manager._pmaskdict.clear()
2161                                         arch_caches[prof.sub_path] = dep_settings
2162
2163                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2164                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2165                                 if xcache is None:
2166                                         portdb.melt()
2167                                         portdb.freeze()
2168                                         xcache = portdb.xcache
2169                                         xcache.update(shared_xmatch_caches)
2170                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2171
2172                                 trees[root]["porttree"].settings = dep_settings
2173                                 portdb.settings = dep_settings
2174                                 portdb.xcache = xcache
2175                                 # for package.use.mask support inside dep_check
2176                                 dep_settings.setcpv(pkg)
2177                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2178                                 # just in case, prevent config.reset() from nuking these.
2179                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2180
2181                                 if not baddepsyntax:
2182                                         ismasked = not ebuild_archs or \
2183                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2184                                         if ismasked:
2185                                                 if not have_pmasked:
2186                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2187                                                                 pkg.cpv, pkg._metadata))
2188                                                 if options.ignore_masked:
2189                                                         continue
2190                                                 #we are testing deps for a masked package; give it some lee-way
2191                                                 suffix="masked"
2192                                                 matchmode = "minimum-all"
2193                                         else:
2194                                                 suffix=""
2195                                                 matchmode = "minimum-visible"
2196
2197                                         if not have_dev_keywords:
2198                                                 have_dev_keywords = \
2199                                                         bool(dev_keywords.intersection(keywords))
2200
2201                                         if prof.status == "dev":
2202                                                 suffix=suffix+"indev"
2203
2204                                         for mytype in Package._dep_keys:
2205
2206                                                 mykey = "dependency.bad" + suffix
2207                                                 myvalue = myaux[mytype]
2208                                                 if not myvalue:
2209                                                         continue
2210
2211                                                 success, atoms = portage.dep_check(myvalue, portdb,
2212                                                         dep_settings, use="all", mode=matchmode,
2213                                                         trees=trees)
2214
2215                                                 if success:
2216                                                         if atoms:
2217
2218                                                                 # Don't bother with dependency.unknown for
2219                                                                 # cases in which *DEPEND.bad is triggered.
2220                                                                 for atom in atoms:
2221                                                                         # dep_check returns all blockers and they
2222                                                                         # aren't counted for *DEPEND.bad, so we
2223                                                                         # ignore them here.
2224                                                                         if not atom.blocker:
2225                                                                                 unknown_pkgs.discard(
2226                                                                                         (mytype, atom.unevaluated_atom))
2227
2228                                                                 if not prof.sub_path:
2229                                                                         # old-style virtuals currently aren't
2230                                                                         # resolvable with empty profile, since
2231                                                                         # 'virtuals' mappings are unavailable
2232                                                                         # (it would be expensive to search
2233                                                                         # for PROVIDE in all ebuilds)
2234                                                                         atoms = [atom for atom in atoms if not \
2235                                                                                 (atom.cp.startswith('virtual/') and \
2236                                                                                 not portdb.cp_list(atom.cp))]
2237
2238                                                                 #we have some unsolvable deps
2239                                                                 #remove ! deps, which always show up as unsatisfiable
2240                                                                 atoms = [str(atom.unevaluated_atom) \
2241                                                                         for atom in atoms if not atom.blocker]
2242
2243                                                                 #if we emptied out our list, continue:
2244                                                                 if not atoms:
2245                                                                         continue
2246                                                                 stats[mykey]=stats[mykey]+1
2247                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2248                                                                         (relative_path, mytype, keyword,
2249                                                                         prof, repr(atoms)))
2250                                                 else:
2251                                                         stats[mykey]=stats[mykey]+1
2252                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2253                                                                 (relative_path, mytype, keyword,
2254                                                                 prof, repr(atoms)))
2255
2256                 if not baddepsyntax and unknown_pkgs:
2257                         type_map = {}
2258                         for mytype, atom in unknown_pkgs:
2259                                 type_map.setdefault(mytype, set()).add(atom)
2260                         for mytype, atoms in type_map.items():
2261                                 stats["dependency.unknown"] += 1
2262                                 fails["dependency.unknown"].append("%s: %s: %s" %
2263                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2264
2265         # check if there are unused local USE-descriptions in metadata.xml
2266         # (unless there are any invalids, to avoid noise)
2267         if allvalid:
2268                 for myflag in muselist.difference(used_useflags):
2269                         stats["metadata.warning"] += 1
2270                         fails["metadata.warning"].append(
2271                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2272                                 (x, myflag))
2273
2274 if options.if_modified == "y" and len(effective_scanlist) < 1:
2275         logging.warn("--if-modified is enabled, but no modified packages were found!")
2276
2277 if options.mode == "manifest":
2278         sys.exit(dofail)
2279
2280 #dofail will be set to 1 if we have failed in at least one non-warning category
2281 dofail=0
2282 #dowarn will be set to 1 if we tripped any warnings
2283 dowarn=0
2284 #dofull will be set if we should print a "repoman full" informational message
2285 dofull = options.mode != 'full'
2286
2287 for x in qacats:
2288         if not stats[x]:
2289                 continue
2290         dowarn = 1
2291         if x not in qawarnings:
2292                 dofail = 1
2293
2294 if dofail or \
2295         (dowarn and not (options.quiet or options.mode == "scan")):
2296         dofull = 0
2297
2298 # Save QA output so that it can be conveniently displayed
2299 # in $EDITOR while the user creates a commit message.
2300 # Otherwise, the user would not be able to see this output
2301 # once the editor has taken over the screen.
2302 qa_output = io.StringIO()
2303 style_file = ConsoleStyleFile(sys.stdout)
2304 if options.mode == 'commit' and \
2305         (not commitmessage or not commitmessage.strip()):
2306         style_file.write_listener = qa_output
2307 console_writer = StyleWriter(file=style_file, maxcol=9999)
2308 console_writer.style_listener = style_file.new_styles
2309
2310 f = formatter.AbstractFormatter(console_writer)
2311
2312 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2313
2314 style_file.flush()
2315 del console_writer, f, style_file
2316 qa_output = qa_output.getvalue()
2317 qa_output = qa_output.splitlines(True)
2318
2319 def grouplist(mylist,seperator="/"):
2320         """(list,seperator="/") -- Takes a list of elements; groups them into
2321         same initial element categories. Returns a dict of {base:[sublist]}
2322         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2323         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2324         mygroups={}
2325         for x in mylist:
2326                 xs=x.split(seperator)
2327                 if xs[0]==".":
2328                         xs=xs[1:]
2329                 if xs[0] not in mygroups:
2330                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2331                 else:
2332                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2333         return mygroups
2334
2335 suggest_ignore_masked = False
2336 suggest_include_dev = False
2337
2338 if have_pmasked and not (options.without_mask or options.ignore_masked):
2339         suggest_ignore_masked = True
2340 if have_dev_keywords and not options.include_dev:
2341         suggest_include_dev = True
2342
2343 if suggest_ignore_masked or suggest_include_dev:
2344         print()
2345         if suggest_ignore_masked:
2346                 print(bold("Note: use --without-mask to check " + \
2347                         "KEYWORDS on dependencies of masked packages"))
2348
2349         if suggest_include_dev:
2350                 print(bold("Note: use --include-dev (-d) to check " + \
2351                         "dependencies for 'dev' profiles"))
2352         print()
2353
2354 if options.mode != 'commit':
2355         if dofull:
2356                 print(bold("Note: type \"repoman full\" for a complete listing."))
2357         if dowarn and not dofail:
2358                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2359         elif not dofail:
2360                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2361         elif dofail:
2362                 print(bad("Please fix these important QA issues first."))
2363                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2364                 sys.exit(1)
2365 else:
2366         if dofail and can_force and options.force and not options.pretend:
2367                 print(green("RepoMan sez:") + \
2368                         " \"You want to commit even with these QA issues?\n" + \
2369                         "              I'll take it this time, but I'm not happy.\"\n")
2370         elif dofail:
2371                 if options.force and not can_force:
2372                         print(bad("The --force option has been disabled due to extraordinary issues."))
2373                 print(bad("Please fix these important QA issues first."))
2374                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2375                 sys.exit(1)
2376
2377         if options.pretend:
2378                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2379
2380         myunadded = []
2381         if vcs == "cvs":
2382                 try:
2383                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2384                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2385                 except SystemExit as e:
2386                         raise  # TODO propagate this
2387                 except:
2388                         err("Error retrieving CVS tree; exiting.")
2389         if vcs == "svn":
2390                 try:
2391                         with repoman_popen("svn status --no-ignore") as f:
2392                                 svnstatus = f.readlines()
2393                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2394                 except SystemExit as e:
2395                         raise  # TODO propagate this
2396                 except:
2397                         err("Error retrieving SVN info; exiting.")
2398         if vcs == "git":
2399                 # get list of files not under version control or missing
2400                 myf = repoman_popen("git ls-files --others")
2401                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2402                 myf.close()
2403         if vcs == "bzr":
2404                 try:
2405                         with repoman_popen("bzr status -S .") as f:
2406                                 bzrstatus = f.readlines()
2407                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2408                 except SystemExit as e:
2409                         raise  # TODO propagate this
2410                 except:
2411                         err("Error retrieving bzr info; exiting.")
2412         if vcs == "hg":
2413                 with repoman_popen("hg status --no-status --unknown .") as f:
2414                         myunadded = f.readlines()
2415                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2416
2417                 # Mercurial doesn't handle manually deleted files as removed from
2418                 # the repository, so the user need to remove them before commit,
2419                 # using "hg remove [FILES]"
2420                 with repoman_popen("hg status --no-status --deleted .") as f:
2421                         mydeleted = f.readlines()
2422                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2423
2424
2425         myautoadd=[]
2426         if myunadded:
2427                 for x in range(len(myunadded)-1,-1,-1):
2428                         xs=myunadded[x].split("/")
2429                         if xs[-1]=="files":
2430                                 print("!!! files dir is not added! Please correct this.")
2431                                 sys.exit(-1)
2432                         elif xs[-1]=="Manifest":
2433                                 # It's a manifest... auto add
2434                                 myautoadd+=[myunadded[x]]
2435                                 del myunadded[x]
2436
2437         if myunadded:
2438                 print(red("!!! The following files are in your local tree but are not added to the master"))
2439                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2440                 for x in myunadded:
2441                         print("   ",x)
2442                 print()
2443                 print()
2444                 sys.exit(1)
2445
2446         if vcs == "hg" and mydeleted:
2447                 print(red("!!! The following files are removed manually from your local tree but are not"))
2448                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2449                 for x in mydeleted:
2450                         print("   ",x)
2451                 print()
2452                 print()
2453                 sys.exit(1)
2454
2455         if vcs == "cvs":
2456                 mycvstree = cvstree.getentries("./", recursive=1)
2457                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2458                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2459                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2460                 bin_blob_pattern = re.compile("^-kb$")
2461                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2462                         recursive=1, basedir="./"))
2463
2464
2465         if vcs == "svn":
2466                 with repoman_popen("svn status") as f:
2467                         svnstatus = f.readlines()
2468                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2469                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2470                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2471
2472                 # Subversion expands keywords specified in svn:keywords properties.
2473                 with repoman_popen("svn propget -R svn:keywords") as f:
2474                         props = f.readlines()
2475                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2476                         for prop in props if " - " in prop)
2477
2478         elif vcs == "git":
2479                 with repoman_popen("git diff-index --name-only "
2480                         "--relative --diff-filter=M HEAD") as f:
2481                         mychanged = f.readlines()
2482                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2483
2484                 with repoman_popen("git diff-index --name-only "
2485                         "--relative --diff-filter=A HEAD") as f:
2486                         mynew = f.readlines()
2487                 mynew = ["./" + elem[:-1] for elem in mynew]
2488
2489                 with repoman_popen("git diff-index --name-only "
2490                         "--relative --diff-filter=D HEAD") as f:
2491                         myremoved = f.readlines()
2492                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2493
2494         if vcs == "bzr":
2495                 with repoman_popen("bzr status -S .") as f:
2496                         bzrstatus = f.readlines()
2497                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2498                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2499                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2500                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2501                 # Bazaar expands nothing.
2502
2503         if vcs == "hg":
2504                 with repoman_popen("hg status --no-status --modified .") as f:
2505                         mychanged = f.readlines()
2506                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2507
2508                 with repoman_popen("hg status --no-status --added .") as f:
2509                         mynew = f.readlines()
2510                 mynew = ["./" + elem.rstrip() for elem in mynew]
2511
2512                 with repoman_popen("hg status --no-status --removed .") as f:
2513                         myremoved = f.readlines()
2514                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2515
2516         if vcs:
2517                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2518                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2519                         print()
2520                         print("(Didn't find any changed files...)")
2521                         print()
2522                         sys.exit(1)
2523
2524         # Manifests need to be regenerated after all other commits, so don't commit
2525         # them now even if they have changed.
2526         mymanifests = set()
2527         myupdates = set()
2528         for f in mychanged + mynew:
2529                 if "Manifest" == os.path.basename(f):
2530                         mymanifests.add(f)
2531                 else:
2532                         myupdates.add(f)
2533         myupdates.difference_update(myremoved)
2534         myupdates = list(myupdates)
2535         mymanifests = list(mymanifests)
2536         myheaders = []
2537         mydirty = []
2538
2539         commitmessage = options.commitmsg
2540         if options.commitmsgfile:
2541                 try:
2542                         f = io.open(_unicode_encode(options.commitmsgfile,
2543                         encoding=_encodings['fs'], errors='strict'),
2544                         mode='r', encoding=_encodings['content'], errors='replace')
2545                         commitmessage = f.read()
2546                         f.close()
2547                         del f
2548                 except (IOError, OSError) as e:
2549                         if e.errno == errno.ENOENT:
2550                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2551                         else:
2552                                 raise
2553                 # We've read the content so the file is no longer needed.
2554                 commitmessagefile = None
2555         if not commitmessage or not commitmessage.strip():
2556                 try:
2557                         editor = os.environ.get("EDITOR")
2558                         if editor and utilities.editor_is_executable(editor):
2559                                 commitmessage = utilities.get_commit_message_with_editor(
2560                                         editor, message=qa_output)
2561                         else:
2562                                 commitmessage = utilities.get_commit_message_with_stdin()
2563                 except KeyboardInterrupt:
2564                         exithandler()
2565                 if not commitmessage or not commitmessage.strip():
2566                         print("* no commit message?  aborting commit.")
2567                         sys.exit(1)
2568         commitmessage = commitmessage.rstrip()
2569         changelog_msg = commitmessage
2570         portage_version = getattr(portage, "VERSION", None)
2571         if portage_version is None:
2572                 sys.stderr.write("Failed to insert portage version in message!\n")
2573                 sys.stderr.flush()
2574                 portage_version = "Unknown"
2575
2576         if vcs == "git":
2577                 # Use new footer only for git (see bug #438364).
2578                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2579                 if options.force:
2580                         commit_footer += "\nRepoMan-Options: --force"
2581                 if sign_manifests:
2582                         commit_footer += "\nManifest-Sign-Key: %s" % \
2583                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2584         else:
2585                 unameout = platform.system() + " "
2586                 if platform.system() in ["Darwin", "SunOS"]:
2587                         unameout += platform.processor()
2588                 else:
2589                         unameout += platform.machine()
2590                 commit_footer = "\n\n(Portage version: %s/%s/%s" % \
2591                         (portage_version, vcs, unameout)
2592                 if options.force:
2593                         commit_footer += ", RepoMan options: --force"
2594                 if sign_manifests:
2595                         commit_footer += ", signed Manifest commit with key %s" % \
2596                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2597                 else:
2598                         commit_footer += ", unsigned Manifest commit"
2599                 commit_footer += ")"
2600
2601         commitmessage += commit_footer
2602
2603         if options.echangelog in ('y', 'force'):
2604                 logging.info("checking for unmodified ChangeLog files")
2605                 committer_name = utilities.get_committer_name(env=repoman_settings)
2606                 for x in sorted(vcs_files_to_cps(
2607                         chain(myupdates, mymanifests, myremoved))):
2608                         catdir, pkgdir = x.split("/")
2609                         checkdir = repodir + "/" + x
2610                         checkdir_relative = ""
2611                         if repolevel < 3:
2612                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2613                         if repolevel < 2:
2614                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2615                         checkdir_relative = os.path.join(".", checkdir_relative)
2616
2617                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2618                         changelog_modified = changelog_path in modified_changelogs
2619                         if changelog_modified and options.echangelog != 'force':
2620                                 continue
2621
2622                         # get changes for this package
2623                         cdrlen = len(checkdir_relative)
2624                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2625                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2626                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2627
2628                         # Skip ChangeLog generation if only the Manifest was modified,
2629                         # as discussed in bug #398009.
2630                         nontrivial_cl_files = set()
2631                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2632                         nontrivial_cl_files.difference_update(['Manifest'])
2633                         if not nontrivial_cl_files and options.echangelog != 'force':
2634                                 continue
2635
2636                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2637                                 committer_name, changelog_msg,
2638                                 os.path.join(repodir, 'skel.ChangeLog'),
2639                                 catdir, pkgdir,
2640                                 new=clnew, removed=clremoved, changed=clchanged,
2641                                 pretend=options.pretend)
2642                         if new_changelog is None:
2643                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2644                                         level=logging.ERROR, noiselevel=-1)
2645                                 sys.exit(1)
2646
2647                         # if the ChangeLog was just created, add it to vcs
2648                         if new_changelog:
2649                                 myautoadd.append(changelog_path)
2650                                 # myautoadd is appended to myupdates below
2651                         else:
2652                                 myupdates.append(changelog_path)
2653
2654                         if options.ask and not options.pretend:
2655                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2656                                 repoman_settings["O"] = checkdir
2657                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2658
2659         if myautoadd:
2660                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2661                 add_cmd = [vcs, "add"]
2662                 add_cmd += myautoadd
2663                 if options.pretend:
2664                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2665                                 noiselevel=-1)
2666                 else:
2667                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2668                                 # Python 3.1 produces the following TypeError if raw bytes are
2669                                 # passed to subprocess.call():
2670                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2671                                 #     errread, errwrite)
2672                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2673                                 #     raise child_exception
2674                                 # TypeError: expected an object with the buffer interface
2675                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2676                         retcode = subprocess.call(add_cmd)
2677                         if retcode != os.EX_OK:
2678                                 logging.error(
2679                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2680                                 sys.exit(retcode)
2681
2682                 myupdates += myautoadd
2683
2684         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2685
2686         if vcs not in ('cvs', 'svn'):
2687                 # With git, bzr and hg, there's never any keyword expansion, so
2688                 # there's no need to regenerate manifests and all files will be
2689                 # committed in one big commit at the end.
2690                 print()
2691         elif not repo_config.thin_manifest:
2692                 if vcs == 'cvs':
2693                         headerstring = "'\$(Header|Id).*\$'"
2694                 elif vcs == "svn":
2695                         svn_keywords = dict((k.lower(), k) for k in [
2696                                         "Rev",
2697                                         "Revision",
2698                                         "LastChangedRevision",
2699                                         "Date",
2700                                         "LastChangedDate",
2701                                         "Author",
2702                                         "LastChangedBy",
2703                                         "URL",
2704                                         "HeadURL",
2705                                         "Id",
2706                                         "Header",
2707                         ])
2708
2709                 for myfile in myupdates:
2710
2711                         # for CVS, no_expansion contains files that are excluded from expansion
2712                         if vcs == "cvs":
2713                                 if myfile in no_expansion:
2714                                         continue
2715
2716                         # for SVN, expansion contains files that are included in expansion
2717                         elif vcs == "svn":
2718                                 if myfile not in expansion:
2719                                         continue
2720
2721                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2722                                 enabled_keywords = []
2723                                 for k in expansion[myfile]:
2724                                         keyword = svn_keywords.get(k.lower())
2725                                         if keyword is not None:
2726                                                 enabled_keywords.append(keyword)
2727
2728                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2729
2730                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2731                                 portage._shell_quote(myfile))
2732                         if myout[0] == 0:
2733                                 myheaders.append(myfile)
2734
2735                 print("%s have headers that will change." % green(str(len(myheaders))))
2736                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2737
2738         logging.info("myupdates: %s", myupdates)
2739         logging.info("myheaders: %s", myheaders)
2740
2741         if options.ask and userquery('Commit changes?', True) != 'Yes':
2742                 print("* aborting commit.")
2743                 sys.exit(128 + signal.SIGINT)
2744
2745         # Handle the case where committed files have keywords which
2746         # will change and need a priming commit before the Manifest
2747         # can be committed.
2748         if (myupdates or myremoved) and myheaders:
2749                 myfiles = myupdates + myremoved
2750                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2751                 mymsg = os.fdopen(fd, "wb")
2752                 mymsg.write(_unicode_encode(commitmessage))
2753                 mymsg.close()
2754
2755                 print()
2756                 print(green("Using commit message:"))
2757                 print(green("------------------------------------------------------------------------------"))
2758                 print(commitmessage)
2759                 print(green("------------------------------------------------------------------------------"))
2760                 print()
2761
2762                 # Having a leading ./ prefix on file paths can trigger a bug in
2763                 # the cvs server when committing files to multiple directories,
2764                 # so strip the prefix.
2765                 myfiles = [f.lstrip("./") for f in myfiles]
2766
2767                 commit_cmd = [vcs]
2768                 commit_cmd.extend(vcs_global_opts)
2769                 commit_cmd.append("commit")
2770                 commit_cmd.extend(vcs_local_opts)
2771                 commit_cmd.extend(["-F", commitmessagefile])
2772                 commit_cmd.extend(myfiles)
2773
2774                 try:
2775                         if options.pretend:
2776                                 print("(%s)" % (" ".join(commit_cmd),))
2777                         else:
2778                                 retval = spawn(commit_cmd, env=os.environ)
2779                                 if retval != os.EX_OK:
2780                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2781                                                 "error code: %s\n") % (vcs, retval),
2782                                                 level=logging.ERROR, noiselevel=-1)
2783                                         sys.exit(retval)
2784                 finally:
2785                         try:
2786                                 os.unlink(commitmessagefile)
2787                         except OSError:
2788                                 pass
2789
2790         # Setup the GPG commands
2791         def gpgsign(filename):
2792                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2793                 if gpgcmd is None:
2794                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2795                                 " Is make.globals missing?")
2796                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2797                         "PORTAGE_GPG_KEY" not in repoman_settings:
2798                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2799                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2800                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2801                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2802                                         os.path.expanduser("~/.gnupg")
2803                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2804                                         % repoman_settings["PORTAGE_GPG_DIR"])
2805                         else:
2806                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2807                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2808                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2809                                 raise portage.exception.InvalidLocation(
2810                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2811                                         repoman_settings["PORTAGE_GPG_DIR"])
2812                 gpgvars = {"FILE": filename}
2813                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2814                         v = repoman_settings.get(k)
2815                         if v is not None:
2816                                 gpgvars[k] = v
2817                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2818                 if options.pretend:
2819                         print("("+gpgcmd+")")
2820                 else:
2821                         # Encode unicode manually for bug #310789.
2822                         gpgcmd = portage.util.shlex_split(gpgcmd)
2823                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2824                                 # Python 3.1 does not support bytes in Popen args.
2825                                 gpgcmd = [_unicode_encode(arg,
2826                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2827                         rValue = subprocess.call(gpgcmd)
2828                         if rValue == os.EX_OK:
2829                                 os.rename(filename+".asc", filename)
2830                         else:
2831                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2832
2833         def need_signature(filename):
2834                 try:
2835                         with open(_unicode_encode(filename,
2836                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2837                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2838                 except IOError as e:
2839                         if e.errno in (errno.ENOENT, errno.ESTALE):
2840                                 return False
2841                         raise
2842
2843         # When files are removed and re-added, the cvs server will put /Attic/
2844         # inside the $Header path. This code detects the problem and corrects it
2845         # so that the Manifest will generate correctly. See bug #169500.
2846         # Use binary mode in order to avoid potential character encoding issues.
2847         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2848         attic_str = b'/Attic/'
2849         attic_replace = b'/'
2850         for x in myheaders:
2851                 f = open(_unicode_encode(x,
2852                         encoding=_encodings['fs'], errors='strict'),
2853                         mode='rb')
2854                 mylines = f.readlines()
2855                 f.close()
2856                 modified = False
2857                 for i, line in enumerate(mylines):
2858                         if cvs_header_re.match(line) is not None and \
2859                                 attic_str in line:
2860                                 mylines[i] = line.replace(attic_str, attic_replace)
2861                                 modified = True
2862                 if modified:
2863                         portage.util.write_atomic(x, b''.join(mylines),
2864                                 mode='wb')
2865
2866         if repolevel == 1:
2867                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2868                         "doing the entire repository.\"\n")
2869
2870         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2871
2872                 for x in sorted(vcs_files_to_cps(
2873                         chain(myupdates, myremoved, mymanifests))):
2874                         repoman_settings["O"] = os.path.join(repodir, x)
2875                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2876
2877         signed = False
2878         if sign_manifests:
2879                 signed = True
2880                 try:
2881                         for x in sorted(vcs_files_to_cps(
2882                                 chain(myupdates, myremoved, mymanifests))):
2883                                 repoman_settings["O"] = os.path.join(repodir, x)
2884                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2885                                 if not need_signature(manifest_path):
2886                                         continue
2887                                 gpgsign(manifest_path)
2888                 except portage.exception.PortageException as e:
2889                         portage.writemsg("!!! %s\n" % str(e))
2890                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2891                         signed = False
2892
2893         if vcs == 'git':
2894                 # It's not safe to use the git commit -a option since there might
2895                 # be some modified files elsewhere in the working tree that the
2896                 # user doesn't want to commit. Therefore, call git update-index
2897                 # in order to ensure that the index is updated with the latest
2898                 # versions of all new and modified files in the relevant portion
2899                 # of the working tree.
2900                 myfiles = mymanifests + myupdates
2901                 myfiles.sort()
2902                 update_index_cmd = ["git", "update-index"]
2903                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2904                 if options.pretend:
2905                         print("(%s)" % (" ".join(update_index_cmd),))
2906                 else:
2907                         retval = spawn(update_index_cmd, env=os.environ)
2908                         if retval != os.EX_OK:
2909                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2910                                         "error code: %s\n") % (vcs, retval),
2911                                         level=logging.ERROR, noiselevel=-1)
2912                                 sys.exit(retval)
2913
2914         if True:
2915
2916                 myfiles = mymanifests[:]
2917                 # If there are no header (SVN/CVS keywords) changes in
2918                 # the files, this Manifest commit must include the
2919                 # other (yet uncommitted) files.
2920                 if not myheaders:
2921                         myfiles += myupdates
2922                         myfiles += myremoved
2923                 myfiles.sort()
2924
2925                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2926                 mymsg = os.fdopen(fd, "wb")
2927                 mymsg.write(_unicode_encode(commitmessage))
2928                 mymsg.close()
2929
2930                 commit_cmd = []
2931                 if options.pretend and vcs is None:
2932                         # substitute a bogus value for pretend output
2933                         commit_cmd.append("cvs")
2934                 else:
2935                         commit_cmd.append(vcs)
2936                 commit_cmd.extend(vcs_global_opts)
2937                 commit_cmd.append("commit")
2938                 commit_cmd.extend(vcs_local_opts)
2939                 if vcs == "hg":
2940                         commit_cmd.extend(["--logfile", commitmessagefile])
2941                         commit_cmd.extend(myfiles)
2942                 else:
2943                         commit_cmd.extend(["-F", commitmessagefile])
2944                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2945
2946                 try:
2947                         if options.pretend:
2948                                 print("(%s)" % (" ".join(commit_cmd),))
2949                         else:
2950                                 retval = spawn(commit_cmd, env=os.environ)
2951                                 if retval != os.EX_OK:
2952
2953                                         if repo_config.sign_commit and vcs == 'git' and \
2954                                                 not git_supports_gpg_sign():
2955                                                 # Inform user that newer git is needed (bug #403323).
2956                                                 logging.error(
2957                                                         "Git >=1.7.9 is required for signed commits!")
2958
2959                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2960                                                 "error code: %s\n") % (vcs, retval),
2961                                                 level=logging.ERROR, noiselevel=-1)
2962                                         sys.exit(retval)
2963                 finally:
2964                         try:
2965                                 os.unlink(commitmessagefile)
2966                         except OSError:
2967                                 pass
2968
2969         print()
2970         if vcs:
2971                 print("Commit complete.")
2972         else:
2973                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2974         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2975 sys.exit(0)
2976