bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2012 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import io
  17 import logging
  18 import optparse
  19 import re
  20 import signal
  21 import stat
  22 import subprocess
  23 import sys
  24 import tempfile
  25 import textwrap
  26 import time
  27 import platform
  28
  29 try:
  30         from urllib.request import urlopen as urllib_request_urlopen
  31 except ImportError:
  32         from urllib import urlopen as urllib_request_urlopen
  33
  34 from itertools import chain
  35 from stat import S_ISDIR
  36
  37 from os import path as osp
  38 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  39 sys.path.insert(0, pym_path)
  40 import portage
  41 portage._disable_legacy_globals()
  42 portage._internal_warnings = True
  43
  44 try:
  45         import xml.etree.ElementTree
  46         from xml.parsers.expat import ExpatError
  47 except (SystemExit, KeyboardInterrupt):
  48         raise
  49 except (ImportError, SystemError, RuntimeError, Exception):
  50         # broken or missing xml support
  51         # http://bugs.python.org/issue14988
  52         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  53         from portage.output import EOutput
  54         out = EOutput()
  55         for line in msg:
  56                 out.eerror(line)
  57         sys.exit(1)
  58
  59 from portage import os
  60 from portage import _encodings
  61 from portage import _unicode_encode
  62 from repoman.checks import run_checks
  63 from repoman import utilities
  64 from repoman.herdbase import make_herd_base
  65 from _emerge.Package import Package
  66 from _emerge.RootConfig import RootConfig
  67 from _emerge.userquery import userquery
  68 import portage.checksum
  69 import portage.const
  70 from portage import cvstree, normalize_path
  71 from portage import util
  72 from portage.exception import (FileNotFound, InvalidAtom, MissingParameter,
  73         ParseError, PermissionDenied)
  74 from portage.dep import Atom
  75 from portage.process import find_binary, spawn
  76 from portage.output import bold, create_color_func, \
  77         green, nocolor, red
  78 from portage.output import ConsoleStyleFile, StyleWriter
  79 from portage.util import writemsg_level
  80 from portage.package.ebuild.digestgen import digestgen
  81 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  82
  83 if sys.hexversion >= 0x3000000:
  84         basestring = str
  85
  86 util.initialize_logger()
  87
  88 # 14 is the length of DESCRIPTION=""
  89 max_desc_len = 100
  90 allowed_filename_chars="a-zA-Z0-9._-+:"
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{16}|[0-9a-fA-F]{24}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40})!?'
  93 bad = create_color_func("BAD")
  94
  95 # A sane umask is needed for files that portage creates.
  96 os.umask(0o22)
  97 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  98 # behave incrementally.
  99 repoman_incrementals = tuple(x for x in \
 100         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 101 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 102 repoman_settings = portage.config(config_root=config_root, local_config=False)
 103
 104 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 105         repoman_settings.get('TERM') == 'dumb' or \
 106         not sys.stdout.isatty():
 107         nocolor()
 108
 109 def warn(txt):
 110         print("repoman: " + txt)
 111
 112 def err(txt):
 113         warn(txt)
 114         sys.exit(1)
 115
 116 def exithandler(signum=None, frame=None):
 117         logging.fatal("Interrupted; exiting...")
 118         if signum is None:
 119                 sys.exit(1)
 120         else:
 121                 sys.exit(128 + signum)
 122
 123 signal.signal(signal.SIGINT,exithandler)
 124
 125 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 126         """Repoman needs it's own HelpFormatter for now, because the default ones
 127         murder the help text."""
 128
 129         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 130                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 131
 132         def format_description(self, description):
 133                 return description
 134
 135 class RepomanOptionParser(optparse.OptionParser):
 136         """Add the on_tail function, ruby has it, optionParser should too
 137         """
 138
 139         def __init__(self, *args, **kwargs):
 140                 optparse.OptionParser.__init__(self, *args, **kwargs)
 141                 self.tail = ""
 142
 143         def on_tail(self, description):
 144                 self.tail += description
 145
 146         def format_help(self, formatter=None):
 147                 result = optparse.OptionParser.format_help(self, formatter)
 148                 result += self.tail
 149                 return result
 150
 151
 152 def ParseArgs(argv, qahelp):
 153         """This function uses a customized optionParser to parse command line arguments for repoman
 154         Args:
 155           argv - a sequence of command line arguments
 156                 qahelp - a dict of qa warning to help message
 157         Returns:
 158           (opts, args), just like a call to parser.parse_args()
 159         """
 160
 161         if argv and isinstance(argv[0], bytes):
 162                 argv = [portage._unicode_decode(x) for x in argv]
 163
 164         modes = {
 165                 'commit' : 'Run a scan then commit changes',
 166                 'ci' : 'Run a scan then commit changes',
 167                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 168                 'full' : 'Scan directory tree and print all issues (not a summary)',
 169                 'help' : 'Show this screen',
 170                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 171                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 172                 'scan' : 'Scan directory tree for QA issues'
 173         }
 174
 175         mode_keys = list(modes)
 176         mode_keys.sort()
 177
 178         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 179         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 180         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 181         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 182         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 183
 184         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 185                 help='Request a confirmation before commiting')
 186
 187         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 188                 help='specify a commit message on the command line')
 189
 190         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 191                 help='specify a path to a file that contains a commit message')
 192
 193         parser.add_option('--digest',
 194                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 195                 help='Automatically update Manifest digests for modified files')
 196
 197         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 198                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 199
 200         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 201                 help='do not print unnecessary messages')
 202
 203         parser.add_option(
 204                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 205                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 206                 'regardless of modification if \'force\' is specified)')
 207
 208         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 209                 help='Commit with QA violations')
 210
 211         parser.add_option('--vcs', dest='vcs',
 212                 help='Force using specific VCS instead of autodetection')
 213
 214         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 215                 help='be very verbose in output', default=0)
 216
 217         parser.add_option('-V', '--version', dest='version', action='store_true',
 218                 help='show version info')
 219
 220         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 221                 default=False, help='forces the metadata.xml parse check to be carried out')
 222
 223         parser.add_option(
 224                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 225                 metavar="<y|n>",
 226                 help='only check packages that have uncommitted modifications')
 227
 228         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 229                 default=False, help='ignore arch-specific failures (where arch != host)')
 230
 231         parser.add_option("--ignore-default-opts",
 232                 action="store_true",
 233                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 234
 235         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 236                 default=False, help='ignore masked packages (not allowed with commit mode)')
 237
 238         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 239                 default=False, help='include dev profiles in dependency checks')
 240
 241         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 242                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 243
 244         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 245                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 246
 247         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 248                 help='specify which mode repoman will run in (default=full)')
 249
 250         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 251
 252         for k in mode_keys:
 253                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 254
 255         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 256
 257         sorted_qa = list(qahelp)
 258         sorted_qa.sort()
 259         for k in sorted_qa:
 260                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 261
 262         opts, args = parser.parse_args(argv[1:])
 263
 264         if not opts.ignore_default_opts:
 265                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 266                 if default_opts:
 267                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 268
 269         if opts.mode == 'help':
 270                 parser.print_help(short=False)
 271
 272         for arg in args:
 273                 if arg in modes:
 274                         if not opts.mode:
 275                                 opts.mode = arg
 276                                 break
 277                 else:
 278                         parser.error("invalid mode: %s" % arg)
 279
 280         if not opts.mode:
 281                 opts.mode = 'full'
 282
 283         if opts.mode == 'ci':
 284                 opts.mode = 'commit'  # backwards compat shortcut
 285
 286         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 287                 if opts.ignore_masked:
 288                         parser.error('Commit mode and --ignore-masked are not compatible')
 289                 if opts.without_mask:
 290                         parser.error('Commit mode and --without-mask are not compatible')
 291
 292         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 293         for val in range(opts.verbosity):
 294                 logger = logging.getLogger()
 295                 logger.setLevel(logger.getEffectiveLevel() - 10)
 296
 297         for val in range(opts.quiet):
 298                 logger = logging.getLogger()
 299                 logger.setLevel(logger.getEffectiveLevel() + 10)
 300
 301         return (opts, args)
 302
 303 qahelp={
 304         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 305         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 306         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 307         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 308         "changelog.missing":"Missing ChangeLog files",
 309         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 310         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 311         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 312         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 313         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 314         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 315         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 316         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 317         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 318         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 319         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 320         "file.size":"Files in the files directory must be under 20 KiB",
 321         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 322         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 323         "file.UTF8":"File is not UTF8 compliant",
 324         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 325         "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it",
 326         "inherit.unused":"Ebuild inherits an eclass but does not use it",
 327         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 328         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 329         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 330         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 331         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 332         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 333         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 334         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 335         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 336         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 337         "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 338         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 339         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 340         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 341         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 342         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 343         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 344         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 345         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 346         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 347         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 348         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 349         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 350         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 351         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 352         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 353         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 354         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 355         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 356         "variable.readonly":"Assigning a readonly variable",
 357         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 358         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 359         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 360         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 361         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 362         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 363         "LICENSE.deprecated":"This ebuild is listing a deprecated license.",
 364         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 365         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 366         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 367         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 368         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 369         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 370         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 371         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 372         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 373         "ebuild.badheader":"This ebuild has a malformed header",
 374         "manifest.bad":"Manifest has missing or incorrect digests",
 375         "metadata.missing":"Missing metadata.xml files",
 376         "metadata.bad":"Bad metadata.xml files",
 377         "metadata.warning":"Warnings in metadata.xml files",
 378         "portage.internal":"The ebuild uses an internal Portage function or variable",
 379         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 380         "virtual.suspect":"Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 381         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 382         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 383 }
 384
 385 qacats = list(qahelp)
 386 qacats.sort()
 387
 388 qawarnings = set((
 389 "changelog.missing",
 390 "changelog.notadded",
 391 "dependency.unknown",
 392 "digest.assumed",
 393 "digest.unused",
 394 "ebuild.notadded",
 395 "ebuild.nesteddie",
 396 "dependency.badmasked",
 397 "dependency.badindev",
 398 "dependency.badmaskedindev",
 399 "dependency.badtilde",
 400 "DESCRIPTION.toolong",
 401 "EAPI.deprecated",
 402 "HOMEPAGE.virtual",
 403 "LICENSE.deprecated",
 404 "LICENSE.virtual",
 405 "KEYWORDS.dropped",
 406 "KEYWORDS.stupid",
 407 "KEYWORDS.missing",
 408 "PDEPEND.suspect",
 409 "RDEPEND.implicit",
 410 "RDEPEND.suspect",
 411 "virtual.suspect",
 412 "RESTRICT.invalid",
 413 "ebuild.minorsyn",
 414 "ebuild.badheader",
 415 "ebuild.patches",
 416 "file.size",
 417 "inherit.unused",
 418 "inherit.deprecated",
 419 "java.eclassesnotused",
 420 "wxwidgets.eclassnotused",
 421 "metadata.warning",
 422 "portage.internal",
 423 "usage.obsolete",
 424 "upstream.workaround",
 425 "LIVEVCS.stable",
 426 "LIVEVCS.unmasked",
 427 ))
 428
 429 if portage.const._ENABLE_INHERIT_CHECK:
 430         # This is experimental, so it's non-fatal.
 431         qawarnings.add("inherit.missing")
 432
 433 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 434
 435 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 436 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 437 allvars.update(Package.metadata_keys)
 438 allvars = sorted(allvars)
 439 commitmessage=None
 440 for x in missingvars:
 441         x += ".missing"
 442         if x not in qacats:
 443                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 444                 qacats.append(x)
 445                 qawarnings.add(x)
 446
 447 valid_restrict = frozenset(["binchecks", "bindist",
 448         "fetch", "installsources", "mirror",
 449         "primaryuri", "strip", "test", "userpriv"])
 450
 451 live_eclasses = frozenset([
 452         "bzr",
 453         "cvs",
 454         "darcs",
 455         "git",
 456         "git-2",
 457         "mercurial",
 458         "subversion",
 459         "tla",
 460 ])
 461
 462 suspect_rdepend = frozenset([
 463         "app-arch/cabextract",
 464         "app-arch/rpm2targz",
 465         "app-doc/doxygen",
 466         "dev-lang/nasm",
 467         "dev-lang/swig",
 468         "dev-lang/yasm",
 469         "dev-perl/extutils-pkgconfig",
 470         "dev-util/byacc",
 471         "dev-util/cmake",
 472         "dev-util/ftjam",
 473         "dev-util/gperf",
 474         "dev-util/gtk-doc",
 475         "dev-util/gtk-doc-am",
 476         "dev-util/intltool",
 477         "dev-util/jam",
 478         "dev-util/pkg-config-lite",
 479         "dev-util/pkgconf",
 480         "dev-util/pkgconfig",
 481         "dev-util/pkgconfig-openbsd",
 482         "dev-util/scons",
 483         "dev-util/unifdef",
 484         "dev-util/yacc",
 485         "media-gfx/ebdftopcf",
 486         "sys-apps/help2man",
 487         "sys-devel/autoconf",
 488         "sys-devel/automake",
 489         "sys-devel/bin86",
 490         "sys-devel/bison",
 491         "sys-devel/dev86",
 492         "sys-devel/flex",
 493         "sys-devel/m4",
 494         "sys-devel/pmake",
 495         "virtual/linux-sources",
 496         "virtual/pkgconfig",
 497         "x11-misc/bdftopcf",
 498         "x11-misc/imake",
 499 ])
 500
 501 suspect_virtual = {
 502         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 503         "dev-util/pkgconf":"virtual/pkgconfig",
 504         "dev-util/pkgconfig":"virtual/pkgconfig",
 505         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 506         "dev-libs/libusb":"virtual/libusb",
 507         "dev-libs/libusbx":"virtual/libusb",
 508         "dev-libs/libusb-compat":"virtual/libusb",
 509 }
 510
 511 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 512 # force refetch if the local copy creation time is older than this
 513 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 514
 515 # file.executable
 516 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 517
 518 options, arguments = ParseArgs(sys.argv, qahelp)
 519
 520 if options.version:
 521         print("Portage", portage.VERSION)
 522         sys.exit(0)
 523
 524 # Set this to False when an extraordinary issue (generally
 525 # something other than a QA issue) makes it impossible to
 526 # commit (like if Manifest generation fails).
 527 can_force = True
 528
 529 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 530 if portdir is None:
 531         sys.exit(1)
 532
 533 myreporoot = os.path.basename(portdir_overlay)
 534 myreporoot += mydir[len(portdir_overlay):]
 535
 536 if options.vcs:
 537         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 538                 vcs = options.vcs
 539         else:
 540                 vcs = None
 541 else:
 542         vcses = utilities.FindVCS()
 543         if len(vcses) > 1:
 544                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 545                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 546                 sys.exit(1)
 547         elif vcses:
 548                 vcs = vcses[0]
 549         else:
 550                 vcs = None
 551
 552 if options.if_modified == "y" and vcs is None:
 553         logging.info("Not in a version controlled repository; "
 554                 "disabling --if-modified.")
 555         options.if_modified = "n"
 556
 557 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 558 vcs_preserves_mtime = vcs in ('cvs',)
 559
 560 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 561 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 562 if vcs_global_opts is None:
 563         if vcs in ('cvs', 'svn'):
 564                 vcs_global_opts = "-q"
 565         else:
 566                 vcs_global_opts = ""
 567 vcs_global_opts = vcs_global_opts.split()
 568
 569 if options.mode == 'commit' and not options.pretend and not vcs:
 570         logging.info("Not in a version controlled repository; enabling pretend mode.")
 571         options.pretend = True
 572
 573 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 574 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 575         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 576         portage._shell_quote(portdir_overlay))
 577 # We have to call the config constructor again so
 578 # that config.repositories is initialized correctly.
 579 repoman_settings = portage.config(config_root=config_root, local_config=False,
 580         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 581
 582 root = repoman_settings['EROOT']
 583 trees = {
 584         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 585 }
 586 portdb = trees[root]['porttree'].dbapi
 587
 588 # Constrain dependency resolution to the master(s)
 589 # that are specified in layout.conf.
 590 repodir = os.path.realpath(portdir_overlay)
 591 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 592 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 593 portdir = portdb.porttrees[0]
 594
 595 if repo_config.allow_provide_virtual:
 596         qawarnings.add("virtual.oldstyle")
 597
 598 if repo_config.sign_commit:
 599         if vcs == 'git':
 600                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 601                 # the commit arguments. If key_id is unspecified, then it must be
 602                 # configured by `git config user.signingkey key_id`.
 603                 vcs_local_opts.append("--gpg-sign")
 604
 605 # In order to disable manifest signatures, repos may set
 606 # "sign-manifests = false" in metadata/layout.conf. This
 607 # can be used to prevent merge conflicts like those that
 608 # thin-manifests is designed to prevent.
 609 sign_manifests = "sign" in repoman_settings.features and \
 610         repo_config.sign_manifest
 611
 612 if sign_manifests and options.mode in ("commit",) and \
 613         repoman_settings.get("PORTAGE_GPG_KEY") and \
 614         re.match(r'^%s$' % GPG_KEY_ID_REGEX,
 615         repoman_settings["PORTAGE_GPG_KEY"]) is None:
 616         logging.error("PORTAGE_GPG_KEY value is invalid: %s" %
 617                 repoman_settings["PORTAGE_GPG_KEY"])
 618         sys.exit(1)
 619
 620 manifest_hashes = repo_config.manifest_hashes
 621 if manifest_hashes is None:
 622         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 623
 624 if options.mode in ("commit", "fix", "manifest"):
 625         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 626                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 627                 "metadata/layout.conf does not contain the '%s' hash which "
 628                 "is required by this portage version. You will have to "
 629                 "upgrade portage if you want to generate valid manifests for "
 630                 "this repository.") % \
 631                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 632                 for line in textwrap.wrap(msg, 70):
 633                         logging.error(line)
 634                 sys.exit(1)
 635
 636         unsupported_hashes = manifest_hashes.difference(
 637                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 638         if unsupported_hashes:
 639                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 640                 "metadata/layout.conf contains one or more hash types '%s' "
 641                 "which are not supported by this portage version. You will "
 642                 "have to upgrade portage if you want to generate valid "
 643                 "manifests for this repository.") % \
 644                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 645                 for line in textwrap.wrap(msg, 70):
 646                         logging.error(line)
 647                 sys.exit(1)
 648
 649 if "commit" == options.mode and \
 650         repo_config.name == "gentoo" and \
 651         "RMD160" in manifest_hashes and \
 652         "RMD160" not in portage.checksum.hashorigin_map:
 653         msg = "Please install " \
 654         "pycrypto or enable python's ssl USE flag in order " \
 655         "to enable RMD160 hash support. See bug #198398 for " \
 656         "more information."
 657         prefix = bad(" * ")
 658         for line in textwrap.wrap(msg, 70):
 659                 print(prefix + line)
 660         sys.exit(1)
 661
 662 if options.echangelog is None and repo_config.update_changelog:
 663         options.echangelog = 'y'
 664
 665 if vcs is None:
 666         options.echangelog = 'n'
 667
 668 # The --echangelog option causes automatic ChangeLog generation,
 669 # which invalidates changelog.ebuildadded and changelog.missing
 670 # checks.
 671 # Note: Some don't use ChangeLogs in distributed SCMs.
 672 # It will be generated on server side from scm log,
 673 # before package moves to the rsync server.
 674 # This is needed because they try to avoid merge collisions.
 675 # Gentoo's Council decided to always use the ChangeLog file.
 676 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 677 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 678
 679 if 'digest' in repoman_settings.features and options.digest != 'n':
 680         options.digest = 'y'
 681
 682 logging.debug("vcs: %s" % (vcs,))
 683 logging.debug("repo config: %s" % (repo_config,))
 684 logging.debug("options: %s" % (options,))
 685
 686 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 687 # profile-specific config constructor calls.
 688 env = os.environ.copy()
 689 env['PORTDIR'] = portdir
 690 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 691
 692 logging.info('Setting paths:')
 693 logging.info('PORTDIR = "' + portdir + '"')
 694 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 695
 696 # It's confusing if these warnings are displayed without the user
 697 # being told which profile they come from, so disable them.
 698 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 699
 700 categories = []
 701 for path in repo_config.eclass_db.porttrees:
 702         categories.extend(portage.util.grabfile(
 703                 os.path.join(path, 'profiles', 'categories')))
 704 repoman_settings.categories = frozenset(
 705         portage.util.stack_lists([categories], incremental=1))
 706 categories = repoman_settings.categories
 707
 708 portdb.settings = repoman_settings
 709 root_config = RootConfig(repoman_settings, trees[root], None)
 710 # We really only need to cache the metadata that's necessary for visibility
 711 # filtering. Anything else can be discarded to reduce memory consumption.
 712 portdb._aux_cache_keys.clear()
 713 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 714
 715 reposplit = myreporoot.split(os.path.sep)
 716 repolevel = len(reposplit)
 717
 718 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 719 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 720 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 721 if options.mode == 'commit' and repolevel not in [1,2,3]:
 722         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 723         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 724         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 725         print(red("***"))
 726         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 727
 728 # Make startdir relative to the canonical repodir, so that we can pass
 729 # it to digestgen and it won't have to be canonicalized again.
 730 if repolevel == 1:
 731         startdir = repodir
 732 else:
 733         startdir = normalize_path(mydir)
 734         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 735
 736 def caterror(mycat):
 737         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 738
 739 def repoman_getstatusoutput(cmd):
 740         """
 741         Implements an interface similar to getstatusoutput(), but with
 742         customized unicode handling (see bug #310789) and without the shell.
 743         """
 744         args = portage.util.shlex_split(cmd)
 745         encoding = _encodings['fs']
 746         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 747                 # Python 3.1 does not support bytes in Popen args.
 748                 args = [_unicode_encode(x,
 749                         encoding=encoding, errors='strict') for x in args]
 750         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 751                 stderr=subprocess.STDOUT)
 752         output = portage._unicode_decode(proc.communicate()[0],
 753                 encoding=encoding, errors='strict')
 754         if output and output[-1] == "\n":
 755                 # getstatusoutput strips one newline
 756                 output = output[:-1]
 757         return (proc.wait(), output)
 758
 759 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 760         """
 761         Implements an interface similar to os.popen(), but with customized
 762         unicode handling (see bug #310789) and without the shell.
 763         """
 764
 765         __slots__ = ('_proc', '_stdout')
 766
 767         def __init__(self, cmd):
 768                 args = portage.util.shlex_split(cmd)
 769                 encoding = _encodings['fs']
 770                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 771                         # Python 3.1 does not support bytes in Popen args.
 772                         args = [_unicode_encode(x,
 773                                 encoding=encoding, errors='strict') for x in args]
 774                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 775                 object.__setattr__(self, '_proc', proc)
 776                 object.__setattr__(self, '_stdout',
 777                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 778
 779         def _get_target(self):
 780                 return object.__getattribute__(self, '_stdout')
 781
 782         __enter__ = _get_target
 783
 784         def __exit__(self, exc_type, exc_value, traceback):
 785                 proc = object.__getattribute__(self, '_proc')
 786                 proc.wait()
 787                 proc.stdout.close()
 788
 789 class ProfileDesc(object):
 790         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 791         def __init__(self, arch, status, sub_path, tree_path):
 792                 self.arch = arch
 793                 self.status = status
 794                 if sub_path:
 795                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 796                 self.sub_path = sub_path
 797                 self.tree_path = tree_path
 798                 if tree_path:
 799                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 800                 else:
 801                         self.abs_path = tree_path
 802
 803         def __str__(self):
 804                 if self.sub_path:
 805                         return self.sub_path
 806                 return 'empty profile'
 807
 808 profile_list = []
 809 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 810
 811 # get lists of valid keywords, licenses, and use
 812 kwlist = set()
 813 liclist = set()
 814 uselist = set()
 815 global_pmasklines = []
 816
 817 for path in portdb.porttrees:
 818         try:
 819                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 820         except OSError:
 821                 pass
 822         kwlist.update(portage.grabfile(os.path.join(path,
 823                 "profiles", "arch.list")))
 824
 825         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 826         for x in use_desc:
 827                 x = x.split()
 828                 if x:
 829                         uselist.add(x[0])
 830
 831         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 832         try:
 833                 expand_list = os.listdir(expand_desc_dir)
 834         except OSError:
 835                 pass
 836         else:
 837                 for fn in expand_list:
 838                         if not fn[-5:] == '.desc':
 839                                 continue
 840                         use_prefix = fn[:-5].lower() + '_'
 841                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 842                                 x = x.split()
 843                                 if x:
 844                                         uselist.add(use_prefix + x[0])
 845
 846         global_pmasklines.append(portage.util.grabfile_package(
 847                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 848
 849         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 850         try:
 851                 desc_file = io.open(_unicode_encode(desc_path,
 852                         encoding=_encodings['fs'], errors='strict'),
 853                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 854         except EnvironmentError:
 855                 pass
 856         else:
 857                 for i, x in enumerate(desc_file):
 858                         if x[0] == "#":
 859                                 continue
 860                         arch = x.split()
 861                         if len(arch) == 0:
 862                                 continue
 863                         if len(arch) != 3:
 864                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 865                                         desc_path + " line %d" % (i+1, ))
 866                         elif arch[0] not in kwlist:
 867                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 868                                         desc_path + " line %d" % (i+1, ))
 869                         elif arch[2] not in valid_profile_types:
 870                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 871                                         desc_path + " line %d" % (i+1, ))
 872                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 873                         if not os.path.isdir(profile_desc.abs_path):
 874                                 logging.error(
 875                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 876                                         arch[2], arch[1], arch[0], desc_path, i+1)
 877                                 continue
 878                         if os.path.exists(
 879                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 880                                 continue
 881                         profile_list.append(profile_desc)
 882                 desc_file.close()
 883
 884 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 885 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 886
 887 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 888 global_pmaskdict = {}
 889 for x in global_pmasklines:
 890         global_pmaskdict.setdefault(x.cp, []).append(x)
 891 del global_pmasklines
 892
 893 def has_global_mask(pkg):
 894         mask_atoms = global_pmaskdict.get(pkg.cp)
 895         if mask_atoms:
 896                 pkg_list = [pkg]
 897                 for x in mask_atoms:
 898                         if portage.dep.match_from_list(x, pkg_list):
 899                                 return x
 900         return None
 901
 902 # Ensure that profile sub_path attributes are unique. Process in reverse order
 903 # so that profiles with duplicate sub_path from overlays will override
 904 # profiles with the same sub_path from parent repos.
 905 profiles = {}
 906 profile_list.reverse()
 907 profile_sub_paths = set()
 908 for prof in profile_list:
 909         if prof.sub_path in profile_sub_paths:
 910                 continue
 911         profile_sub_paths.add(prof.sub_path)
 912         profiles.setdefault(prof.arch, []).append(prof)
 913
 914 # Use an empty profile for checking dependencies of
 915 # packages that have empty KEYWORDS.
 916 prof = ProfileDesc('**', 'stable', '', '')
 917 profiles.setdefault(prof.arch, []).append(prof)
 918
 919 for x in repoman_settings.archlist():
 920         if x[0] == "~":
 921                 continue
 922         if x not in profiles:
 923                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 924                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 925                 print(red("up with the "+x+" team."))
 926                 print()
 927
 928 liclist_deprecated = set()
 929 if "DEPRECATED" in repoman_settings._license_manager._license_groups:
 930         liclist_deprecated.update(
 931                 repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]))
 932
 933 if not liclist:
 934         logging.fatal("Couldn't find licenses?")
 935         sys.exit(1)
 936
 937 if not kwlist:
 938         logging.fatal("Couldn't read KEYWORDS from arch.list")
 939         sys.exit(1)
 940
 941 if not uselist:
 942         logging.fatal("Couldn't find use.desc?")
 943         sys.exit(1)
 944
 945 scanlist=[]
 946 if repolevel==2:
 947         #we are inside a category directory
 948         catdir=reposplit[-1]
 949         if catdir not in categories:
 950                 caterror(catdir)
 951         mydirlist=os.listdir(startdir)
 952         for x in mydirlist:
 953                 if x == "CVS" or x.startswith("."):
 954                         continue
 955                 if os.path.isdir(startdir+"/"+x):
 956                         scanlist.append(catdir+"/"+x)
 957         repo_subdir = catdir + os.sep
 958 elif repolevel==1:
 959         for x in categories:
 960                 if not os.path.isdir(startdir+"/"+x):
 961                         continue
 962                 for y in os.listdir(startdir+"/"+x):
 963                         if y == "CVS" or y.startswith("."):
 964                                 continue
 965                         if os.path.isdir(startdir+"/"+x+"/"+y):
 966                                 scanlist.append(x+"/"+y)
 967         repo_subdir = ""
 968 elif repolevel==3:
 969         catdir = reposplit[-2]
 970         if catdir not in categories:
 971                 caterror(catdir)
 972         scanlist.append(catdir+"/"+reposplit[-1])
 973         repo_subdir = scanlist[-1] + os.sep
 974 else:
 975         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 976                 ' from the current working directory'
 977         logging.critical(msg)
 978         sys.exit(1)
 979
 980 repo_subdir_len = len(repo_subdir)
 981 scanlist.sort()
 982
 983 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 984
 985 def vcs_files_to_cps(vcs_file_iter):
 986         """
 987         Iterate over the given modified file paths returned from the vcs,
 988         and return a frozenset containing category/pn strings for each
 989         modified package.
 990         """
 991
 992         modified_cps = []
 993
 994         if repolevel == 3:
 995                 if reposplit[-2] in categories and \
 996                         next(vcs_file_iter, None) is not None:
 997                         modified_cps.append("/".join(reposplit[-2:]))
 998
 999         elif repolevel == 2:
1000                 category = reposplit[-1]
1001                 if category in categories:
1002                         for filename in vcs_file_iter:
1003                                 f_split = filename.split(os.sep)
1004                                 # ['.', pn,...]
1005                                 if len(f_split) > 2:
1006                                         modified_cps.append(category + "/" + f_split[1])
1007
1008         else:
1009                 # repolevel == 1
1010                 for filename in vcs_file_iter:
1011                         f_split = filename.split(os.sep)
1012                         # ['.', category, pn,...]
1013                         if len(f_split) > 3 and f_split[1] in categories:
1014                                 modified_cps.append("/".join(f_split[1:3]))
1015
1016         return frozenset(modified_cps)
1017
1018 def git_supports_gpg_sign():
1019         status, cmd_output = \
1020                 repoman_getstatusoutput("git --version")
1021         cmd_output = cmd_output.split()
1022         if cmd_output:
1023                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1024                 if version is not None:
1025                         version = [int(x) for x in version.groups()[1:]]
1026                         if version[0] > 1 or \
1027                                 (version[0] == 1 and version[1] > 7) or \
1028                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1029                                 return True
1030         return False
1031
1032 def dev_keywords(profiles):
1033         """
1034         Create a set of KEYWORDS values that exist in 'dev'
1035         profiles. These are used
1036         to trigger a message notifying the user when they might
1037         want to add the --include-dev option.
1038         """
1039         type_arch_map = {}
1040         for arch, arch_profiles in profiles.items():
1041                 for prof in arch_profiles:
1042                         arch_set = type_arch_map.get(prof.status)
1043                         if arch_set is None:
1044                                 arch_set = set()
1045                                 type_arch_map[prof.status] = arch_set
1046                         arch_set.add(arch)
1047
1048         dev_keywords = type_arch_map.get('dev', set())
1049         dev_keywords.update(['~' + arch for arch in dev_keywords])
1050         return frozenset(dev_keywords)
1051
1052 dev_keywords = dev_keywords(profiles)
1053
1054 stats={}
1055 fails={}
1056
1057 for x in qacats:
1058         stats[x]=0
1059         fails[x]=[]
1060
1061 xmllint_capable = False
1062 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1063
1064 def parsedate(s):
1065         """Parse a RFC 822 date and time string.
1066         This is required for python3 compatibility, since the
1067         rfc822.parsedate() function is not available."""
1068
1069         s_split = []
1070         for x in s.upper().split():
1071                 for y in x.split(','):
1072                         if y:
1073                                 s_split.append(y)
1074
1075         if len(s_split) != 6:
1076                 return None
1077
1078         # %a, %d %b %Y %H:%M:%S %Z
1079         a, d, b, Y, H_M_S, Z = s_split
1080
1081         # Convert month to integer, since strptime %w is locale-dependent.
1082         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1083                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1084         m = month_map.get(b)
1085         if m is None:
1086                 return None
1087         m = str(m).rjust(2, '0')
1088
1089         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1090
1091 def fetch_metadata_dtd():
1092         """
1093         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1094         metadata_dtd_ctime_interval.
1095         @rtype: bool
1096         @return: True if successful, otherwise False
1097         """
1098
1099         must_fetch = True
1100         metadata_dtd_st = None
1101         current_time = int(time.time())
1102         try:
1103                 metadata_dtd_st = os.stat(metadata_dtd)
1104         except EnvironmentError as e:
1105                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1106                         raise
1107                 del e
1108         else:
1109                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1110                 if abs(current_time - metadata_dtd_st.st_ctime) \
1111                         < metadata_dtd_ctime_interval:
1112                         must_fetch = False
1113
1114         if must_fetch:
1115                 print()
1116                 print(green("***") + " the local copy of metadata.dtd " + \
1117                         "needs to be refetched, doing that now")
1118                 print()
1119                 try:
1120                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1121                         msg_info = url_f.info()
1122                         last_modified = msg_info.get('last-modified')
1123                         if last_modified is not None:
1124                                 last_modified = parsedate(last_modified)
1125                                 if last_modified is not None:
1126                                         last_modified = calendar.timegm(last_modified)
1127
1128                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1129                         try:
1130                                 local_f = open(metadata_dtd_tmp, mode='wb')
1131                                 local_f.write(url_f.read())
1132                                 local_f.close()
1133                                 if last_modified is not None:
1134                                         try:
1135                                                 os.utime(metadata_dtd_tmp,
1136                                                         (int(last_modified), int(last_modified)))
1137                                         except OSError:
1138                                                 # This fails on some odd non-unix-like filesystems.
1139                                                 # We don't really need the mtime to be preserved
1140                                                 # anyway here (currently we use ctime to trigger
1141                                                 # fetch), so just ignore it.
1142                                                 pass
1143                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1144                         finally:
1145                                 try:
1146                                         os.unlink(metadata_dtd_tmp)
1147                                 except OSError:
1148                                         pass
1149
1150                         url_f.close()
1151
1152                 except EnvironmentError as e:
1153                         print()
1154                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1155                         print(red("!!!")+" exception '%s' though." % (e,))
1156                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1157                         return False
1158
1159         return True
1160
1161 if options.mode == "manifest":
1162         pass
1163 elif not find_binary('xmllint'):
1164         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1165         if options.xml_parse or repolevel==3:
1166                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1167                 sys.exit(1)
1168 else:
1169         if not fetch_metadata_dtd():
1170                 sys.exit(1)
1171         #this can be problematic if xmllint changes their output
1172         xmllint_capable=True
1173
1174 if options.mode == 'commit' and vcs:
1175         utilities.detect_vcs_conflicts(options, vcs)
1176
1177 if options.mode == "manifest":
1178         pass
1179 elif options.pretend:
1180         print(green("\nRepoMan does a once-over of the neighborhood..."))
1181 else:
1182         print(green("\nRepoMan scours the neighborhood..."))
1183
1184 new_ebuilds = set()
1185 modified_ebuilds = set()
1186 modified_changelogs = set()
1187 mychanged = []
1188 mynew = []
1189 myremoved = []
1190
1191 if vcs == "cvs":
1192         mycvstree = cvstree.getentries("./", recursive=1)
1193         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1194         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1195         if options.if_modified == "y":
1196                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1197
1198 elif vcs == "svn":
1199         with repoman_popen("svn status") as f:
1200                 svnstatus = f.readlines()
1201         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1202         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1203         if options.if_modified == "y":
1204                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1205
1206 elif vcs == "git":
1207         with repoman_popen("git diff-index --name-only "
1208                 "--relative --diff-filter=M HEAD") as f:
1209                 mychanged = f.readlines()
1210         mychanged = ["./" + elem[:-1] for elem in mychanged]
1211
1212         with repoman_popen("git diff-index --name-only "
1213                 "--relative --diff-filter=A HEAD") as f:
1214                 mynew = f.readlines()
1215         mynew = ["./" + elem[:-1] for elem in mynew]
1216         if options.if_modified == "y":
1217                 with repoman_popen("git diff-index --name-only "
1218                         "--relative --diff-filter=D HEAD") as f:
1219                         myremoved = f.readlines()
1220                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1221
1222 elif vcs == "bzr":
1223         with repoman_popen("bzr status -S .") as f:
1224                 bzrstatus = f.readlines()
1225         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1226         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1227         if options.if_modified == "y":
1228                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1229
1230 elif vcs == "hg":
1231         with repoman_popen("hg status --no-status --modified .") as f:
1232                 mychanged = f.readlines()
1233         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1234         mynew = repoman_popen("hg status --no-status --added .").readlines()
1235         mynew = ["./" + elem.rstrip() for elem in mynew]
1236         if options.if_modified == "y":
1237                 with repoman_popen("hg status --no-status --removed .") as f:
1238                         myremoved = f.readlines()
1239                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1240
1241 if vcs:
1242         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1243         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1244         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1245                 if os.path.basename(x) == "ChangeLog")
1246
1247 def vcs_new_changed(relative_path):
1248         for x in chain(mychanged, mynew):
1249                 if x == relative_path:
1250                         return True
1251         return False
1252
1253 have_pmasked = False
1254 have_dev_keywords = False
1255 dofail = 0
1256
1257 # NOTE: match-all caches are not shared due to potential
1258 # differences between profiles in _get_implicit_iuse.
1259 arch_caches={}
1260 arch_xmatch_caches = {}
1261 shared_xmatch_caches = {"cp-list":{}}
1262
1263 # Disable the "ebuild.notadded" check when not in commit mode and
1264 # running `svn status` in every package dir will be too expensive.
1265
1266 check_ebuild_notadded = not \
1267         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1268
1269 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1270 thirdpartymirrors = []
1271 for v in repoman_settings.thirdpartymirrors().values():
1272         for v in v:
1273                 if not v.endswith("/"):
1274                         v += "/"
1275                 thirdpartymirrors.append(v)
1276
1277 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1278         """
1279         Implements doctype() as required to avoid deprecation warnings with
1280         >=python-2.7.
1281         """
1282         def doctype(self, name, pubid, system):
1283                 pass
1284
1285 try:
1286         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1287 except (EnvironmentError, ParseError, PermissionDenied) as e:
1288         err(str(e))
1289 except FileNotFound:
1290         # TODO: Download as we do for metadata.dtd, but add a way to
1291         # disable for non-gentoo repoman users who may not have herds.
1292         herd_base = None
1293
1294 effective_scanlist = scanlist
1295 if options.if_modified == "y":
1296         effective_scanlist = sorted(vcs_files_to_cps(
1297                 chain(mychanged, mynew, myremoved)))
1298
1299 for x in effective_scanlist:
1300         #ebuilds and digests added to cvs respectively.
1301         logging.info("checking package %s" % x)
1302         # save memory by discarding xmatch caches from previous package(s)
1303         arch_xmatch_caches.clear()
1304         eadded=[]
1305         catdir,pkgdir=x.split("/")
1306         checkdir=repodir+"/"+x
1307         checkdir_relative = ""
1308         if repolevel < 3:
1309                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1310         if repolevel < 2:
1311                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1312         checkdir_relative = os.path.join(".", checkdir_relative)
1313         generated_manifest = False
1314
1315         if options.mode == "manifest" or \
1316           (options.mode != 'manifest-check' and options.digest == 'y') or \
1317           options.mode in ('commit', 'fix') and not options.pretend:
1318                 auto_assumed = set()
1319                 fetchlist_dict = portage.FetchlistDict(checkdir,
1320                         repoman_settings, portdb)
1321                 if options.mode == 'manifest' and options.force:
1322                         portage._doebuild_manifest_exempt_depend += 1
1323                         try:
1324                                 distdir = repoman_settings['DISTDIR']
1325                                 mf = repoman_settings.repositories.get_repo_for_location(
1326                                         os.path.dirname(os.path.dirname(checkdir)))
1327                                 mf = mf.load_manifest(checkdir, distdir,
1328                                         fetchlist_dict=fetchlist_dict)
1329                                 mf.create(requiredDistfiles=None,
1330                                         assumeDistHashesAlways=True)
1331                                 for distfiles in fetchlist_dict.values():
1332                                         for distfile in distfiles:
1333                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1334                                                         mf.fhashdict['DIST'].pop(distfile, None)
1335                                                 else:
1336                                                         auto_assumed.add(distfile)
1337                                 mf.write()
1338                         finally:
1339                                 portage._doebuild_manifest_exempt_depend -= 1
1340
1341                 repoman_settings["O"] = checkdir
1342                 try:
1343                         generated_manifest = digestgen(
1344                                 mysettings=repoman_settings, myportdb=portdb)
1345                 except portage.exception.PermissionDenied as e:
1346                         generated_manifest = False
1347                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1348                                 level=logging.ERROR, noiselevel=-1)
1349
1350                 if not generated_manifest:
1351                         print("Unable to generate manifest.")
1352                         dofail = 1
1353
1354                 if options.mode == "manifest":
1355                         if not dofail and options.force and auto_assumed and \
1356                                 'assume-digests' in repoman_settings.features:
1357                                 # Show which digests were assumed despite the --force option
1358                                 # being given. This output will already have been shown by
1359                                 # digestgen() if assume-digests is not enabled, so only show
1360                                 # it here if assume-digests is enabled.
1361                                 pkgs = list(fetchlist_dict)
1362                                 pkgs.sort()
1363                                 portage.writemsg_stdout("  digest.assumed" + \
1364                                         portage.output.colorize("WARN",
1365                                         str(len(auto_assumed)).rjust(18)) + "\n")
1366                                 for cpv in pkgs:
1367                                         fetchmap = fetchlist_dict[cpv]
1368                                         pf = portage.catsplit(cpv)[1]
1369                                         for distfile in sorted(fetchmap):
1370                                                 if distfile in auto_assumed:
1371                                                         portage.writemsg_stdout(
1372                                                                 "   %s::%s\n" % (pf, distfile))
1373                         continue
1374                 elif dofail:
1375                         sys.exit(1)
1376
1377         if not generated_manifest:
1378                 repoman_settings['O'] = checkdir
1379                 repoman_settings['PORTAGE_QUIET'] = '1'
1380                 if not portage.digestcheck([], repoman_settings, strict=1):
1381                         stats["manifest.bad"] += 1
1382                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1383                 repoman_settings.pop('PORTAGE_QUIET', None)
1384
1385         if options.mode == 'manifest-check':
1386                 continue
1387
1388         checkdirlist=os.listdir(checkdir)
1389         ebuildlist=[]
1390         pkgs = {}
1391         allvalid = True
1392         for y in checkdirlist:
1393                 if (y in no_exec or y.endswith(".ebuild")) and \
1394                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1395                                 stats["file.executable"] += 1
1396                                 fails["file.executable"].append(os.path.join(checkdir, y))
1397                 if y.endswith(".ebuild"):
1398                         pf = y[:-7]
1399                         ebuildlist.append(pf)
1400                         cpv = "%s/%s" % (catdir, pf)
1401                         try:
1402                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1403                         except KeyError:
1404                                 allvalid = False
1405                                 stats["ebuild.syntax"] += 1
1406                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1407                                 continue
1408                         except IOError:
1409                                 allvalid = False
1410                                 stats["ebuild.output"] += 1
1411                                 fails["ebuild.output"].append(os.path.join(x, y))
1412                                 continue
1413                         if not portage.eapi_is_supported(myaux["EAPI"]):
1414                                 allvalid = False
1415                                 stats["EAPI.unsupported"] += 1
1416                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1417                                 continue
1418                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1419                                 root_config=root_config, type_name="ebuild")
1420
1421         slot_keywords = {}
1422
1423         if len(pkgs) != len(ebuildlist):
1424                 # If we can't access all the metadata then it's totally unsafe to
1425                 # commit since there's no way to generate a correct Manifest.
1426                 # Do not try to do any more QA checks on this package since missing
1427                 # metadata leads to false positives for several checks, and false
1428                 # positives confuse users.
1429                 can_force = False
1430                 continue
1431
1432         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1433         ebuildlist = sorted(pkgs.values())
1434         ebuildlist = [pkg.pf for pkg in ebuildlist]
1435
1436         for y in checkdirlist:
1437                 index = repo_config.find_invalid_path_char(y)
1438                 if index != -1:
1439                         y_relative = os.path.join(checkdir_relative, y)
1440                         if vcs is not None and not vcs_new_changed(y_relative):
1441                                 # If the file isn't in the VCS new or changed set, then
1442                                 # assume that it's an irrelevant temporary file (Manifest
1443                                 # entries are not generated for file names containing
1444                                 # prohibited characters). See bug #406877.
1445                                 index = -1
1446                 if index != -1:
1447                         stats["file.name"] += 1
1448                         fails["file.name"].append("%s/%s: char '%s'" % \
1449                                 (checkdir, y, y[index]))
1450
1451                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1452                         continue
1453                 f = None
1454                 try:
1455                         line = 1
1456                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1457                                 encoding=_encodings['fs'], errors='strict'),
1458                                 mode='r', encoding=_encodings['repo.content'])
1459                         for l in f:
1460                                 line +=1
1461                 except UnicodeDecodeError as ue:
1462                         stats["file.UTF8"] += 1
1463                         s = ue.object[:ue.start]
1464                         l2 = s.count("\n")
1465                         line += l2
1466                         if l2 != 0:
1467                                 s = s[s.rfind("\n") + 1:]
1468                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1469                 finally:
1470                         if f is not None:
1471                                 f.close()
1472
1473         if vcs in ("git", "hg") and check_ebuild_notadded:
1474                 if vcs == "git":
1475                         myf = repoman_popen("git ls-files --others %s" % \
1476                                 (portage._shell_quote(checkdir_relative),))
1477                 if vcs == "hg":
1478                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1479                                 (portage._shell_quote(checkdir_relative),))
1480                 for l in myf:
1481                         if l[:-1][-7:] == ".ebuild":
1482                                 stats["ebuild.notadded"] += 1
1483                                 fails["ebuild.notadded"].append(
1484                                         os.path.join(x, os.path.basename(l[:-1])))
1485                 myf.close()
1486
1487         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1488                 try:
1489                         if vcs == "cvs":
1490                                 myf=open(checkdir+"/CVS/Entries","r")
1491                         if vcs == "svn":
1492                                 myf = repoman_popen("svn status --depth=files --verbose " +
1493                                         portage._shell_quote(checkdir))
1494                         if vcs == "bzr":
1495                                 myf = repoman_popen("bzr ls -v --kind=file " +
1496                                         portage._shell_quote(checkdir))
1497                         myl = myf.readlines()
1498                         myf.close()
1499                         for l in myl:
1500                                 if vcs == "cvs":
1501                                         if l[0]!="/":
1502                                                 continue
1503                                         splitl=l[1:].split("/")
1504                                         if not len(splitl):
1505                                                 continue
1506                                         if splitl[0][-7:]==".ebuild":
1507                                                 eadded.append(splitl[0][:-7])
1508                                 if vcs == "svn":
1509                                         if l[:1] == "?":
1510                                                 continue
1511                                         if l[:7] == '      >':
1512                                                 # tree conflict, new in subversion 1.6
1513                                                 continue
1514                                         l = l.split()[-1]
1515                                         if l[-7:] == ".ebuild":
1516                                                 eadded.append(os.path.basename(l[:-7]))
1517                                 if vcs == "bzr":
1518                                         if l[1:2] == "?":
1519                                                 continue
1520                                         l = l.split()[-1]
1521                                         if l[-7:] == ".ebuild":
1522                                                 eadded.append(os.path.basename(l[:-7]))
1523                         if vcs == "svn":
1524                                 myf = repoman_popen("svn status " +
1525                                         portage._shell_quote(checkdir))
1526                                 myl=myf.readlines()
1527                                 myf.close()
1528                                 for l in myl:
1529                                         if l[0] == "A":
1530                                                 l = l.rstrip().split(' ')[-1]
1531                                                 if l[-7:] == ".ebuild":
1532                                                         eadded.append(os.path.basename(l[:-7]))
1533                 except IOError:
1534                         if vcs == "cvs":
1535                                 stats["CVS/Entries.IO_error"] += 1
1536                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1537                         else:
1538                                 raise
1539                         continue
1540
1541         mf = repoman_settings.repositories.get_repo_for_location(
1542                 os.path.dirname(os.path.dirname(checkdir)))
1543         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1544         mydigests=mf.getTypeDigests("DIST")
1545
1546         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1547         myfiles_all = []
1548         src_uri_error = False
1549         for mykey in fetchlist_dict:
1550                 try:
1551                         myfiles_all.extend(fetchlist_dict[mykey])
1552                 except portage.exception.InvalidDependString as e:
1553                         src_uri_error = True
1554                         try:
1555                                 portdb.aux_get(mykey, ["SRC_URI"])
1556                         except KeyError:
1557                                 # This will be reported as an "ebuild.syntax" error.
1558                                 pass
1559                         else:
1560                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1561                                 fails["SRC_URI.syntax"].append(
1562                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1563         del fetchlist_dict
1564         if not src_uri_error:
1565                 # This test can produce false positives if SRC_URI could not
1566                 # be parsed for one or more ebuilds. There's no point in
1567                 # producing a false error here since the root cause will
1568                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1569                 # or "ebuild.sytax".
1570                 myfiles_all = set(myfiles_all)
1571                 for entry in mydigests:
1572                         if entry not in myfiles_all:
1573                                 stats["digest.unused"] += 1
1574                                 fails["digest.unused"].append(checkdir+"::"+entry)
1575                 for entry in myfiles_all:
1576                         if entry not in mydigests:
1577                                 stats["digest.missing"] += 1
1578                                 fails["digest.missing"].append(checkdir+"::"+entry)
1579         del myfiles_all
1580
1581         if os.path.exists(checkdir+"/files"):
1582                 filesdirlist=os.listdir(checkdir+"/files")
1583
1584                 # recurse through files directory
1585                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1586                 while filesdirlist:
1587                         y = filesdirlist.pop(0)
1588                         relative_path = os.path.join(x, "files", y)
1589                         full_path = os.path.join(repodir, relative_path)
1590                         try:
1591                                 mystat = os.stat(full_path)
1592                         except OSError as oe:
1593                                 if oe.errno == 2:
1594                                         # don't worry about it.  it likely was removed via fix above.
1595                                         continue
1596                                 else:
1597                                         raise oe
1598                         if S_ISDIR(mystat.st_mode):
1599                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1600                                 if y == "CVS" or y == ".svn":
1601                                         continue
1602                                 for z in os.listdir(checkdir+"/files/"+y):
1603                                         if z == "CVS" or z == ".svn":
1604                                                 continue
1605                                         filesdirlist.append(y+"/"+z)
1606                         # Current policy is no files over 20 KiB, these are the checks. File size between
1607                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1608                         elif mystat.st_size > 61440:
1609                                 stats["file.size.fatal"] += 1
1610                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1611                         elif mystat.st_size > 20480:
1612                                 stats["file.size"] += 1
1613                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1614
1615                         index = repo_config.find_invalid_path_char(y)
1616                         if index != -1:
1617                                 y_relative = os.path.join(checkdir_relative, "files", y)
1618                                 if vcs is not None and not vcs_new_changed(y_relative):
1619                                         # If the file isn't in the VCS new or changed set, then
1620                                         # assume that it's an irrelevant temporary file (Manifest
1621                                         # entries are not generated for file names containing
1622                                         # prohibited characters). See bug #406877.
1623                                         index = -1
1624                         if index != -1:
1625                                 stats["file.name"] += 1
1626                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1627                                         (checkdir, y, y[index]))
1628         del mydigests
1629
1630         if check_changelog and "ChangeLog" not in checkdirlist:
1631                 stats["changelog.missing"]+=1
1632                 fails["changelog.missing"].append(x+"/ChangeLog")
1633
1634         musedict = {}
1635         #metadata.xml file check
1636         if "metadata.xml" not in checkdirlist:
1637                 stats["metadata.missing"]+=1
1638                 fails["metadata.missing"].append(x+"/metadata.xml")
1639         #metadata.xml parse check
1640         else:
1641                 metadata_bad = False
1642
1643                 # read metadata.xml into memory
1644                 try:
1645                         _metadata_xml = xml.etree.ElementTree.parse(
1646                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1647                                 encoding=_encodings['fs'], errors='strict'),
1648                                 parser=xml.etree.ElementTree.XMLParser(
1649                                         target=_MetadataTreeBuilder()))
1650                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1651                         metadata_bad = True
1652                         stats["metadata.bad"] += 1
1653                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1654                         del e
1655                 else:
1656                         # load USE flags from metadata.xml
1657                         try:
1658                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1659                         except portage.exception.ParseError as e:
1660                                 metadata_bad = True
1661                                 stats["metadata.bad"] += 1
1662                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1663                         else:
1664                                 for atom in chain(*musedict.values()):
1665                                         if atom is None:
1666                                                 continue
1667                                         try:
1668                                                 atom = Atom(atom)
1669                                         except InvalidAtom as e:
1670                                                 stats["metadata.bad"] += 1
1671                                                 fails["metadata.bad"].append(
1672                                                         "%s/metadata.xml: Invalid atom: %s" % (x, e))
1673                                         else:
1674                                                 if atom.cp != x:
1675                                                         stats["metadata.bad"] += 1
1676                                                         fails["metadata.bad"].append(
1677                                                                 ("%s/metadata.xml: Atom contains "
1678                                                                 "unexpected cat/pn: %s") % (x, atom))
1679
1680                         # Run other metadata.xml checkers
1681                         try:
1682                                 utilities.check_metadata(_metadata_xml, herd_base)
1683                         except (utilities.UnknownHerdsError, ) as e:
1684                                 metadata_bad = True
1685                                 stats["metadata.bad"] += 1
1686                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1687                                 del e
1688
1689                 #Only carry out if in package directory or check forced
1690                 if xmllint_capable and not metadata_bad:
1691                         # xmlint can produce garbage output even on success, so only dump
1692                         # the ouput when it fails.
1693                         st, out = repoman_getstatusoutput(
1694                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1695                                  (portage._shell_quote(metadata_dtd),
1696                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1697                         if st != os.EX_OK:
1698                                 print(red("!!!") + " metadata.xml is invalid:")
1699                                 for z in out.splitlines():
1700                                         print(red("!!! ")+z)
1701                                 stats["metadata.bad"]+=1
1702                                 fails["metadata.bad"].append(x+"/metadata.xml")
1703
1704                 del metadata_bad
1705         muselist = frozenset(musedict)
1706
1707         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1708         changelog_modified = changelog_path in modified_changelogs
1709
1710         # detect unused local USE-descriptions
1711         used_useflags = set()
1712
1713         for y in ebuildlist:
1714                 relative_path = os.path.join(x, y + ".ebuild")
1715                 full_path = os.path.join(repodir, relative_path)
1716                 ebuild_path = y + ".ebuild"
1717                 if repolevel < 3:
1718                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1719                 if repolevel < 2:
1720                         ebuild_path = os.path.join(catdir, ebuild_path)
1721                 ebuild_path = os.path.join(".", ebuild_path)
1722                 if check_changelog and not changelog_modified \
1723                         and ebuild_path in new_ebuilds:
1724                         stats['changelog.ebuildadded'] += 1
1725                         fails['changelog.ebuildadded'].append(relative_path)
1726
1727                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1728                         #ebuild not added to vcs
1729                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1730                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1731                 myesplit=portage.pkgsplit(y)
1732                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1733                             or pv_toolong_re.search(myesplit[1]) \
1734                             or pv_toolong_re.search(myesplit[2]):
1735                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1736                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1737                         continue
1738                 elif myesplit[0]!=pkgdir:
1739                         print(pkgdir,myesplit[0])
1740                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1741                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1742                         continue
1743
1744                 pkg = pkgs[y]
1745
1746                 if pkg.invalid:
1747                         allvalid = False
1748                         for k, msgs in pkg.invalid.items():
1749                                 for msg in msgs:
1750                                         stats[k] = stats[k] + 1
1751                                         fails[k].append("%s: %s" % (relative_path, msg))
1752                         continue
1753
1754                 myaux = pkg._metadata
1755                 eapi = myaux["EAPI"]
1756                 inherited = pkg.inherited
1757                 live_ebuild = live_eclasses.intersection(inherited)
1758
1759                 for k, v in myaux.items():
1760                         if not isinstance(v, basestring):
1761                                 continue
1762                         m = non_ascii_re.search(v)
1763                         if m is not None:
1764                                 stats["variable.invalidchar"] += 1
1765                                 fails["variable.invalidchar"].append(
1766                                         ("%s: %s variable contains non-ASCII " + \
1767                                         "character at position %s") % \
1768                                         (relative_path, k, m.start() + 1))
1769
1770                 if not src_uri_error:
1771                         # Check that URIs don't reference a server from thirdpartymirrors.
1772                         for uri in portage.dep.use_reduce( \
1773                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1774                                 contains_mirror = False
1775                                 for mirror in thirdpartymirrors:
1776                                         if uri.startswith(mirror):
1777                                                 contains_mirror = True
1778                                                 break
1779                                 if not contains_mirror:
1780                                         continue
1781
1782                                 stats["SRC_URI.mirror"] += 1
1783                                 fails["SRC_URI.mirror"].append(
1784                                         "%s: '%s' found in thirdpartymirrors" % \
1785                                         (relative_path, mirror))
1786
1787                 if myaux.get("PROVIDE"):
1788                         stats["virtual.oldstyle"]+=1
1789                         fails["virtual.oldstyle"].append(relative_path)
1790
1791                 for pos, missing_var in enumerate(missingvars):
1792                         if not myaux.get(missing_var):
1793                                 if catdir == "virtual" and \
1794                                         missing_var in ("HOMEPAGE", "LICENSE"):
1795                                         continue
1796                                 if live_ebuild and missing_var == "KEYWORDS":
1797                                         continue
1798                                 myqakey=missingvars[pos]+".missing"
1799                                 stats[myqakey]=stats[myqakey]+1
1800                                 fails[myqakey].append(x+"/"+y+".ebuild")
1801
1802                 if catdir == "virtual":
1803                         for var in ("HOMEPAGE", "LICENSE"):
1804                                 if myaux.get(var):
1805                                         myqakey = var + ".virtual"
1806                                         stats[myqakey] = stats[myqakey] + 1
1807                                         fails[myqakey].append(relative_path)
1808
1809                 # 14 is the length of DESCRIPTION=""
1810                 if len(myaux['DESCRIPTION']) > max_desc_len:
1811                         stats['DESCRIPTION.toolong'] += 1
1812                         fails['DESCRIPTION.toolong'].append(
1813                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1814                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1815
1816                 keywords = myaux["KEYWORDS"].split()
1817                 stable_keywords = []
1818                 for keyword in keywords:
1819                         if not keyword.startswith("~") and \
1820                                 not keyword.startswith("-"):
1821                                 stable_keywords.append(keyword)
1822                 if stable_keywords:
1823                         if ebuild_path in new_ebuilds:
1824                                 stable_keywords.sort()
1825                                 stats["KEYWORDS.stable"] += 1
1826                                 fails["KEYWORDS.stable"].append(
1827                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1828                                                 " ".join(stable_keywords))
1829
1830                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1831                         if not kw.startswith("-"))
1832
1833                 previous_keywords = slot_keywords.get(pkg.slot)
1834                 if previous_keywords is None:
1835                         slot_keywords[pkg.slot] = set()
1836                 elif ebuild_archs and "*" not in ebuild_archs and not live_ebuild:
1837                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1838                         if dropped_keywords:
1839                                 stats["KEYWORDS.dropped"] += 1
1840                                 fails["KEYWORDS.dropped"].append(
1841                                         relative_path + ": %s" % \
1842                                         " ".join(sorted(dropped_keywords)))
1843
1844                 slot_keywords[pkg.slot].update(ebuild_archs)
1845
1846                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1847                 if "-*" in keywords:
1848                         haskeyword = False
1849                         for kw in keywords:
1850                                 if kw[0] == "~":
1851                                         kw = kw[1:]
1852                                 if kw in kwlist:
1853                                         haskeyword = True
1854                         if not haskeyword:
1855                                 stats["KEYWORDS.stupid"] += 1
1856                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1857
1858                 """
1859                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1860                 not be allowed to be marked stable
1861                 """
1862                 if live_ebuild and repo_config.name == "gentoo":
1863                         bad_stable_keywords = []
1864                         for keyword in keywords:
1865                                 if not keyword.startswith("~") and \
1866                                         not keyword.startswith("-"):
1867                                         bad_stable_keywords.append(keyword)
1868                                 del keyword
1869                         if bad_stable_keywords:
1870                                 stats["LIVEVCS.stable"] += 1
1871                                 fails["LIVEVCS.stable"].append(
1872                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1873                                                 bad_stable_keywords)
1874                         del bad_stable_keywords
1875
1876                         if keywords and not has_global_mask(pkg):
1877                                 stats["LIVEVCS.unmasked"] += 1
1878                                 fails["LIVEVCS.unmasked"].append(relative_path)
1879
1880                 if options.ignore_arches:
1881                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1882                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1883                 else:
1884                         arches=[]
1885                         for keyword in myaux["KEYWORDS"].split():
1886                                 if (keyword[0]=="-"):
1887                                         continue
1888                                 elif (keyword[0]=="~"):
1889                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1890                                 else:
1891                                         arches.append([keyword, keyword, [keyword]])
1892                         if not arches:
1893                                 # Use an empty profile for checking dependencies of
1894                                 # packages that have empty KEYWORDS.
1895                                 arches.append(['**', '**', ['**']])
1896
1897                 unknown_pkgs = set()
1898                 baddepsyntax = False
1899                 badlicsyntax = False
1900                 badprovsyntax = False
1901                 catpkg = catdir+"/"+y
1902
1903                 inherited_java_eclass = "java-pkg-2" in inherited or \
1904                         "java-pkg-opt-2" in inherited
1905                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1906                 operator_tokens = set(["||", "(", ")"])
1907                 type_list, badsyntax = [], []
1908                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
1909                         mydepstr = myaux[mytype]
1910
1911                         buildtime = mytype in Package._buildtime_keys
1912                         runtime = mytype in Package._runtime_keys
1913                         token_class = None
1914                         if mytype.endswith("DEPEND"):
1915                                 token_class=portage.dep.Atom
1916
1917                         try:
1918                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1919                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1920                         except portage.exception.InvalidDependString as e:
1921                                 atoms = None
1922                                 badsyntax.append(str(e))
1923
1924                         if atoms and mytype.endswith("DEPEND"):
1925                                 if runtime and \
1926                                         "test?" in mydepstr.split():
1927                                         stats[mytype + '.suspect'] += 1
1928                                         fails[mytype + '.suspect'].append(relative_path + \
1929                                                 ": 'test?' USE conditional in %s" % mytype)
1930
1931                                 for atom in atoms:
1932                                         if atom == "||":
1933                                                 continue
1934
1935                                         # Skip dependency.unknown for blockers, so that we
1936                                         # don't encourage people to remove necessary blockers,
1937                                         # as discussed in bug #382407.
1938                                         if atom.blocker is None and \
1939                                                 not portdb.xmatch("match-all", atom) and \
1940                                                 not atom.cp.startswith("virtual/"):
1941                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1942
1943                                         is_blocker = atom.blocker
1944
1945                                         if catdir != "virtual":
1946                                                 if not is_blocker and \
1947                                                         atom.cp in suspect_virtual:
1948                                                         stats['virtual.suspect'] += 1
1949                                                         fails['virtual.suspect'].append(
1950                                                                 relative_path +
1951                                                                 ": %s: consider using '%s' instead of '%s'" %
1952                                                                 (mytype, suspect_virtual[atom.cp], atom))
1953
1954                                         if buildtime and \
1955                                                 not is_blocker and \
1956                                                 not inherited_java_eclass and \
1957                                                 atom.cp == "virtual/jdk":
1958                                                 stats['java.eclassesnotused'] += 1
1959                                                 fails['java.eclassesnotused'].append(relative_path)
1960                                         elif buildtime and \
1961                                                 not is_blocker and \
1962                                                 not inherited_wxwidgets_eclass and \
1963                                                 atom.cp == "x11-libs/wxGTK":
1964                                                 stats['wxwidgets.eclassnotused'] += 1
1965                                                 fails['wxwidgets.eclassnotused'].append(
1966                                                         (relative_path + ": %ss on x11-libs/wxGTK"
1967                                                         " without inheriting wxwidgets.eclass") % mytype)
1968                                         elif runtime:
1969                                                 if not is_blocker and \
1970                                                         atom.cp in suspect_rdepend:
1971                                                         stats[mytype + '.suspect'] += 1
1972                                                         fails[mytype + '.suspect'].append(
1973                                                                 relative_path + ": '%s'" % atom)
1974
1975                                         if atom.operator == "~" and \
1976                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1977                                                 qacat = 'dependency.badtilde'
1978                                                 stats[qacat] += 1
1979                                                 fails[qacat].append(
1980                                                         (relative_path + ": %s uses the ~ operator"
1981                                                          " with a non-zero revision:" + \
1982                                                          " '%s'") % (mytype, atom))
1983
1984                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1985
1986                 for m,b in zip(type_list, badsyntax):
1987                         if m.endswith("DEPEND"):
1988                                 qacat = "dependency.syntax"
1989                         else:
1990                                 qacat = m + ".syntax"
1991                         stats[qacat] += 1
1992                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
1993
1994                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1995                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1996                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1997                 badlicsyntax = badlicsyntax > 0
1998                 badprovsyntax = badprovsyntax > 0
1999
2000                 # uselist checks - global
2001                 myuse = []
2002                 default_use = []
2003                 for myflag in myaux["IUSE"].split():
2004                         flag_name = myflag.lstrip("+-")
2005                         used_useflags.add(flag_name)
2006                         if myflag != flag_name:
2007                                 default_use.append(myflag)
2008                         if flag_name not in uselist:
2009                                 myuse.append(flag_name)
2010
2011                 # uselist checks - metadata
2012                 for mypos in range(len(myuse)-1,-1,-1):
2013                         if myuse[mypos] and (myuse[mypos] in muselist):
2014                                 del myuse[mypos]
2015
2016                 if default_use and not eapi_has_iuse_defaults(eapi):
2017                         for myflag in default_use:
2018                                 stats['EAPI.incompatible'] += 1
2019                                 fails['EAPI.incompatible'].append(
2020                                         (relative_path + ": IUSE defaults" + \
2021                                         " not supported with EAPI='%s':" + \
2022                                         " '%s'") % (eapi, myflag))
2023
2024                 for mypos in range(len(myuse)):
2025                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
2026                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
2027
2028                 # license checks
2029                 if not badlicsyntax:
2030                         # Parse the LICENSE variable, remove USE conditions and
2031                         # flatten it.
2032                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2033                         # Check each entry to ensure that it exists in PORTDIR's
2034                         # license directory.
2035                         for lic in licenses:
2036                                 # Need to check for "||" manually as no portage
2037                                 # function will remove it without removing values.
2038                                 if lic not in liclist and lic != "||":
2039                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
2040                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
2041                                 elif lic in liclist_deprecated:
2042                                         stats["LICENSE.deprecated"] = stats["LICENSE.deprecated"] + 1
2043                                         fails["LICENSE.deprecated"].append("%s: %s" % (relative_path, lic))
2044
2045                 #keyword checks
2046                 myuse = myaux["KEYWORDS"].split()
2047                 for mykey in myuse:
2048                         if mykey not in ("-*", "*", "~*"):
2049                                 myskey = mykey
2050                                 if myskey[:1] == "-":
2051                                         myskey = myskey[1:]
2052                                 if myskey[:1] == "~":
2053                                         myskey = myskey[1:]
2054                                 if myskey not in kwlist:
2055                                         stats["KEYWORDS.invalid"] += 1
2056                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
2057                                 elif myskey not in profiles:
2058                                         stats["KEYWORDS.invalid"] += 1
2059                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
2060
2061                 #restrict checks
2062                 myrestrict = None
2063                 try:
2064                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2065                 except portage.exception.InvalidDependString as e:
2066                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
2067                         fails["RESTRICT.syntax"].append(
2068                                 "%s: RESTRICT: %s" % (relative_path, e))
2069                         del e
2070                 if myrestrict:
2071                         myrestrict = set(myrestrict)
2072                         mybadrestrict = myrestrict.difference(valid_restrict)
2073                         if mybadrestrict:
2074                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2075                                 for mybad in mybadrestrict:
2076                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
2077                 #REQUIRED_USE check
2078                 required_use = myaux["REQUIRED_USE"]
2079                 if required_use:
2080                         if not eapi_has_required_use(eapi):
2081                                 stats['EAPI.incompatible'] += 1
2082                                 fails['EAPI.incompatible'].append(
2083                                         relative_path + ": REQUIRED_USE" + \
2084                                         " not supported with EAPI='%s'" % (eapi,))
2085                         try:
2086                                 portage.dep.check_required_use(required_use, (),
2087                                         pkg.iuse.is_valid_flag, eapi=eapi)
2088                         except portage.exception.InvalidDependString as e:
2089                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2090                                 fails["REQUIRED_USE.syntax"].append(
2091                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2092                                 del e
2093
2094                 # Syntax Checks
2095                 relative_path = os.path.join(x, y + ".ebuild")
2096                 full_path = os.path.join(repodir, relative_path)
2097                 if not vcs_preserves_mtime:
2098                         if ebuild_path not in new_ebuilds and \
2099                                 ebuild_path not in modified_ebuilds:
2100                                 pkg.mtime = None
2101                 try:
2102                         # All ebuilds should have utf_8 encoding.
2103                         f = io.open(_unicode_encode(full_path,
2104                                 encoding=_encodings['fs'], errors='strict'),
2105                                 mode='r', encoding=_encodings['repo.content'])
2106                         try:
2107                                 for check_name, e in run_checks(f, pkg):
2108                                         stats[check_name] += 1
2109                                         fails[check_name].append(relative_path + ': %s' % e)
2110                         finally:
2111                                 f.close()
2112                 except UnicodeDecodeError:
2113                         # A file.UTF8 failure will have already been recorded above.
2114                         pass
2115
2116                 if options.force:
2117                         # The dep_check() calls are the most expensive QA test. If --force
2118                         # is enabled, there's no point in wasting time on these since the
2119                         # user is intent on forcing the commit anyway.
2120                         continue
2121
2122                 for keyword,arch,groups in arches:
2123
2124                         if arch not in profiles:
2125                                 # A missing profile will create an error further down
2126                                 # during the KEYWORDS verification.
2127                                 continue
2128
2129                         for prof in profiles[arch]:
2130
2131                                 if prof.status not in ("stable", "dev") or \
2132                                         prof.status == "dev" and not options.include_dev:
2133                                         continue
2134
2135                                 dep_settings = arch_caches.get(prof.sub_path)
2136                                 if dep_settings is None:
2137                                         dep_settings = portage.config(
2138                                                 config_profile_path=prof.abs_path,
2139                                                 config_incrementals=repoman_incrementals,
2140                                                 config_root=config_root,
2141                                                 local_config=False,
2142                                                 _unmatched_removal=options.unmatched_removal,
2143                                                 env=env)
2144                                         dep_settings.categories = repoman_settings.categories
2145                                         if options.without_mask:
2146                                                 dep_settings._mask_manager_obj = \
2147                                                         copy.deepcopy(dep_settings._mask_manager)
2148                                                 dep_settings._mask_manager._pmaskdict.clear()
2149                                         arch_caches[prof.sub_path] = dep_settings
2150
2151                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2152                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2153                                 if xcache is None:
2154                                         portdb.melt()
2155                                         portdb.freeze()
2156                                         xcache = portdb.xcache
2157                                         xcache.update(shared_xmatch_caches)
2158                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2159
2160                                 trees[root]["porttree"].settings = dep_settings
2161                                 portdb.settings = dep_settings
2162                                 portdb.xcache = xcache
2163                                 # for package.use.mask support inside dep_check
2164                                 dep_settings.setcpv(pkg)
2165                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2166                                 # just in case, prevent config.reset() from nuking these.
2167                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2168
2169                                 if not baddepsyntax:
2170                                         ismasked = not ebuild_archs or \
2171                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2172                                         if ismasked:
2173                                                 if not have_pmasked:
2174                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2175                                                                 pkg.cpv, pkg._metadata))
2176                                                 if options.ignore_masked:
2177                                                         continue
2178                                                 #we are testing deps for a masked package; give it some lee-way
2179                                                 suffix="masked"
2180                                                 matchmode = "minimum-all"
2181                                         else:
2182                                                 suffix=""
2183                                                 matchmode = "minimum-visible"
2184
2185                                         if not have_dev_keywords:
2186                                                 have_dev_keywords = \
2187                                                         bool(dev_keywords.intersection(keywords))
2188
2189                                         if prof.status == "dev":
2190                                                 suffix=suffix+"indev"
2191
2192                                         for mytype in Package._dep_keys:
2193
2194                                                 mykey = "dependency.bad" + suffix
2195                                                 myvalue = myaux[mytype]
2196                                                 if not myvalue:
2197                                                         continue
2198
2199                                                 success, atoms = portage.dep_check(myvalue, portdb,
2200                                                         dep_settings, use="all", mode=matchmode,
2201                                                         trees=trees)
2202
2203                                                 if success:
2204                                                         if atoms:
2205
2206                                                                 # Don't bother with dependency.unknown for
2207                                                                 # cases in which *DEPEND.bad is triggered.
2208                                                                 for atom in atoms:
2209                                                                         # dep_check returns all blockers and they
2210                                                                         # aren't counted for *DEPEND.bad, so we
2211                                                                         # ignore them here.
2212                                                                         if not atom.blocker:
2213                                                                                 unknown_pkgs.discard(
2214                                                                                         (mytype, atom.unevaluated_atom))
2215
2216                                                                 if not prof.sub_path:
2217                                                                         # old-style virtuals currently aren't
2218                                                                         # resolvable with empty profile, since
2219                                                                         # 'virtuals' mappings are unavailable
2220                                                                         # (it would be expensive to search
2221                                                                         # for PROVIDE in all ebuilds)
2222                                                                         atoms = [atom for atom in atoms if not \
2223                                                                                 (atom.cp.startswith('virtual/') and \
2224                                                                                 not portdb.cp_list(atom.cp))]
2225
2226                                                                 #we have some unsolvable deps
2227                                                                 #remove ! deps, which always show up as unsatisfiable
2228                                                                 atoms = [str(atom.unevaluated_atom) \
2229                                                                         for atom in atoms if not atom.blocker]
2230
2231                                                                 #if we emptied out our list, continue:
2232                                                                 if not atoms:
2233                                                                         continue
2234                                                                 stats[mykey]=stats[mykey]+1
2235                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2236                                                                         (relative_path, mytype, keyword,
2237                                                                         prof, repr(atoms)))
2238                                                 else:
2239                                                         stats[mykey]=stats[mykey]+1
2240                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2241                                                                 (relative_path, mytype, keyword,
2242                                                                 prof, repr(atoms)))
2243
2244                 if not baddepsyntax and unknown_pkgs:
2245                         type_map = {}
2246                         for mytype, atom in unknown_pkgs:
2247                                 type_map.setdefault(mytype, set()).add(atom)
2248                         for mytype, atoms in type_map.items():
2249                                 stats["dependency.unknown"] += 1
2250                                 fails["dependency.unknown"].append("%s: %s: %s" %
2251                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2252
2253         # check if there are unused local USE-descriptions in metadata.xml
2254         # (unless there are any invalids, to avoid noise)
2255         if allvalid:
2256                 for myflag in muselist.difference(used_useflags):
2257                         stats["metadata.warning"] += 1
2258                         fails["metadata.warning"].append(
2259                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2260                                 (x, myflag))
2261
2262 if options.if_modified == "y" and len(effective_scanlist) < 1:
2263         logging.warn("--if-modified is enabled, but no modified packages were found!")
2264
2265 if options.mode == "manifest":
2266         sys.exit(dofail)
2267
2268 #dofail will be set to 1 if we have failed in at least one non-warning category
2269 dofail=0
2270 #dowarn will be set to 1 if we tripped any warnings
2271 dowarn=0
2272 #dofull will be set if we should print a "repoman full" informational message
2273 dofull = options.mode != 'full'
2274
2275 for x in qacats:
2276         if not stats[x]:
2277                 continue
2278         dowarn = 1
2279         if x not in qawarnings:
2280                 dofail = 1
2281
2282 if dofail or \
2283         (dowarn and not (options.quiet or options.mode == "scan")):
2284         dofull = 0
2285
2286 # Save QA output so that it can be conveniently displayed
2287 # in $EDITOR while the user creates a commit message.
2288 # Otherwise, the user would not be able to see this output
2289 # once the editor has taken over the screen.
2290 qa_output = io.StringIO()
2291 style_file = ConsoleStyleFile(sys.stdout)
2292 if options.mode == 'commit' and \
2293         (not commitmessage or not commitmessage.strip()):
2294         style_file.write_listener = qa_output
2295 console_writer = StyleWriter(file=style_file, maxcol=9999)
2296 console_writer.style_listener = style_file.new_styles
2297
2298 f = formatter.AbstractFormatter(console_writer)
2299
2300 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2301
2302 style_file.flush()
2303 del console_writer, f, style_file
2304 qa_output = qa_output.getvalue()
2305 qa_output = qa_output.splitlines(True)
2306
2307 def grouplist(mylist,seperator="/"):
2308         """(list,seperator="/") -- Takes a list of elements; groups them into
2309         same initial element categories. Returns a dict of {base:[sublist]}
2310         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2311         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2312         mygroups={}
2313         for x in mylist:
2314                 xs=x.split(seperator)
2315                 if xs[0]==".":
2316                         xs=xs[1:]
2317                 if xs[0] not in mygroups:
2318                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2319                 else:
2320                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2321         return mygroups
2322
2323 suggest_ignore_masked = False
2324 suggest_include_dev = False
2325
2326 if have_pmasked and not (options.without_mask or options.ignore_masked):
2327         suggest_ignore_masked = True
2328 if have_dev_keywords and not options.include_dev:
2329         suggest_include_dev = True
2330
2331 if suggest_ignore_masked or suggest_include_dev:
2332         print()
2333         if suggest_ignore_masked:
2334                 print(bold("Note: use --without-mask to check " + \
2335                         "KEYWORDS on dependencies of masked packages"))
2336
2337         if suggest_include_dev:
2338                 print(bold("Note: use --include-dev (-d) to check " + \
2339                         "dependencies for 'dev' profiles"))
2340         print()
2341
2342 if options.mode != 'commit':
2343         if dofull:
2344                 print(bold("Note: type \"repoman full\" for a complete listing."))
2345         if dowarn and not dofail:
2346                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2347         elif not dofail:
2348                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2349         elif dofail:
2350                 print(bad("Please fix these important QA issues first."))
2351                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2352                 sys.exit(1)
2353 else:
2354         if dofail and can_force and options.force and not options.pretend:
2355                 print(green("RepoMan sez:") + \
2356                         " \"You want to commit even with these QA issues?\n" + \
2357                         "              I'll take it this time, but I'm not happy.\"\n")
2358         elif dofail:
2359                 if options.force and not can_force:
2360                         print(bad("The --force option has been disabled due to extraordinary issues."))
2361                 print(bad("Please fix these important QA issues first."))
2362                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2363                 sys.exit(1)
2364
2365         if options.pretend:
2366                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2367
2368         myunadded = []
2369         if vcs == "cvs":
2370                 try:
2371                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2372                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2373                 except SystemExit as e:
2374                         raise  # TODO propagate this
2375                 except:
2376                         err("Error retrieving CVS tree; exiting.")
2377         if vcs == "svn":
2378                 try:
2379                         with repoman_popen("svn status --no-ignore") as f:
2380                                 svnstatus = f.readlines()
2381                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2382                 except SystemExit as e:
2383                         raise  # TODO propagate this
2384                 except:
2385                         err("Error retrieving SVN info; exiting.")
2386         if vcs == "git":
2387                 # get list of files not under version control or missing
2388                 myf = repoman_popen("git ls-files --others")
2389                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2390                 myf.close()
2391         if vcs == "bzr":
2392                 try:
2393                         with repoman_popen("bzr status -S .") as f:
2394                                 bzrstatus = f.readlines()
2395                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2396                 except SystemExit as e:
2397                         raise  # TODO propagate this
2398                 except:
2399                         err("Error retrieving bzr info; exiting.")
2400         if vcs == "hg":
2401                 with repoman_popen("hg status --no-status --unknown .") as f:
2402                         myunadded = f.readlines()
2403                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2404
2405                 # Mercurial doesn't handle manually deleted files as removed from
2406                 # the repository, so the user need to remove them before commit,
2407                 # using "hg remove [FILES]"
2408                 with repoman_popen("hg status --no-status --deleted .") as f:
2409                         mydeleted = f.readlines()
2410                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2411
2412
2413         myautoadd=[]
2414         if myunadded:
2415                 for x in range(len(myunadded)-1,-1,-1):
2416                         xs=myunadded[x].split("/")
2417                         if xs[-1]=="files":
2418                                 print("!!! files dir is not added! Please correct this.")
2419                                 sys.exit(-1)
2420                         elif xs[-1]=="Manifest":
2421                                 # It's a manifest... auto add
2422                                 myautoadd+=[myunadded[x]]
2423                                 del myunadded[x]
2424
2425         if myunadded:
2426                 print(red("!!! The following files are in your local tree but are not added to the master"))
2427                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2428                 for x in myunadded:
2429                         print("   ",x)
2430                 print()
2431                 print()
2432                 sys.exit(1)
2433
2434         if vcs == "hg" and mydeleted:
2435                 print(red("!!! The following files are removed manually from your local tree but are not"))
2436                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2437                 for x in mydeleted:
2438                         print("   ",x)
2439                 print()
2440                 print()
2441                 sys.exit(1)
2442
2443         if vcs == "cvs":
2444                 mycvstree = cvstree.getentries("./", recursive=1)
2445                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2446                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2447                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2448                 bin_blob_pattern = re.compile("^-kb$")
2449                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2450                         recursive=1, basedir="./"))
2451
2452
2453         if vcs == "svn":
2454                 with repoman_popen("svn status") as f:
2455                         svnstatus = f.readlines()
2456                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2457                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2458                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2459
2460                 # Subversion expands keywords specified in svn:keywords properties.
2461                 with repoman_popen("svn propget -R svn:keywords") as f:
2462                         props = f.readlines()
2463                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2464                         for prop in props if " - " in prop)
2465
2466         elif vcs == "git":
2467                 with repoman_popen("git diff-index --name-only "
2468                         "--relative --diff-filter=M HEAD") as f:
2469                         mychanged = f.readlines()
2470                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2471
2472                 with repoman_popen("git diff-index --name-only "
2473                         "--relative --diff-filter=A HEAD") as f:
2474                         mynew = f.readlines()
2475                 mynew = ["./" + elem[:-1] for elem in mynew]
2476
2477                 with repoman_popen("git diff-index --name-only "
2478                         "--relative --diff-filter=D HEAD") as f:
2479                         myremoved = f.readlines()
2480                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2481
2482         if vcs == "bzr":
2483                 with repoman_popen("bzr status -S .") as f:
2484                         bzrstatus = f.readlines()
2485                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2486                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2487                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2488                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2489                 # Bazaar expands nothing.
2490
2491         if vcs == "hg":
2492                 with repoman_popen("hg status --no-status --modified .") as f:
2493                         mychanged = f.readlines()
2494                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2495
2496                 with repoman_popen("hg status --no-status --added .") as f:
2497                         mynew = f.readlines()
2498                 mynew = ["./" + elem.rstrip() for elem in mynew]
2499
2500                 with repoman_popen("hg status --no-status --removed .") as f:
2501                         myremoved = f.readlines()
2502                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2503
2504         if vcs:
2505                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2506                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2507                         print()
2508                         print("(Didn't find any changed files...)")
2509                         print()
2510                         sys.exit(1)
2511
2512         # Manifests need to be regenerated after all other commits, so don't commit
2513         # them now even if they have changed.
2514         mymanifests = set()
2515         myupdates = set()
2516         for f in mychanged + mynew:
2517                 if "Manifest" == os.path.basename(f):
2518                         mymanifests.add(f)
2519                 else:
2520                         myupdates.add(f)
2521         myupdates.difference_update(myremoved)
2522         myupdates = list(myupdates)
2523         mymanifests = list(mymanifests)
2524         myheaders = []
2525         mydirty = []
2526
2527         commitmessage = options.commitmsg
2528         if options.commitmsgfile:
2529                 try:
2530                         f = io.open(_unicode_encode(options.commitmsgfile,
2531                         encoding=_encodings['fs'], errors='strict'),
2532                         mode='r', encoding=_encodings['content'], errors='replace')
2533                         commitmessage = f.read()
2534                         f.close()
2535                         del f
2536                 except (IOError, OSError) as e:
2537                         if e.errno == errno.ENOENT:
2538                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2539                         else:
2540                                 raise
2541                 # We've read the content so the file is no longer needed.
2542                 commitmessagefile = None
2543         if not commitmessage or not commitmessage.strip():
2544                 try:
2545                         editor = os.environ.get("EDITOR")
2546                         if editor and utilities.editor_is_executable(editor):
2547                                 commitmessage = utilities.get_commit_message_with_editor(
2548                                         editor, message=qa_output)
2549                         else:
2550                                 commitmessage = utilities.get_commit_message_with_stdin()
2551                 except KeyboardInterrupt:
2552                         exithandler()
2553                 if not commitmessage or not commitmessage.strip():
2554                         print("* no commit message?  aborting commit.")
2555                         sys.exit(1)
2556         commitmessage = commitmessage.rstrip()
2557         changelog_msg = commitmessage
2558         portage_version = getattr(portage, "VERSION", None)
2559         if portage_version is None:
2560                 sys.stderr.write("Failed to insert portage version in message!\n")
2561                 sys.stderr.flush()
2562                 portage_version = "Unknown"
2563
2564         if vcs == "git":
2565                 # Use new footer only for git (see bug #438364).
2566                 commit_footer = "\n\nPackage-Manager: portage-%s" % portage_version
2567                 if options.force:
2568                         commit_footer += "\nRepoMan-Options: --force"
2569                 if sign_manifests:
2570                         commit_footer += "\nManifest-Sign-Key: %s" % \
2571                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2572         else:
2573                 unameout = platform.system() + " "
2574                 if platform.system() in ["Darwin", "SunOS"]:
2575                         unameout += platform.processor()
2576                 else:
2577                         unameout += platform.machine()
2578                 commit_footer = "\n\n(Portage version: %s/%s/%s" % \
2579                         (portage_version, vcs, unameout)
2580                 if options.force:
2581                         commit_footer += ", RepoMan options: --force"
2582                 if sign_manifests:
2583                         commit_footer += ", signed Manifest commit with key %s" % \
2584                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2585                 else:
2586                         commit_footer += ", unsigned Manifest commit"
2587                 commit_footer += ")"
2588
2589         commitmessage += commit_footer
2590
2591         if options.echangelog in ('y', 'force'):
2592                 logging.info("checking for unmodified ChangeLog files")
2593                 committer_name = utilities.get_committer_name(env=repoman_settings)
2594                 for x in sorted(vcs_files_to_cps(
2595                         chain(myupdates, mymanifests, myremoved))):
2596                         catdir, pkgdir = x.split("/")
2597                         checkdir = repodir + "/" + x
2598                         checkdir_relative = ""
2599                         if repolevel < 3:
2600                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2601                         if repolevel < 2:
2602                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2603                         checkdir_relative = os.path.join(".", checkdir_relative)
2604
2605                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2606                         changelog_modified = changelog_path in modified_changelogs
2607                         if changelog_modified and options.echangelog != 'force':
2608                                 continue
2609
2610                         # get changes for this package
2611                         cdrlen = len(checkdir_relative)
2612                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2613                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2614                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2615
2616                         # Skip ChangeLog generation if only the Manifest was modified,
2617                         # as discussed in bug #398009.
2618                         nontrivial_cl_files = set()
2619                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2620                         nontrivial_cl_files.difference_update(['Manifest'])
2621                         if not nontrivial_cl_files and options.echangelog != 'force':
2622                                 continue
2623
2624                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2625                                 committer_name, changelog_msg,
2626                                 os.path.join(repodir, 'skel.ChangeLog'),
2627                                 catdir, pkgdir,
2628                                 new=clnew, removed=clremoved, changed=clchanged,
2629                                 pretend=options.pretend)
2630                         if new_changelog is None:
2631                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2632                                         level=logging.ERROR, noiselevel=-1)
2633                                 sys.exit(1)
2634
2635                         # if the ChangeLog was just created, add it to vcs
2636                         if new_changelog:
2637                                 myautoadd.append(changelog_path)
2638                                 # myautoadd is appended to myupdates below
2639                         else:
2640                                 myupdates.append(changelog_path)
2641
2642                         if options.ask and not options.pretend:
2643                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2644                                 repoman_settings["O"] = checkdir
2645                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2646
2647         if myautoadd:
2648                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2649                 add_cmd = [vcs, "add"]
2650                 add_cmd += myautoadd
2651                 if options.pretend:
2652                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2653                                 noiselevel=-1)
2654                 else:
2655                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2656                                 # Python 3.1 produces the following TypeError if raw bytes are
2657                                 # passed to subprocess.call():
2658                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2659                                 #     errread, errwrite)
2660                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2661                                 #     raise child_exception
2662                                 # TypeError: expected an object with the buffer interface
2663                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2664                         retcode = subprocess.call(add_cmd)
2665                         if retcode != os.EX_OK:
2666                                 logging.error(
2667                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2668                                 sys.exit(retcode)
2669
2670                 myupdates += myautoadd
2671
2672         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2673
2674         if vcs not in ('cvs', 'svn'):
2675                 # With git, bzr and hg, there's never any keyword expansion, so
2676                 # there's no need to regenerate manifests and all files will be
2677                 # committed in one big commit at the end.
2678                 print()
2679         elif not repo_config.thin_manifest:
2680                 if vcs == 'cvs':
2681                         headerstring = "'\$(Header|Id).*\$'"
2682                 elif vcs == "svn":
2683                         svn_keywords = dict((k.lower(), k) for k in [
2684                                         "Rev",
2685                                         "Revision",
2686                                         "LastChangedRevision",
2687                                         "Date",
2688                                         "LastChangedDate",
2689                                         "Author",
2690                                         "LastChangedBy",
2691                                         "URL",
2692                                         "HeadURL",
2693                                         "Id",
2694                                         "Header",
2695                         ])
2696
2697                 for myfile in myupdates:
2698
2699                         # for CVS, no_expansion contains files that are excluded from expansion
2700                         if vcs == "cvs":
2701                                 if myfile in no_expansion:
2702                                         continue
2703
2704                         # for SVN, expansion contains files that are included in expansion
2705                         elif vcs == "svn":
2706                                 if myfile not in expansion:
2707                                         continue
2708
2709                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2710                                 enabled_keywords = []
2711                                 for k in expansion[myfile]:
2712                                         keyword = svn_keywords.get(k.lower())
2713                                         if keyword is not None:
2714                                                 enabled_keywords.append(keyword)
2715
2716                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2717
2718                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2719                                 portage._shell_quote(myfile))
2720                         if myout[0] == 0:
2721                                 myheaders.append(myfile)
2722
2723                 print("%s have headers that will change." % green(str(len(myheaders))))
2724                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2725
2726         logging.info("myupdates: %s", myupdates)
2727         logging.info("myheaders: %s", myheaders)
2728
2729         if options.ask and userquery('Commit changes?', True) != 'Yes':
2730                 print("* aborting commit.")
2731                 sys.exit(128 + signal.SIGINT)
2732
2733         # Handle the case where committed files have keywords which
2734         # will change and need a priming commit before the Manifest
2735         # can be committed.
2736         if (myupdates or myremoved) and myheaders:
2737                 myfiles = myupdates + myremoved
2738                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2739                 mymsg = os.fdopen(fd, "wb")
2740                 mymsg.write(_unicode_encode(commitmessage))
2741                 mymsg.close()
2742
2743                 print()
2744                 print(green("Using commit message:"))
2745                 print(green("------------------------------------------------------------------------------"))
2746                 print(commitmessage)
2747                 print(green("------------------------------------------------------------------------------"))
2748                 print()
2749
2750                 # Having a leading ./ prefix on file paths can trigger a bug in
2751                 # the cvs server when committing files to multiple directories,
2752                 # so strip the prefix.
2753                 myfiles = [f.lstrip("./") for f in myfiles]
2754
2755                 commit_cmd = [vcs]
2756                 commit_cmd.extend(vcs_global_opts)
2757                 commit_cmd.append("commit")
2758                 commit_cmd.extend(vcs_local_opts)
2759                 commit_cmd.extend(["-F", commitmessagefile])
2760                 commit_cmd.extend(myfiles)
2761
2762                 try:
2763                         if options.pretend:
2764                                 print("(%s)" % (" ".join(commit_cmd),))
2765                         else:
2766                                 retval = spawn(commit_cmd, env=os.environ)
2767                                 if retval != os.EX_OK:
2768                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2769                                                 "error code: %s\n") % (vcs, retval),
2770                                                 level=logging.ERROR, noiselevel=-1)
2771                                         sys.exit(retval)
2772                 finally:
2773                         try:
2774                                 os.unlink(commitmessagefile)
2775                         except OSError:
2776                                 pass
2777
2778         # Setup the GPG commands
2779         def gpgsign(filename):
2780                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2781                 if gpgcmd is None:
2782                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2783                                 " Is make.globals missing?")
2784                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2785                         "PORTAGE_GPG_KEY" not in repoman_settings:
2786                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2787                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2788                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2789                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2790                                         os.path.expanduser("~/.gnupg")
2791                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2792                                         % repoman_settings["PORTAGE_GPG_DIR"])
2793                         else:
2794                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2795                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2796                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2797                                 raise portage.exception.InvalidLocation(
2798                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2799                                         repoman_settings["PORTAGE_GPG_DIR"])
2800                 gpgvars = {"FILE": filename}
2801                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2802                         v = repoman_settings.get(k)
2803                         if v is not None:
2804                                 gpgvars[k] = v
2805                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2806                 if options.pretend:
2807                         print("("+gpgcmd+")")
2808                 else:
2809                         # Encode unicode manually for bug #310789.
2810                         gpgcmd = portage.util.shlex_split(gpgcmd)
2811                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2812                                 # Python 3.1 does not support bytes in Popen args.
2813                                 gpgcmd = [_unicode_encode(arg,
2814                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2815                         rValue = subprocess.call(gpgcmd)
2816                         if rValue == os.EX_OK:
2817                                 os.rename(filename+".asc", filename)
2818                         else:
2819                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2820
2821         def need_signature(filename):
2822                 try:
2823                         with open(_unicode_encode(filename,
2824                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2825                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2826                 except IOError as e:
2827                         if e.errno in (errno.ENOENT, errno.ESTALE):
2828                                 return False
2829                         raise
2830
2831         # When files are removed and re-added, the cvs server will put /Attic/
2832         # inside the $Header path. This code detects the problem and corrects it
2833         # so that the Manifest will generate correctly. See bug #169500.
2834         # Use binary mode in order to avoid potential character encoding issues.
2835         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2836         attic_str = b'/Attic/'
2837         attic_replace = b'/'
2838         for x in myheaders:
2839                 f = open(_unicode_encode(x,
2840                         encoding=_encodings['fs'], errors='strict'),
2841                         mode='rb')
2842                 mylines = f.readlines()
2843                 f.close()
2844                 modified = False
2845                 for i, line in enumerate(mylines):
2846                         if cvs_header_re.match(line) is not None and \
2847                                 attic_str in line:
2848                                 mylines[i] = line.replace(attic_str, attic_replace)
2849                                 modified = True
2850                 if modified:
2851                         portage.util.write_atomic(x, b''.join(mylines),
2852                                 mode='wb')
2853
2854         if repolevel == 1:
2855                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2856                         "doing the entire repository.\"\n")
2857
2858         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2859
2860                 for x in sorted(vcs_files_to_cps(
2861                         chain(myupdates, myremoved, mymanifests))):
2862                         repoman_settings["O"] = os.path.join(repodir, x)
2863                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2864
2865         signed = False
2866         if sign_manifests:
2867                 signed = True
2868                 try:
2869                         for x in sorted(vcs_files_to_cps(
2870                                 chain(myupdates, myremoved, mymanifests))):
2871                                 repoman_settings["O"] = os.path.join(repodir, x)
2872                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2873                                 if not need_signature(manifest_path):
2874                                         continue
2875                                 gpgsign(manifest_path)
2876                 except portage.exception.PortageException as e:
2877                         portage.writemsg("!!! %s\n" % str(e))
2878                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2879                         signed = False
2880
2881         if vcs == 'git':
2882                 # It's not safe to use the git commit -a option since there might
2883                 # be some modified files elsewhere in the working tree that the
2884                 # user doesn't want to commit. Therefore, call git update-index
2885                 # in order to ensure that the index is updated with the latest
2886                 # versions of all new and modified files in the relevant portion
2887                 # of the working tree.
2888                 myfiles = mymanifests + myupdates
2889                 myfiles.sort()
2890                 update_index_cmd = ["git", "update-index"]
2891                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2892                 if options.pretend:
2893                         print("(%s)" % (" ".join(update_index_cmd),))
2894                 else:
2895                         retval = spawn(update_index_cmd, env=os.environ)
2896                         if retval != os.EX_OK:
2897                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2898                                         "error code: %s\n") % (vcs, retval),
2899                                         level=logging.ERROR, noiselevel=-1)
2900                                 sys.exit(retval)
2901
2902         if True:
2903
2904                 myfiles = mymanifests[:]
2905                 # If there are no header (SVN/CVS keywords) changes in
2906                 # the files, this Manifest commit must include the
2907                 # other (yet uncommitted) files.
2908                 if not myheaders:
2909                         myfiles += myupdates
2910                         myfiles += myremoved
2911                 myfiles.sort()
2912
2913                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2914                 mymsg = os.fdopen(fd, "wb")
2915                 mymsg.write(_unicode_encode(commitmessage))
2916                 mymsg.close()
2917
2918                 commit_cmd = []
2919                 if options.pretend and vcs is None:
2920                         # substitute a bogus value for pretend output
2921                         commit_cmd.append("cvs")
2922                 else:
2923                         commit_cmd.append(vcs)
2924                 commit_cmd.extend(vcs_global_opts)
2925                 commit_cmd.append("commit")
2926                 commit_cmd.extend(vcs_local_opts)
2927                 if vcs == "hg":
2928                         commit_cmd.extend(["--logfile", commitmessagefile])
2929                         commit_cmd.extend(myfiles)
2930                 else:
2931                         commit_cmd.extend(["-F", commitmessagefile])
2932                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2933
2934                 try:
2935                         if options.pretend:
2936                                 print("(%s)" % (" ".join(commit_cmd),))
2937                         else:
2938                                 retval = spawn(commit_cmd, env=os.environ)
2939                                 if retval != os.EX_OK:
2940
2941                                         if repo_config.sign_commit and vcs == 'git' and \
2942                                                 not git_supports_gpg_sign():
2943                                                 # Inform user that newer git is needed (bug #403323).
2944                                                 logging.error(
2945                                                         "Git >=1.7.9 is required for signed commits!")
2946
2947                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2948                                                 "error code: %s\n") % (vcs, retval),
2949                                                 level=logging.ERROR, noiselevel=-1)
2950                                         sys.exit(retval)
2951                 finally:
2952                         try:
2953                                 os.unlink(commitmessagefile)
2954                         except OSError:
2955                                 pass
2956
2957         print()
2958         if vcs:
2959                 print("Commit complete.")
2960         else:
2961                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2962         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2963 sys.exit(0)
2964