bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2012 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import io
  17 import logging
  18 import optparse
  19 import re
  20 import signal
  21 import stat
  22 import subprocess
  23 import sys
  24 import tempfile
  25 import textwrap
  26 import time
  27 import platform
  28
  29 try:
  30         from urllib.request import urlopen as urllib_request_urlopen
  31 except ImportError:
  32         from urllib import urlopen as urllib_request_urlopen
  33
  34 from itertools import chain
  35 from stat import S_ISDIR
  36
  37 from os import path as osp
  38 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  39 sys.path.insert(0, pym_path)
  40 import portage
  41 portage._disable_legacy_globals()
  42 portage.dep._internal_warnings = True
  43
  44 try:
  45         import xml.etree.ElementTree
  46         from xml.parsers.expat import ExpatError
  47 except (SystemExit, KeyboardInterrupt):
  48         raise
  49 except (ImportError, SystemError, RuntimeError, Exception):
  50         # broken or missing xml support
  51         # http://bugs.python.org/issue14988
  52         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  53         from portage.output import EOutput
  54         out = EOutput()
  55         for line in msg:
  56                 out.eerror(line)
  57         sys.exit(1)
  58
  59 from portage import os
  60 from portage import _encodings
  61 from portage import _unicode_encode
  62 from repoman.checks import run_checks
  63 from repoman import utilities
  64 from repoman.herdbase import make_herd_base
  65 from _emerge.Package import Package
  66 from _emerge.RootConfig import RootConfig
  67 from _emerge.userquery import userquery
  68 import portage.checksum
  69 import portage.const
  70 from portage import cvstree, normalize_path
  71 from portage import util
  72 from portage.exception import (FileNotFound, MissingParameter,
  73         ParseError, PermissionDenied)
  74 from portage.process import find_binary, spawn
  75 from portage.output import bold, create_color_func, \
  76         green, nocolor, red
  77 from portage.output import ConsoleStyleFile, StyleWriter
  78 from portage.util import writemsg_level
  79 from portage.package.ebuild.digestgen import digestgen
  80 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  81
  82 if sys.hexversion >= 0x3000000:
  83         basestring = str
  84
  85 util.initialize_logger()
  86
  87 # 14 is the length of DESCRIPTION=""
  88 max_desc_len = 100
  89 allowed_filename_chars="a-zA-Z0-9._-+:"
  90 pv_toolong_re = re.compile(r'[0-9]{19,}')
  91 bad = create_color_func("BAD")
  92
  93 # A sane umask is needed for files that portage creates.
  94 os.umask(0o22)
  95 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  96 # behave incrementally.
  97 repoman_incrementals = tuple(x for x in \
  98         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
  99 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 100 repoman_settings = portage.config(config_root=config_root, local_config=False)
 101
 102 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 103         repoman_settings.get('TERM') == 'dumb' or \
 104         not sys.stdout.isatty():
 105         nocolor()
 106
 107 def warn(txt):
 108         print("repoman: " + txt)
 109
 110 def err(txt):
 111         warn(txt)
 112         sys.exit(1)
 113
 114 def exithandler(signum=None, frame=None):
 115         logging.fatal("Interrupted; exiting...")
 116         if signum is None:
 117                 sys.exit(1)
 118         else:
 119                 sys.exit(128 + signum)
 120
 121 signal.signal(signal.SIGINT,exithandler)
 122
 123 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 124         """Repoman needs it's own HelpFormatter for now, because the default ones
 125         murder the help text."""
 126
 127         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 128                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 129
 130         def format_description(self, description):
 131                 return description
 132
 133 class RepomanOptionParser(optparse.OptionParser):
 134         """Add the on_tail function, ruby has it, optionParser should too
 135         """
 136
 137         def __init__(self, *args, **kwargs):
 138                 optparse.OptionParser.__init__(self, *args, **kwargs)
 139                 self.tail = ""
 140
 141         def on_tail(self, description):
 142                 self.tail += description
 143
 144         def format_help(self, formatter=None):
 145                 result = optparse.OptionParser.format_help(self, formatter)
 146                 result += self.tail
 147                 return result
 148
 149
 150 def ParseArgs(argv, qahelp):
 151         """This function uses a customized optionParser to parse command line arguments for repoman
 152         Args:
 153           argv - a sequence of command line arguments
 154                 qahelp - a dict of qa warning to help message
 155         Returns:
 156           (opts, args), just like a call to parser.parse_args()
 157         """
 158
 159         if argv and isinstance(argv[0], bytes):
 160                 argv = [portage._unicode_decode(x) for x in argv]
 161
 162         modes = {
 163                 'commit' : 'Run a scan then commit changes',
 164                 'ci' : 'Run a scan then commit changes',
 165                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 166                 'full' : 'Scan directory tree and print all issues (not a summary)',
 167                 'help' : 'Show this screen',
 168                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 169                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 170                 'scan' : 'Scan directory tree for QA issues'
 171         }
 172
 173         mode_keys = list(modes)
 174         mode_keys.sort()
 175
 176         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 177         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 178         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 179         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 180         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 181
 182         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 183                 help='Request a confirmation before commiting')
 184
 185         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 186                 help='specify a commit message on the command line')
 187
 188         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 189                 help='specify a path to a file that contains a commit message')
 190
 191         parser.add_option('--digest',
 192                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 193                 help='Automatically update Manifest digests for modified files')
 194
 195         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 196                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 197
 198         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 199                 help='do not print unnecessary messages')
 200
 201         parser.add_option(
 202                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 203                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 204                 'regardless of modification if \'force\' is specified)')
 205
 206         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 207                 help='Commit with QA violations')
 208
 209         parser.add_option('--vcs', dest='vcs',
 210                 help='Force using specific VCS instead of autodetection')
 211
 212         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 213                 help='be very verbose in output', default=0)
 214
 215         parser.add_option('-V', '--version', dest='version', action='store_true',
 216                 help='show version info')
 217
 218         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 219                 default=False, help='forces the metadata.xml parse check to be carried out')
 220
 221         parser.add_option(
 222                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 223                 metavar="<y|n>",
 224                 help='only check packages that have uncommitted modifications')
 225
 226         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 227                 default=False, help='ignore arch-specific failures (where arch != host)')
 228
 229         parser.add_option("--ignore-default-opts",
 230                 action="store_true",
 231                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 232
 233         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 234                 default=False, help='ignore masked packages (not allowed with commit mode)')
 235
 236         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 237                 default=False, help='include dev profiles in dependency checks')
 238
 239         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 240                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 241
 242         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 243                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 244
 245         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 246                 help='specify which mode repoman will run in (default=full)')
 247
 248         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 249
 250         for k in mode_keys:
 251                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 252
 253         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 254
 255         sorted_qa = list(qahelp)
 256         sorted_qa.sort()
 257         for k in sorted_qa:
 258                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 259
 260         opts, args = parser.parse_args(argv[1:])
 261
 262         if not opts.ignore_default_opts:
 263                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 264                 if default_opts:
 265                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 266
 267         if opts.mode == 'help':
 268                 parser.print_help(short=False)
 269
 270         for arg in args:
 271                 if arg in modes:
 272                         if not opts.mode:
 273                                 opts.mode = arg
 274                                 break
 275                 else:
 276                         parser.error("invalid mode: %s" % arg)
 277
 278         if not opts.mode:
 279                 opts.mode = 'full'
 280
 281         if opts.mode == 'ci':
 282                 opts.mode = 'commit'  # backwards compat shortcut
 283
 284         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 285                 if opts.ignore_masked:
 286                         parser.error('Commit mode and --ignore-masked are not compatible')
 287                 if opts.without_mask:
 288                         parser.error('Commit mode and --without-mask are not compatible')
 289
 290         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 291         for val in range(opts.verbosity):
 292                 logger = logging.getLogger()
 293                 logger.setLevel(logger.getEffectiveLevel() - 10)
 294
 295         for val in range(opts.quiet):
 296                 logger = logging.getLogger()
 297                 logger.setLevel(logger.getEffectiveLevel() + 10)
 298
 299         return (opts, args)
 300
 301 qahelp={
 302         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 303         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 304         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 305         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 306         "changelog.missing":"Missing ChangeLog files",
 307         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 308         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 309         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 310         "dependency.bad": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)",
 311         "dependency.badmasked": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)",
 312         "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch",
 313         "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch",
 314         "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 315         "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)",
 316         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 317         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 318         "file.size":"Files in the files directory must be under 20 KiB",
 319         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 320         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 321         "file.UTF8":"File is not UTF8 compliant",
 322         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 323         "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it",
 324         "inherit.unused":"Ebuild inherits an eclass but does not use it",
 325         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 326         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 327         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 328         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 329         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 330         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 331         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 332         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 333         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 334         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 335         "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 336         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 337         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 338         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 339         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 340         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 341         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 342         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 343         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 344         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 345         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 346         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 347         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 348         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 349         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 350         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 351         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 352         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 353         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 354         "variable.readonly":"Assigning a readonly variable",
 355         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 356         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 357         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 358         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 359         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 360         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 361         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 362         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 363         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 364         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 365         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 366         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 367         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 368         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 369         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 370         "ebuild.badheader":"This ebuild has a malformed header",
 371         "manifest.bad":"Manifest has missing or incorrect digests",
 372         "metadata.missing":"Missing metadata.xml files",
 373         "metadata.bad":"Bad metadata.xml files",
 374         "metadata.warning":"Warnings in metadata.xml files",
 375         "portage.internal":"The ebuild uses an internal Portage function or variable",
 376         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 377         "virtual.suspect":"Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 378         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 379         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 380 }
 381
 382 qacats = list(qahelp)
 383 qacats.sort()
 384
 385 qawarnings = set((
 386 "changelog.missing",
 387 "changelog.notadded",
 388 "dependency.unknown",
 389 "digest.assumed",
 390 "digest.unused",
 391 "ebuild.notadded",
 392 "ebuild.nesteddie",
 393 "dependency.badmasked",
 394 "dependency.badindev",
 395 "dependency.badmaskedindev",
 396 "dependency.badtilde",
 397 "DESCRIPTION.toolong",
 398 "EAPI.deprecated",
 399 "HOMEPAGE.virtual",
 400 "LICENSE.virtual",
 401 "KEYWORDS.dropped",
 402 "KEYWORDS.stupid",
 403 "KEYWORDS.missing",
 404 "PDEPEND.suspect",
 405 "RDEPEND.implicit",
 406 "RDEPEND.suspect",
 407 "virtual.suspect",
 408 "RESTRICT.invalid",
 409 "ebuild.minorsyn",
 410 "ebuild.badheader",
 411 "ebuild.patches",
 412 "file.size",
 413 "inherit.unused",
 414 "inherit.deprecated",
 415 "java.eclassesnotused",
 416 "wxwidgets.eclassnotused",
 417 "metadata.warning",
 418 "portage.internal",
 419 "usage.obsolete",
 420 "upstream.workaround",
 421 "LIVEVCS.stable",
 422 "LIVEVCS.unmasked",
 423 ))
 424
 425 if portage.const._ENABLE_INHERIT_CHECK:
 426         # This is experimental, so it's non-fatal.
 427         qawarnings.add("inherit.missing")
 428
 429 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 430
 431 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 432 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 433 allvars.update(Package.metadata_keys)
 434 allvars = sorted(allvars)
 435 commitmessage=None
 436 for x in missingvars:
 437         x += ".missing"
 438         if x not in qacats:
 439                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 440                 qacats.append(x)
 441                 qawarnings.add(x)
 442
 443 valid_restrict = frozenset(["binchecks", "bindist",
 444         "fetch", "installsources", "mirror",
 445         "primaryuri", "strip", "test", "userpriv"])
 446
 447 live_eclasses = frozenset([
 448         "bzr",
 449         "cvs",
 450         "darcs",
 451         "git",
 452         "git-2",
 453         "mercurial",
 454         "subversion",
 455         "tla",
 456 ])
 457
 458 suspect_rdepend = frozenset([
 459         "app-arch/cabextract",
 460         "app-arch/rpm2targz",
 461         "app-doc/doxygen",
 462         "dev-lang/nasm",
 463         "dev-lang/swig",
 464         "dev-lang/yasm",
 465         "dev-perl/extutils-pkgconfig",
 466         "dev-util/byacc",
 467         "dev-util/cmake",
 468         "dev-util/ftjam",
 469         "dev-util/gperf",
 470         "dev-util/gtk-doc",
 471         "dev-util/gtk-doc-am",
 472         "dev-util/intltool",
 473         "dev-util/jam",
 474         "dev-util/pkg-config-lite",
 475         "dev-util/pkgconf",
 476         "dev-util/pkgconfig",
 477         "dev-util/pkgconfig-openbsd",
 478         "dev-util/scons",
 479         "dev-util/unifdef",
 480         "dev-util/yacc",
 481         "media-gfx/ebdftopcf",
 482         "sys-apps/help2man",
 483         "sys-devel/autoconf",
 484         "sys-devel/automake",
 485         "sys-devel/bin86",
 486         "sys-devel/bison",
 487         "sys-devel/dev86",
 488         "sys-devel/flex",
 489         "sys-devel/m4",
 490         "sys-devel/pmake",
 491         "virtual/linux-sources",
 492         "virtual/pkgconfig",
 493         "x11-misc/bdftopcf",
 494         "x11-misc/imake",
 495 ])
 496
 497 suspect_virtual = {
 498         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 499         "dev-util/pkgconf":"virtual/pkgconfig",
 500         "dev-util/pkgconfig":"virtual/pkgconfig",
 501         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 502         "dev-libs/libusb":"virtual/libusb",
 503         "dev-libs/libusbx":"virtual/libusb",
 504         "dev-libs/libusb-compat":"virtual/libusb",
 505 }
 506
 507 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 508 # force refetch if the local copy creation time is older than this
 509 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 510
 511 # file.executable
 512 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 513
 514 options, arguments = ParseArgs(sys.argv, qahelp)
 515
 516 if options.version:
 517         print("Portage", portage.VERSION)
 518         sys.exit(0)
 519
 520 # Set this to False when an extraordinary issue (generally
 521 # something other than a QA issue) makes it impossible to
 522 # commit (like if Manifest generation fails).
 523 can_force = True
 524
 525 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 526 if portdir is None:
 527         sys.exit(1)
 528
 529 myreporoot = os.path.basename(portdir_overlay)
 530 myreporoot += mydir[len(portdir_overlay):]
 531
 532 if options.vcs:
 533         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 534                 vcs = options.vcs
 535         else:
 536                 vcs = None
 537 else:
 538         vcses = utilities.FindVCS()
 539         if len(vcses) > 1:
 540                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 541                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 542                 sys.exit(1)
 543         elif vcses:
 544                 vcs = vcses[0]
 545         else:
 546                 vcs = None
 547
 548 if options.if_modified == "y" and vcs is None:
 549         logging.info("Not in a version controlled repository; "
 550                 "disabling --if-modified.")
 551         options.if_modified = "n"
 552
 553 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 554 vcs_preserves_mtime = vcs in ('cvs',)
 555
 556 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 557 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 558 if vcs_global_opts is None:
 559         if vcs in ('cvs', 'svn'):
 560                 vcs_global_opts = "-q"
 561         else:
 562                 vcs_global_opts = ""
 563 vcs_global_opts = vcs_global_opts.split()
 564
 565 if options.mode == 'commit' and not options.pretend and not vcs:
 566         logging.info("Not in a version controlled repository; enabling pretend mode.")
 567         options.pretend = True
 568
 569 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 570 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 571         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 572         portage._shell_quote(portdir_overlay))
 573 # We have to call the config constructor again so
 574 # that config.repositories is initialized correctly.
 575 repoman_settings = portage.config(config_root=config_root, local_config=False,
 576         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 577
 578 root = repoman_settings['EROOT']
 579 trees = {
 580         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 581 }
 582 portdb = trees[root]['porttree'].dbapi
 583
 584 # Constrain dependency resolution to the master(s)
 585 # that are specified in layout.conf.
 586 repodir = os.path.realpath(portdir_overlay)
 587 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 588 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 589 portdir = portdb.porttrees[0]
 590
 591 if repo_config.allow_provide_virtual:
 592         qawarnings.add("virtual.oldstyle")
 593
 594 if repo_config.sign_commit:
 595         if vcs == 'git':
 596                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 597                 # the commit arguments. If key_id is unspecified, then it must be
 598                 # configured by `git config user.signingkey key_id`.
 599                 vcs_local_opts.append("--gpg-sign")
 600
 601 # In order to disable manifest signatures, repos may set
 602 # "sign-manifests = false" in metadata/layout.conf. This
 603 # can be used to prevent merge conflicts like those that
 604 # thin-manifests is designed to prevent.
 605 sign_manifests = "sign" in repoman_settings.features and \
 606         repo_config.sign_manifest
 607
 608 manifest_hashes = repo_config.manifest_hashes
 609 if manifest_hashes is None:
 610         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 611
 612 if options.mode in ("commit", "fix", "manifest"):
 613         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 614                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 615                 "metadata/layout.conf does not contain the '%s' hash which "
 616                 "is required by this portage version. You will have to "
 617                 "upgrade portage if you want to generate valid manifests for "
 618                 "this repository.") % \
 619                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 620                 for line in textwrap.wrap(msg, 70):
 621                         logging.error(line)
 622                 sys.exit(1)
 623
 624         unsupported_hashes = manifest_hashes.difference(
 625                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 626         if unsupported_hashes:
 627                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 628                 "metadata/layout.conf contains one or more hash types '%s' "
 629                 "which are not supported by this portage version. You will "
 630                 "have to upgrade portage if you want to generate valid "
 631                 "manifests for this repository.") % \
 632                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 633                 for line in textwrap.wrap(msg, 70):
 634                         logging.error(line)
 635                 sys.exit(1)
 636
 637 if "commit" == options.mode and \
 638         repo_config.name == "gentoo" and \
 639         "RMD160" in manifest_hashes and \
 640         "RMD160" not in portage.checksum.hashorigin_map:
 641         msg = "Please install " \
 642         "pycrypto or enable python's ssl USE flag in order " \
 643         "to enable RMD160 hash support. See bug #198398 for " \
 644         "more information."
 645         prefix = bad(" * ")
 646         for line in textwrap.wrap(msg, 70):
 647                 print(prefix + line)
 648         sys.exit(1)
 649
 650 if options.echangelog is None and repo_config.update_changelog:
 651         options.echangelog = 'y'
 652
 653 if vcs is None:
 654         options.echangelog = 'n'
 655
 656 # The --echangelog option causes automatic ChangeLog generation,
 657 # which invalidates changelog.ebuildadded and changelog.missing
 658 # checks.
 659 # Note: Some don't use ChangeLogs in distributed SCMs.
 660 # It will be generated on server side from scm log,
 661 # before package moves to the rsync server.
 662 # This is needed because they try to avoid merge collisions.
 663 # Gentoo's Council decided to always use the ChangeLog file.
 664 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 665 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 666
 667 if 'digest' in repoman_settings.features and options.digest != 'n':
 668         options.digest = 'y'
 669
 670 logging.debug("vcs: %s" % (vcs,))
 671 logging.debug("repo config: %s" % (repo_config,))
 672 logging.debug("options: %s" % (options,))
 673
 674 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 675 # profile-specific config constructor calls.
 676 env = os.environ.copy()
 677 env['PORTDIR'] = portdir
 678 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 679
 680 logging.info('Setting paths:')
 681 logging.info('PORTDIR = "' + portdir + '"')
 682 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 683
 684 # It's confusing if these warnings are displayed without the user
 685 # being told which profile they come from, so disable them.
 686 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 687
 688 categories = []
 689 for path in repo_config.eclass_db.porttrees:
 690         categories.extend(portage.util.grabfile(
 691                 os.path.join(path, 'profiles', 'categories')))
 692 repoman_settings.categories = frozenset(
 693         portage.util.stack_lists([categories], incremental=1))
 694 categories = repoman_settings.categories
 695
 696 portdb.settings = repoman_settings
 697 root_config = RootConfig(repoman_settings, trees[root], None)
 698 # We really only need to cache the metadata that's necessary for visibility
 699 # filtering. Anything else can be discarded to reduce memory consumption.
 700 portdb._aux_cache_keys.clear()
 701 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 702
 703 reposplit = myreporoot.split(os.path.sep)
 704 repolevel = len(reposplit)
 705
 706 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 707 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 708 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 709 if options.mode == 'commit' and repolevel not in [1,2,3]:
 710         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 711         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 712         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 713         print(red("***"))
 714         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 715
 716 # Make startdir relative to the canonical repodir, so that we can pass
 717 # it to digestgen and it won't have to be canonicalized again.
 718 if repolevel == 1:
 719         startdir = repodir
 720 else:
 721         startdir = normalize_path(mydir)
 722         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 723
 724 def caterror(mycat):
 725         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 726
 727 def repoman_getstatusoutput(cmd):
 728         """
 729         Implements an interface similar to getstatusoutput(), but with
 730         customized unicode handling (see bug #310789) and without the shell.
 731         """
 732         args = portage.util.shlex_split(cmd)
 733         encoding = _encodings['fs']
 734         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 735                 # Python 3.1 does not support bytes in Popen args.
 736                 args = [_unicode_encode(x,
 737                         encoding=encoding, errors='strict') for x in args]
 738         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 739                 stderr=subprocess.STDOUT)
 740         output = portage._unicode_decode(proc.communicate()[0],
 741                 encoding=encoding, errors='strict')
 742         if output and output[-1] == "\n":
 743                 # getstatusoutput strips one newline
 744                 output = output[:-1]
 745         return (proc.wait(), output)
 746
 747 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 748         """
 749         Implements an interface similar to os.popen(), but with customized
 750         unicode handling (see bug #310789) and without the shell.
 751         """
 752
 753         __slots__ = ('_proc', '_stdout')
 754
 755         def __init__(self, cmd):
 756                 args = portage.util.shlex_split(cmd)
 757                 encoding = _encodings['fs']
 758                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 759                         # Python 3.1 does not support bytes in Popen args.
 760                         args = [_unicode_encode(x,
 761                                 encoding=encoding, errors='strict') for x in args]
 762                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 763                 object.__setattr__(self, '_proc', proc)
 764                 object.__setattr__(self, '_stdout',
 765                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 766
 767         def _get_target(self):
 768                 return object.__getattribute__(self, '_stdout')
 769
 770         __enter__ = _get_target
 771
 772         def __exit__(self, exc_type, exc_value, traceback):
 773                 proc = object.__getattribute__(self, '_proc')
 774                 proc.wait()
 775                 proc.stdout.close()
 776
 777 class ProfileDesc(object):
 778         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 779         def __init__(self, arch, status, sub_path, tree_path):
 780                 self.arch = arch
 781                 self.status = status
 782                 if sub_path:
 783                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 784                 self.sub_path = sub_path
 785                 self.tree_path = tree_path
 786                 if tree_path:
 787                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 788                 else:
 789                         self.abs_path = tree_path
 790
 791         def __str__(self):
 792                 if self.sub_path:
 793                         return self.sub_path
 794                 return 'empty profile'
 795
 796 profile_list = []
 797 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 798
 799 # get lists of valid keywords, licenses, and use
 800 kwlist = set()
 801 liclist = set()
 802 uselist = set()
 803 global_pmasklines = []
 804
 805 for path in portdb.porttrees:
 806         try:
 807                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 808         except OSError:
 809                 pass
 810         kwlist.update(portage.grabfile(os.path.join(path,
 811                 "profiles", "arch.list")))
 812
 813         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 814         for x in use_desc:
 815                 x = x.split()
 816                 if x:
 817                         uselist.add(x[0])
 818
 819         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 820         try:
 821                 expand_list = os.listdir(expand_desc_dir)
 822         except OSError:
 823                 pass
 824         else:
 825                 for fn in expand_list:
 826                         if not fn[-5:] == '.desc':
 827                                 continue
 828                         use_prefix = fn[:-5].lower() + '_'
 829                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 830                                 x = x.split()
 831                                 if x:
 832                                         uselist.add(use_prefix + x[0])
 833
 834         global_pmasklines.append(portage.util.grabfile_package(
 835                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 836
 837         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 838         try:
 839                 desc_file = io.open(_unicode_encode(desc_path,
 840                         encoding=_encodings['fs'], errors='strict'),
 841                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 842         except EnvironmentError:
 843                 pass
 844         else:
 845                 for i, x in enumerate(desc_file):
 846                         if x[0] == "#":
 847                                 continue
 848                         arch = x.split()
 849                         if len(arch) == 0:
 850                                 continue
 851                         if len(arch) != 3:
 852                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 853                                         desc_path + " line %d" % (i+1, ))
 854                         elif arch[0] not in kwlist:
 855                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 856                                         desc_path + " line %d" % (i+1, ))
 857                         elif arch[2] not in valid_profile_types:
 858                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 859                                         desc_path + " line %d" % (i+1, ))
 860                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 861                         if not os.path.isdir(profile_desc.abs_path):
 862                                 logging.error(
 863                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 864                                         arch[2], arch[1], arch[0], desc_path, i+1)
 865                                 continue
 866                         if os.path.exists(
 867                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 868                                 continue
 869                         profile_list.append(profile_desc)
 870                 desc_file.close()
 871
 872 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 873 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 874
 875 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 876 global_pmaskdict = {}
 877 for x in global_pmasklines:
 878         global_pmaskdict.setdefault(x.cp, []).append(x)
 879 del global_pmasklines
 880
 881 def has_global_mask(pkg):
 882         mask_atoms = global_pmaskdict.get(pkg.cp)
 883         if mask_atoms:
 884                 pkg_list = [pkg]
 885                 for x in mask_atoms:
 886                         if portage.dep.match_from_list(x, pkg_list):
 887                                 return x
 888         return None
 889
 890 # Ensure that profile sub_path attributes are unique. Process in reverse order
 891 # so that profiles with duplicate sub_path from overlays will override
 892 # profiles with the same sub_path from parent repos.
 893 profiles = {}
 894 profile_list.reverse()
 895 profile_sub_paths = set()
 896 for prof in profile_list:
 897         if prof.sub_path in profile_sub_paths:
 898                 continue
 899         profile_sub_paths.add(prof.sub_path)
 900         profiles.setdefault(prof.arch, []).append(prof)
 901
 902 # Use an empty profile for checking dependencies of
 903 # packages that have empty KEYWORDS.
 904 prof = ProfileDesc('**', 'stable', '', '')
 905 profiles.setdefault(prof.arch, []).append(prof)
 906
 907 for x in repoman_settings.archlist():
 908         if x[0] == "~":
 909                 continue
 910         if x not in profiles:
 911                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 912                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 913                 print(red("up with the "+x+" team."))
 914                 print()
 915
 916 if not liclist:
 917         logging.fatal("Couldn't find licenses?")
 918         sys.exit(1)
 919
 920 if not kwlist:
 921         logging.fatal("Couldn't read KEYWORDS from arch.list")
 922         sys.exit(1)
 923
 924 if not uselist:
 925         logging.fatal("Couldn't find use.desc?")
 926         sys.exit(1)
 927
 928 scanlist=[]
 929 if repolevel==2:
 930         #we are inside a category directory
 931         catdir=reposplit[-1]
 932         if catdir not in categories:
 933                 caterror(catdir)
 934         mydirlist=os.listdir(startdir)
 935         for x in mydirlist:
 936                 if x == "CVS" or x.startswith("."):
 937                         continue
 938                 if os.path.isdir(startdir+"/"+x):
 939                         scanlist.append(catdir+"/"+x)
 940         repo_subdir = catdir + os.sep
 941 elif repolevel==1:
 942         for x in categories:
 943                 if not os.path.isdir(startdir+"/"+x):
 944                         continue
 945                 for y in os.listdir(startdir+"/"+x):
 946                         if y == "CVS" or y.startswith("."):
 947                                 continue
 948                         if os.path.isdir(startdir+"/"+x+"/"+y):
 949                                 scanlist.append(x+"/"+y)
 950         repo_subdir = ""
 951 elif repolevel==3:
 952         catdir = reposplit[-2]
 953         if catdir not in categories:
 954                 caterror(catdir)
 955         scanlist.append(catdir+"/"+reposplit[-1])
 956         repo_subdir = scanlist[-1] + os.sep
 957 else:
 958         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 959                 ' from the current working directory'
 960         logging.critical(msg)
 961         sys.exit(1)
 962
 963 repo_subdir_len = len(repo_subdir)
 964 scanlist.sort()
 965
 966 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 967
 968 def vcs_files_to_cps(vcs_file_iter):
 969         """
 970         Iterate over the given modified file paths returned from the vcs,
 971         and return a frozenset containing category/pn strings for each
 972         modified package.
 973         """
 974
 975         modified_cps = []
 976
 977         if repolevel == 3:
 978                 if reposplit[-2] in categories and \
 979                         next(vcs_file_iter, None) is not None:
 980                         modified_cps.append("/".join(reposplit[-2:]))
 981
 982         elif repolevel == 2:
 983                 category = reposplit[-1]
 984                 if category in categories:
 985                         for filename in vcs_file_iter:
 986                                 f_split = filename.split(os.sep)
 987                                 # ['.', pn,...]
 988                                 if len(f_split) > 2:
 989                                         modified_cps.append(category + "/" + f_split[1])
 990
 991         else:
 992                 # repolevel == 1
 993                 for filename in vcs_file_iter:
 994                         f_split = filename.split(os.sep)
 995                         # ['.', category, pn,...]
 996                         if len(f_split) > 3 and f_split[1] in categories:
 997                                 modified_cps.append("/".join(f_split[1:3]))
 998
 999         return frozenset(modified_cps)
1000
1001 def git_supports_gpg_sign():
1002         status, cmd_output = \
1003                 repoman_getstatusoutput("git --version")
1004         cmd_output = cmd_output.split()
1005         if cmd_output:
1006                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1007                 if version is not None:
1008                         version = [int(x) for x in version.groups()[1:]]
1009                         if version[0] > 1 or \
1010                                 (version[0] == 1 and version[1] > 7) or \
1011                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1012                                 return True
1013         return False
1014
1015 def dev_keywords(profiles):
1016         """
1017         Create a set of KEYWORDS values that exist in 'dev'
1018         profiles. These are used
1019         to trigger a message notifying the user when they might
1020         want to add the --include-dev option.
1021         """
1022         type_arch_map = {}
1023         for arch, arch_profiles in profiles.items():
1024                 for prof in arch_profiles:
1025                         arch_set = type_arch_map.get(prof.status)
1026                         if arch_set is None:
1027                                 arch_set = set()
1028                                 type_arch_map[prof.status] = arch_set
1029                         arch_set.add(arch)
1030
1031         dev_keywords = type_arch_map.get('dev', set())
1032         dev_keywords.update(['~' + arch for arch in dev_keywords])
1033         return frozenset(dev_keywords)
1034
1035 dev_keywords = dev_keywords(profiles)
1036
1037 stats={}
1038 fails={}
1039
1040 for x in qacats:
1041         stats[x]=0
1042         fails[x]=[]
1043
1044 xmllint_capable = False
1045 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1046
1047 def parsedate(s):
1048         """Parse a RFC 822 date and time string.
1049         This is required for python3 compatibility, since the
1050         rfc822.parsedate() function is not available."""
1051
1052         s_split = []
1053         for x in s.upper().split():
1054                 for y in x.split(','):
1055                         if y:
1056                                 s_split.append(y)
1057
1058         if len(s_split) != 6:
1059                 return None
1060
1061         # %a, %d %b %Y %H:%M:%S %Z
1062         a, d, b, Y, H_M_S, Z = s_split
1063
1064         # Convert month to integer, since strptime %w is locale-dependent.
1065         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1066                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1067         m = month_map.get(b)
1068         if m is None:
1069                 return None
1070         m = str(m).rjust(2, '0')
1071
1072         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1073
1074 def fetch_metadata_dtd():
1075         """
1076         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1077         metadata_dtd_ctime_interval.
1078         @rtype: bool
1079         @return: True if successful, otherwise False
1080         """
1081
1082         must_fetch = True
1083         metadata_dtd_st = None
1084         current_time = int(time.time())
1085         try:
1086                 metadata_dtd_st = os.stat(metadata_dtd)
1087         except EnvironmentError as e:
1088                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1089                         raise
1090                 del e
1091         else:
1092                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1093                 if abs(current_time - metadata_dtd_st.st_ctime) \
1094                         < metadata_dtd_ctime_interval:
1095                         must_fetch = False
1096
1097         if must_fetch:
1098                 print()
1099                 print(green("***") + " the local copy of metadata.dtd " + \
1100                         "needs to be refetched, doing that now")
1101                 print()
1102                 try:
1103                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1104                         msg_info = url_f.info()
1105                         last_modified = msg_info.get('last-modified')
1106                         if last_modified is not None:
1107                                 last_modified = parsedate(last_modified)
1108                                 if last_modified is not None:
1109                                         last_modified = calendar.timegm(last_modified)
1110
1111                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1112                         try:
1113                                 local_f = open(metadata_dtd_tmp, mode='wb')
1114                                 local_f.write(url_f.read())
1115                                 local_f.close()
1116                                 if last_modified is not None:
1117                                         try:
1118                                                 os.utime(metadata_dtd_tmp,
1119                                                         (int(last_modified), int(last_modified)))
1120                                         except OSError:
1121                                                 # This fails on some odd non-unix-like filesystems.
1122                                                 # We don't really need the mtime to be preserved
1123                                                 # anyway here (currently we use ctime to trigger
1124                                                 # fetch), so just ignore it.
1125                                                 pass
1126                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1127                         finally:
1128                                 try:
1129                                         os.unlink(metadata_dtd_tmp)
1130                                 except OSError:
1131                                         pass
1132
1133                         url_f.close()
1134
1135                 except EnvironmentError as e:
1136                         print()
1137                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1138                         print(red("!!!")+" exception '%s' though." % (e,))
1139                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1140                         return False
1141
1142         return True
1143
1144 if options.mode == "manifest":
1145         pass
1146 elif not find_binary('xmllint'):
1147         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1148         if options.xml_parse or repolevel==3:
1149                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1150                 sys.exit(1)
1151 else:
1152         if not fetch_metadata_dtd():
1153                 sys.exit(1)
1154         #this can be problematic if xmllint changes their output
1155         xmllint_capable=True
1156
1157 if options.mode == 'commit' and vcs:
1158         utilities.detect_vcs_conflicts(options, vcs)
1159
1160 if options.mode == "manifest":
1161         pass
1162 elif options.pretend:
1163         print(green("\nRepoMan does a once-over of the neighborhood..."))
1164 else:
1165         print(green("\nRepoMan scours the neighborhood..."))
1166
1167 new_ebuilds = set()
1168 modified_ebuilds = set()
1169 modified_changelogs = set()
1170 mychanged = []
1171 mynew = []
1172 myremoved = []
1173
1174 if vcs == "cvs":
1175         mycvstree = cvstree.getentries("./", recursive=1)
1176         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1177         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1178         if options.if_modified == "y":
1179                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1180
1181 elif vcs == "svn":
1182         with repoman_popen("svn status") as f:
1183                 svnstatus = f.readlines()
1184         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1185         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1186         if options.if_modified == "y":
1187                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1188
1189 elif vcs == "git":
1190         with repoman_popen("git diff-index --name-only "
1191                 "--relative --diff-filter=M HEAD") as f:
1192                 mychanged = f.readlines()
1193         mychanged = ["./" + elem[:-1] for elem in mychanged]
1194
1195         with repoman_popen("git diff-index --name-only "
1196                 "--relative --diff-filter=A HEAD") as f:
1197                 mynew = f.readlines()
1198         mynew = ["./" + elem[:-1] for elem in mynew]
1199         if options.if_modified == "y":
1200                 with repoman_popen("git diff-index --name-only "
1201                         "--relative --diff-filter=D HEAD") as f:
1202                         myremoved = f.readlines()
1203                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1204
1205 elif vcs == "bzr":
1206         with repoman_popen("bzr status -S .") as f:
1207                 bzrstatus = f.readlines()
1208         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1209         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1210         if options.if_modified == "y":
1211                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1212
1213 elif vcs == "hg":
1214         with repoman_popen("hg status --no-status --modified .") as f:
1215                 mychanged = f.readlines()
1216         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1217         mynew = repoman_popen("hg status --no-status --added .").readlines()
1218         mynew = ["./" + elem.rstrip() for elem in mynew]
1219         if options.if_modified == "y":
1220                 with repoman_popen("hg status --no-status --removed .") as f:
1221                         myremoved = f.readlines()
1222                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1223
1224 if vcs:
1225         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1226         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1227         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1228                 if os.path.basename(x) == "ChangeLog")
1229
1230 def vcs_new_changed(relative_path):
1231         for x in chain(mychanged, mynew):
1232                 if x == relative_path:
1233                         return True
1234         return False
1235
1236 have_pmasked = False
1237 have_dev_keywords = False
1238 dofail = 0
1239
1240 # NOTE: match-all caches are not shared due to potential
1241 # differences between profiles in _get_implicit_iuse.
1242 arch_caches={}
1243 arch_xmatch_caches = {}
1244 shared_xmatch_caches = {"cp-list":{}}
1245
1246 # Disable the "ebuild.notadded" check when not in commit mode and
1247 # running `svn status` in every package dir will be too expensive.
1248
1249 check_ebuild_notadded = not \
1250         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1251
1252 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1253 thirdpartymirrors = []
1254 for v in repoman_settings.thirdpartymirrors().values():
1255         for v in v:
1256                 if not v.endswith("/"):
1257                         v += "/"
1258                 thirdpartymirrors.append(v)
1259
1260 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1261         """
1262         Implements doctype() as required to avoid deprecation warnings with
1263         >=python-2.7.
1264         """
1265         def doctype(self, name, pubid, system):
1266                 pass
1267
1268 try:
1269         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1270 except (EnvironmentError, ParseError, PermissionDenied) as e:
1271         err(str(e))
1272 except FileNotFound:
1273         # TODO: Download as we do for metadata.dtd, but add a way to
1274         # disable for non-gentoo repoman users who may not have herds.
1275         herd_base = None
1276
1277 effective_scanlist = scanlist
1278 if options.if_modified == "y":
1279         effective_scanlist = sorted(vcs_files_to_cps(
1280                 chain(mychanged, mynew, myremoved)))
1281
1282 for x in effective_scanlist:
1283         #ebuilds and digests added to cvs respectively.
1284         logging.info("checking package %s" % x)
1285         # save memory by discarding xmatch caches from previous package(s)
1286         arch_xmatch_caches.clear()
1287         eadded=[]
1288         catdir,pkgdir=x.split("/")
1289         checkdir=repodir+"/"+x
1290         checkdir_relative = ""
1291         if repolevel < 3:
1292                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1293         if repolevel < 2:
1294                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1295         checkdir_relative = os.path.join(".", checkdir_relative)
1296         generated_manifest = False
1297
1298         if options.mode == "manifest" or \
1299           (options.mode != 'manifest-check' and options.digest == 'y') or \
1300           options.mode in ('commit', 'fix') and not options.pretend:
1301                 auto_assumed = set()
1302                 fetchlist_dict = portage.FetchlistDict(checkdir,
1303                         repoman_settings, portdb)
1304                 if options.mode == 'manifest' and options.force:
1305                         portage._doebuild_manifest_exempt_depend += 1
1306                         try:
1307                                 distdir = repoman_settings['DISTDIR']
1308                                 mf = repoman_settings.repositories.get_repo_for_location(
1309                                         os.path.dirname(os.path.dirname(checkdir)))
1310                                 mf = mf.load_manifest(checkdir, distdir,
1311                                         fetchlist_dict=fetchlist_dict)
1312                                 mf.create(requiredDistfiles=None,
1313                                         assumeDistHashesAlways=True)
1314                                 for distfiles in fetchlist_dict.values():
1315                                         for distfile in distfiles:
1316                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1317                                                         mf.fhashdict['DIST'].pop(distfile, None)
1318                                                 else:
1319                                                         auto_assumed.add(distfile)
1320                                 mf.write()
1321                         finally:
1322                                 portage._doebuild_manifest_exempt_depend -= 1
1323
1324                 repoman_settings["O"] = checkdir
1325                 try:
1326                         generated_manifest = digestgen(
1327                                 mysettings=repoman_settings, myportdb=portdb)
1328                 except portage.exception.PermissionDenied as e:
1329                         generated_manifest = False
1330                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1331                                 level=logging.ERROR, noiselevel=-1)
1332
1333                 if not generated_manifest:
1334                         print("Unable to generate manifest.")
1335                         dofail = 1
1336
1337                 if options.mode == "manifest":
1338                         if not dofail and options.force and auto_assumed and \
1339                                 'assume-digests' in repoman_settings.features:
1340                                 # Show which digests were assumed despite the --force option
1341                                 # being given. This output will already have been shown by
1342                                 # digestgen() if assume-digests is not enabled, so only show
1343                                 # it here if assume-digests is enabled.
1344                                 pkgs = list(fetchlist_dict)
1345                                 pkgs.sort()
1346                                 portage.writemsg_stdout("  digest.assumed" + \
1347                                         portage.output.colorize("WARN",
1348                                         str(len(auto_assumed)).rjust(18)) + "\n")
1349                                 for cpv in pkgs:
1350                                         fetchmap = fetchlist_dict[cpv]
1351                                         pf = portage.catsplit(cpv)[1]
1352                                         for distfile in sorted(fetchmap):
1353                                                 if distfile in auto_assumed:
1354                                                         portage.writemsg_stdout(
1355                                                                 "   %s::%s\n" % (pf, distfile))
1356                         continue
1357                 elif dofail:
1358                         sys.exit(1)
1359
1360         if not generated_manifest:
1361                 repoman_settings['O'] = checkdir
1362                 repoman_settings['PORTAGE_QUIET'] = '1'
1363                 if not portage.digestcheck([], repoman_settings, strict=1):
1364                         stats["manifest.bad"] += 1
1365                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1366                 repoman_settings.pop('PORTAGE_QUIET', None)
1367
1368         if options.mode == 'manifest-check':
1369                 continue
1370
1371         checkdirlist=os.listdir(checkdir)
1372         ebuildlist=[]
1373         pkgs = {}
1374         allvalid = True
1375         for y in checkdirlist:
1376                 if (y in no_exec or y.endswith(".ebuild")) and \
1377                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1378                                 stats["file.executable"] += 1
1379                                 fails["file.executable"].append(os.path.join(checkdir, y))
1380                 if y.endswith(".ebuild"):
1381                         pf = y[:-7]
1382                         ebuildlist.append(pf)
1383                         cpv = "%s/%s" % (catdir, pf)
1384                         try:
1385                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1386                         except KeyError:
1387                                 allvalid = False
1388                                 stats["ebuild.syntax"] += 1
1389                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1390                                 continue
1391                         except IOError:
1392                                 allvalid = False
1393                                 stats["ebuild.output"] += 1
1394                                 fails["ebuild.output"].append(os.path.join(x, y))
1395                                 continue
1396                         if not portage.eapi_is_supported(myaux["EAPI"]):
1397                                 allvalid = False
1398                                 stats["EAPI.unsupported"] += 1
1399                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1400                                 continue
1401                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1402                                 root_config=root_config, type_name="ebuild")
1403
1404         slot_keywords = {}
1405
1406         if len(pkgs) != len(ebuildlist):
1407                 # If we can't access all the metadata then it's totally unsafe to
1408                 # commit since there's no way to generate a correct Manifest.
1409                 # Do not try to do any more QA checks on this package since missing
1410                 # metadata leads to false positives for several checks, and false
1411                 # positives confuse users.
1412                 can_force = False
1413                 continue
1414
1415         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1416         ebuildlist = sorted(pkgs.values())
1417         ebuildlist = [pkg.pf for pkg in ebuildlist]
1418
1419         for y in checkdirlist:
1420                 index = repo_config.find_invalid_path_char(y)
1421                 if index != -1:
1422                         y_relative = os.path.join(checkdir_relative, y)
1423                         if vcs is not None and not vcs_new_changed(y_relative):
1424                                 # If the file isn't in the VCS new or changed set, then
1425                                 # assume that it's an irrelevant temporary file (Manifest
1426                                 # entries are not generated for file names containing
1427                                 # prohibited characters). See bug #406877.
1428                                 index = -1
1429                 if index != -1:
1430                         stats["file.name"] += 1
1431                         fails["file.name"].append("%s/%s: char '%s'" % \
1432                                 (checkdir, y, y[index]))
1433
1434                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1435                         continue
1436                 f = None
1437                 try:
1438                         line = 1
1439                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1440                                 encoding=_encodings['fs'], errors='strict'),
1441                                 mode='r', encoding=_encodings['repo.content'])
1442                         for l in f:
1443                                 line +=1
1444                 except UnicodeDecodeError as ue:
1445                         stats["file.UTF8"] += 1
1446                         s = ue.object[:ue.start]
1447                         l2 = s.count("\n")
1448                         line += l2
1449                         if l2 != 0:
1450                                 s = s[s.rfind("\n") + 1:]
1451                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1452                 finally:
1453                         if f is not None:
1454                                 f.close()
1455
1456         if vcs in ("git", "hg") and check_ebuild_notadded:
1457                 if vcs == "git":
1458                         myf = repoman_popen("git ls-files --others %s" % \
1459                                 (portage._shell_quote(checkdir_relative),))
1460                 if vcs == "hg":
1461                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1462                                 (portage._shell_quote(checkdir_relative),))
1463                 for l in myf:
1464                         if l[:-1][-7:] == ".ebuild":
1465                                 stats["ebuild.notadded"] += 1
1466                                 fails["ebuild.notadded"].append(
1467                                         os.path.join(x, os.path.basename(l[:-1])))
1468                 myf.close()
1469
1470         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1471                 try:
1472                         if vcs == "cvs":
1473                                 myf=open(checkdir+"/CVS/Entries","r")
1474                         if vcs == "svn":
1475                                 myf = repoman_popen("svn status --depth=files --verbose " +
1476                                         portage._shell_quote(checkdir))
1477                         if vcs == "bzr":
1478                                 myf = repoman_popen("bzr ls -v --kind=file " +
1479                                         portage._shell_quote(checkdir))
1480                         myl = myf.readlines()
1481                         myf.close()
1482                         for l in myl:
1483                                 if vcs == "cvs":
1484                                         if l[0]!="/":
1485                                                 continue
1486                                         splitl=l[1:].split("/")
1487                                         if not len(splitl):
1488                                                 continue
1489                                         if splitl[0][-7:]==".ebuild":
1490                                                 eadded.append(splitl[0][:-7])
1491                                 if vcs == "svn":
1492                                         if l[:1] == "?":
1493                                                 continue
1494                                         if l[:7] == '      >':
1495                                                 # tree conflict, new in subversion 1.6
1496                                                 continue
1497                                         l = l.split()[-1]
1498                                         if l[-7:] == ".ebuild":
1499                                                 eadded.append(os.path.basename(l[:-7]))
1500                                 if vcs == "bzr":
1501                                         if l[1:2] == "?":
1502                                                 continue
1503                                         l = l.split()[-1]
1504                                         if l[-7:] == ".ebuild":
1505                                                 eadded.append(os.path.basename(l[:-7]))
1506                         if vcs == "svn":
1507                                 myf = repoman_popen("svn status " +
1508                                         portage._shell_quote(checkdir))
1509                                 myl=myf.readlines()
1510                                 myf.close()
1511                                 for l in myl:
1512                                         if l[0] == "A":
1513                                                 l = l.rstrip().split(' ')[-1]
1514                                                 if l[-7:] == ".ebuild":
1515                                                         eadded.append(os.path.basename(l[:-7]))
1516                 except IOError:
1517                         if vcs == "cvs":
1518                                 stats["CVS/Entries.IO_error"] += 1
1519                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1520                         else:
1521                                 raise
1522                         continue
1523
1524         mf = repoman_settings.repositories.get_repo_for_location(
1525                 os.path.dirname(os.path.dirname(checkdir)))
1526         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1527         mydigests=mf.getTypeDigests("DIST")
1528
1529         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1530         myfiles_all = []
1531         src_uri_error = False
1532         for mykey in fetchlist_dict:
1533                 try:
1534                         myfiles_all.extend(fetchlist_dict[mykey])
1535                 except portage.exception.InvalidDependString as e:
1536                         src_uri_error = True
1537                         try:
1538                                 portdb.aux_get(mykey, ["SRC_URI"])
1539                         except KeyError:
1540                                 # This will be reported as an "ebuild.syntax" error.
1541                                 pass
1542                         else:
1543                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1544                                 fails["SRC_URI.syntax"].append(
1545                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1546         del fetchlist_dict
1547         if not src_uri_error:
1548                 # This test can produce false positives if SRC_URI could not
1549                 # be parsed for one or more ebuilds. There's no point in
1550                 # producing a false error here since the root cause will
1551                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1552                 # or "ebuild.sytax".
1553                 myfiles_all = set(myfiles_all)
1554                 for entry in mydigests:
1555                         if entry not in myfiles_all:
1556                                 stats["digest.unused"] += 1
1557                                 fails["digest.unused"].append(checkdir+"::"+entry)
1558                 for entry in myfiles_all:
1559                         if entry not in mydigests:
1560                                 stats["digest.missing"] += 1
1561                                 fails["digest.missing"].append(checkdir+"::"+entry)
1562         del myfiles_all
1563
1564         if os.path.exists(checkdir+"/files"):
1565                 filesdirlist=os.listdir(checkdir+"/files")
1566
1567                 # recurse through files directory
1568                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1569                 while filesdirlist:
1570                         y = filesdirlist.pop(0)
1571                         relative_path = os.path.join(x, "files", y)
1572                         full_path = os.path.join(repodir, relative_path)
1573                         try:
1574                                 mystat = os.stat(full_path)
1575                         except OSError as oe:
1576                                 if oe.errno == 2:
1577                                         # don't worry about it.  it likely was removed via fix above.
1578                                         continue
1579                                 else:
1580                                         raise oe
1581                         if S_ISDIR(mystat.st_mode):
1582                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1583                                 if y == "CVS" or y == ".svn":
1584                                         continue
1585                                 for z in os.listdir(checkdir+"/files/"+y):
1586                                         if z == "CVS" or z == ".svn":
1587                                                 continue
1588                                         filesdirlist.append(y+"/"+z)
1589                         # Current policy is no files over 20 KiB, these are the checks. File size between
1590                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1591                         elif mystat.st_size > 61440:
1592                                 stats["file.size.fatal"] += 1
1593                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1594                         elif mystat.st_size > 20480:
1595                                 stats["file.size"] += 1
1596                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1597
1598                         index = repo_config.find_invalid_path_char(y)
1599                         if index != -1:
1600                                 y_relative = os.path.join(checkdir_relative, "files", y)
1601                                 if vcs is not None and not vcs_new_changed(y_relative):
1602                                         # If the file isn't in the VCS new or changed set, then
1603                                         # assume that it's an irrelevant temporary file (Manifest
1604                                         # entries are not generated for file names containing
1605                                         # prohibited characters). See bug #406877.
1606                                         index = -1
1607                         if index != -1:
1608                                 stats["file.name"] += 1
1609                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1610                                         (checkdir, y, y[index]))
1611         del mydigests
1612
1613         if check_changelog and "ChangeLog" not in checkdirlist:
1614                 stats["changelog.missing"]+=1
1615                 fails["changelog.missing"].append(x+"/ChangeLog")
1616
1617         musedict = {}
1618         #metadata.xml file check
1619         if "metadata.xml" not in checkdirlist:
1620                 stats["metadata.missing"]+=1
1621                 fails["metadata.missing"].append(x+"/metadata.xml")
1622         #metadata.xml parse check
1623         else:
1624                 metadata_bad = False
1625
1626                 # read metadata.xml into memory
1627                 try:
1628                         _metadata_xml = xml.etree.ElementTree.parse(
1629                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1630                                 encoding=_encodings['fs'], errors='strict'),
1631                                 parser=xml.etree.ElementTree.XMLParser(
1632                                         target=_MetadataTreeBuilder()))
1633                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1634                         metadata_bad = True
1635                         stats["metadata.bad"] += 1
1636                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1637                         del e
1638                 else:
1639                         # load USE flags from metadata.xml
1640                         try:
1641                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1642                         except portage.exception.ParseError as e:
1643                                 metadata_bad = True
1644                                 stats["metadata.bad"] += 1
1645                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1646
1647                         # Run other metadata.xml checkers
1648                         try:
1649                                 utilities.check_metadata(_metadata_xml, herd_base)
1650                         except (utilities.UnknownHerdsError, ) as e:
1651                                 metadata_bad = True
1652                                 stats["metadata.bad"] += 1
1653                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1654                                 del e
1655
1656                 #Only carry out if in package directory or check forced
1657                 if xmllint_capable and not metadata_bad:
1658                         # xmlint can produce garbage output even on success, so only dump
1659                         # the ouput when it fails.
1660                         st, out = repoman_getstatusoutput(
1661                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1662                                  (portage._shell_quote(metadata_dtd),
1663                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1664                         if st != os.EX_OK:
1665                                 print(red("!!!") + " metadata.xml is invalid:")
1666                                 for z in out.splitlines():
1667                                         print(red("!!! ")+z)
1668                                 stats["metadata.bad"]+=1
1669                                 fails["metadata.bad"].append(x+"/metadata.xml")
1670
1671                 del metadata_bad
1672         muselist = frozenset(musedict)
1673
1674         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1675         changelog_modified = changelog_path in modified_changelogs
1676
1677         # detect unused local USE-descriptions
1678         used_useflags = set()
1679
1680         for y in ebuildlist:
1681                 relative_path = os.path.join(x, y + ".ebuild")
1682                 full_path = os.path.join(repodir, relative_path)
1683                 ebuild_path = y + ".ebuild"
1684                 if repolevel < 3:
1685                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1686                 if repolevel < 2:
1687                         ebuild_path = os.path.join(catdir, ebuild_path)
1688                 ebuild_path = os.path.join(".", ebuild_path)
1689                 if check_changelog and not changelog_modified \
1690                         and ebuild_path in new_ebuilds:
1691                         stats['changelog.ebuildadded'] += 1
1692                         fails['changelog.ebuildadded'].append(relative_path)
1693
1694                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1695                         #ebuild not added to vcs
1696                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1697                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1698                 myesplit=portage.pkgsplit(y)
1699                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1700                             or pv_toolong_re.search(myesplit[1]) \
1701                             or pv_toolong_re.search(myesplit[2]):
1702                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1703                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1704                         continue
1705                 elif myesplit[0]!=pkgdir:
1706                         print(pkgdir,myesplit[0])
1707                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1708                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1709                         continue
1710
1711                 pkg = pkgs[y]
1712
1713                 if pkg.invalid:
1714                         allvalid = False
1715                         for k, msgs in pkg.invalid.items():
1716                                 for msg in msgs:
1717                                         stats[k] = stats[k] + 1
1718                                         fails[k].append("%s: %s" % (relative_path, msg))
1719                         continue
1720
1721                 myaux = pkg.metadata
1722                 eapi = myaux["EAPI"]
1723                 inherited = pkg.inherited
1724                 live_ebuild = live_eclasses.intersection(inherited)
1725
1726                 for k, v in myaux.items():
1727                         if not isinstance(v, basestring):
1728                                 continue
1729                         m = non_ascii_re.search(v)
1730                         if m is not None:
1731                                 stats["variable.invalidchar"] += 1
1732                                 fails["variable.invalidchar"].append(
1733                                         ("%s: %s variable contains non-ASCII " + \
1734                                         "character at position %s") % \
1735                                         (relative_path, k, m.start() + 1))
1736
1737                 if not src_uri_error:
1738                         # Check that URIs don't reference a server from thirdpartymirrors.
1739                         for uri in portage.dep.use_reduce( \
1740                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1741                                 contains_mirror = False
1742                                 for mirror in thirdpartymirrors:
1743                                         if uri.startswith(mirror):
1744                                                 contains_mirror = True
1745                                                 break
1746                                 if not contains_mirror:
1747                                         continue
1748
1749                                 stats["SRC_URI.mirror"] += 1
1750                                 fails["SRC_URI.mirror"].append(
1751                                         "%s: '%s' found in thirdpartymirrors" % \
1752                                         (relative_path, mirror))
1753
1754                 if myaux.get("PROVIDE"):
1755                         stats["virtual.oldstyle"]+=1
1756                         fails["virtual.oldstyle"].append(relative_path)
1757
1758                 for pos, missing_var in enumerate(missingvars):
1759                         if not myaux.get(missing_var):
1760                                 if catdir == "virtual" and \
1761                                         missing_var in ("HOMEPAGE", "LICENSE"):
1762                                         continue
1763                                 if live_ebuild and missing_var == "KEYWORDS":
1764                                         continue
1765                                 myqakey=missingvars[pos]+".missing"
1766                                 stats[myqakey]=stats[myqakey]+1
1767                                 fails[myqakey].append(x+"/"+y+".ebuild")
1768
1769                 if catdir == "virtual":
1770                         for var in ("HOMEPAGE", "LICENSE"):
1771                                 if myaux.get(var):
1772                                         myqakey = var + ".virtual"
1773                                         stats[myqakey] = stats[myqakey] + 1
1774                                         fails[myqakey].append(relative_path)
1775
1776                 # 14 is the length of DESCRIPTION=""
1777                 if len(myaux['DESCRIPTION']) > max_desc_len:
1778                         stats['DESCRIPTION.toolong'] += 1
1779                         fails['DESCRIPTION.toolong'].append(
1780                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1781                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1782
1783                 keywords = myaux["KEYWORDS"].split()
1784                 stable_keywords = []
1785                 for keyword in keywords:
1786                         if not keyword.startswith("~") and \
1787                                 not keyword.startswith("-"):
1788                                 stable_keywords.append(keyword)
1789                 if stable_keywords:
1790                         if ebuild_path in new_ebuilds:
1791                                 stable_keywords.sort()
1792                                 stats["KEYWORDS.stable"] += 1
1793                                 fails["KEYWORDS.stable"].append(
1794                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1795                                                 " ".join(stable_keywords))
1796
1797                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1798                         if not kw.startswith("-"))
1799
1800                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1801                 if previous_keywords is None:
1802                         slot_keywords[myaux["SLOT"]] = set()
1803                 elif ebuild_archs and not live_ebuild:
1804                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1805                         if dropped_keywords:
1806                                 stats["KEYWORDS.dropped"] += 1
1807                                 fails["KEYWORDS.dropped"].append(
1808                                         relative_path + ": %s" % \
1809                                         " ".join(sorted(dropped_keywords)))
1810
1811                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1812
1813                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1814                 if "-*" in keywords:
1815                         haskeyword = False
1816                         for kw in keywords:
1817                                 if kw[0] == "~":
1818                                         kw = kw[1:]
1819                                 if kw in kwlist:
1820                                         haskeyword = True
1821                         if not haskeyword:
1822                                 stats["KEYWORDS.stupid"] += 1
1823                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1824
1825                 """
1826                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1827                 not be allowed to be marked stable
1828                 """
1829                 if live_ebuild and repo_config.name == "gentoo":
1830                         bad_stable_keywords = []
1831                         for keyword in keywords:
1832                                 if not keyword.startswith("~") and \
1833                                         not keyword.startswith("-"):
1834                                         bad_stable_keywords.append(keyword)
1835                                 del keyword
1836                         if bad_stable_keywords:
1837                                 stats["LIVEVCS.stable"] += 1
1838                                 fails["LIVEVCS.stable"].append(
1839                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1840                                                 bad_stable_keywords)
1841                         del bad_stable_keywords
1842
1843                         if keywords and not has_global_mask(pkg):
1844                                 stats["LIVEVCS.unmasked"] += 1
1845                                 fails["LIVEVCS.unmasked"].append(relative_path)
1846
1847                 if options.ignore_arches:
1848                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1849                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1850                 else:
1851                         arches=[]
1852                         for keyword in myaux["KEYWORDS"].split():
1853                                 if (keyword[0]=="-"):
1854                                         continue
1855                                 elif (keyword[0]=="~"):
1856                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1857                                 else:
1858                                         arches.append([keyword, keyword, [keyword]])
1859                         if not arches:
1860                                 # Use an empty profile for checking dependencies of
1861                                 # packages that have empty KEYWORDS.
1862                                 arches.append(['**', '**', ['**']])
1863
1864                 unknown_pkgs = set()
1865                 baddepsyntax = False
1866                 badlicsyntax = False
1867                 badprovsyntax = False
1868                 catpkg = catdir+"/"+y
1869
1870                 inherited_java_eclass = "java-pkg-2" in inherited or \
1871                         "java-pkg-opt-2" in inherited
1872                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1873                 operator_tokens = set(["||", "(", ")"])
1874                 type_list, badsyntax = [], []
1875                 for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
1876                         mydepstr = myaux[mytype]
1877
1878                         buildtime = mytype in Package._buildtime_keys
1879                         runtime = mytype in Package._runtime_keys
1880                         token_class = None
1881                         if mytype.endswith("DEPEND"):
1882                                 token_class=portage.dep.Atom
1883
1884                         try:
1885                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1886                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1887                         except portage.exception.InvalidDependString as e:
1888                                 atoms = None
1889                                 badsyntax.append(str(e))
1890
1891                         if atoms and mytype.endswith("DEPEND"):
1892                                 if runtime and \
1893                                         "test?" in mydepstr.split():
1894                                         stats[mytype + '.suspect'] += 1
1895                                         fails[mytype + '.suspect'].append(relative_path + \
1896                                                 ": 'test?' USE conditional in %s" % mytype)
1897
1898                                 for atom in atoms:
1899                                         if atom == "||":
1900                                                 continue
1901
1902                                         # Skip dependency.unknown for blockers, so that we
1903                                         # don't encourage people to remove necessary blockers,
1904                                         # as discussed in bug #382407.
1905                                         if atom.blocker is None and \
1906                                                 not portdb.xmatch("match-all", atom) and \
1907                                                 not atom.cp.startswith("virtual/"):
1908                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1909
1910                                         is_blocker = atom.blocker
1911
1912                                         if catdir != "virtual":
1913                                                 if not is_blocker and \
1914                                                         atom.cp in suspect_virtual:
1915                                                         stats['virtual.suspect'] += 1
1916                                                         fails['virtual.suspect'].append(
1917                                                                 relative_path +
1918                                                                 ": %s: consider using '%s' instead of '%s'" %
1919                                                                 (mytype, suspect_virtual[atom.cp], atom))
1920
1921                                         if buildtime and \
1922                                                 not is_blocker and \
1923                                                 not inherited_java_eclass and \
1924                                                 atom.cp == "virtual/jdk":
1925                                                 stats['java.eclassesnotused'] += 1
1926                                                 fails['java.eclassesnotused'].append(relative_path)
1927                                         elif buildtime and \
1928                                                 not is_blocker and \
1929                                                 not inherited_wxwidgets_eclass and \
1930                                                 atom.cp == "x11-libs/wxGTK":
1931                                                 stats['wxwidgets.eclassnotused'] += 1
1932                                                 fails['wxwidgets.eclassnotused'].append(
1933                                                         (relative_path + ": %ss on x11-libs/wxGTK"
1934                                                         " without inheriting wxwidgets.eclass") % mytype)
1935                                         elif runtime:
1936                                                 if not is_blocker and \
1937                                                         atom.cp in suspect_rdepend:
1938                                                         stats[mytype + '.suspect'] += 1
1939                                                         fails[mytype + '.suspect'].append(
1940                                                                 relative_path + ": '%s'" % atom)
1941
1942                                         if atom.operator == "~" and \
1943                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1944                                                 qacat = 'dependency.badtilde'
1945                                                 stats[qacat] += 1
1946                                                 fails[qacat].append(
1947                                                         (relative_path + ": %s uses the ~ operator"
1948                                                          " with a non-zero revision:" + \
1949                                                          " '%s'") % (mytype, atom))
1950
1951                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1952
1953                 for m,b in zip(type_list, badsyntax):
1954                         if m.endswith("DEPEND"):
1955                                 qacat = "dependency.syntax"
1956                         else:
1957                                 qacat = m + ".syntax"
1958                         stats[qacat] += 1
1959                         fails[qacat].append("%s: %s: %s" % (relative_path, m, b))
1960
1961                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1962                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1963                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1964                 badlicsyntax = badlicsyntax > 0
1965                 badprovsyntax = badprovsyntax > 0
1966
1967                 # uselist checks - global
1968                 myuse = []
1969                 default_use = []
1970                 for myflag in myaux["IUSE"].split():
1971                         flag_name = myflag.lstrip("+-")
1972                         used_useflags.add(flag_name)
1973                         if myflag != flag_name:
1974                                 default_use.append(myflag)
1975                         if flag_name not in uselist:
1976                                 myuse.append(flag_name)
1977
1978                 # uselist checks - metadata
1979                 for mypos in range(len(myuse)-1,-1,-1):
1980                         if myuse[mypos] and (myuse[mypos] in muselist):
1981                                 del myuse[mypos]
1982
1983                 if default_use and not eapi_has_iuse_defaults(eapi):
1984                         for myflag in default_use:
1985                                 stats['EAPI.incompatible'] += 1
1986                                 fails['EAPI.incompatible'].append(
1987                                         (relative_path + ": IUSE defaults" + \
1988                                         " not supported with EAPI='%s':" + \
1989                                         " '%s'") % (eapi, myflag))
1990
1991                 for mypos in range(len(myuse)):
1992                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1993                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1994
1995                 # license checks
1996                 if not badlicsyntax:
1997                         # Parse the LICENSE variable, remove USE conditions and
1998                         # flatten it.
1999                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2000                         # Check each entry to ensure that it exists in PORTDIR's
2001                         # license directory.
2002                         for lic in licenses:
2003                                 # Need to check for "||" manually as no portage
2004                                 # function will remove it without removing values.
2005                                 if lic not in liclist and lic != "||":
2006                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
2007                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
2008
2009                 #keyword checks
2010                 myuse = myaux["KEYWORDS"].split()
2011                 for mykey in myuse:
2012                         if mykey not in ("-*", "*", "~*"):
2013                                 myskey = mykey
2014                                 if myskey[:1] == "-":
2015                                         myskey = myskey[1:]
2016                                 if myskey[:1] == "~":
2017                                         myskey = myskey[1:]
2018                                 if myskey not in kwlist:
2019                                         stats["KEYWORDS.invalid"] += 1
2020                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
2021                                 elif myskey not in profiles:
2022                                         stats["KEYWORDS.invalid"] += 1
2023                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
2024
2025                 #restrict checks
2026                 myrestrict = None
2027                 try:
2028                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2029                 except portage.exception.InvalidDependString as e:
2030                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
2031                         fails["RESTRICT.syntax"].append(
2032                                 "%s: RESTRICT: %s" % (relative_path, e))
2033                         del e
2034                 if myrestrict:
2035                         myrestrict = set(myrestrict)
2036                         mybadrestrict = myrestrict.difference(valid_restrict)
2037                         if mybadrestrict:
2038                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2039                                 for mybad in mybadrestrict:
2040                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
2041                 #REQUIRED_USE check
2042                 required_use = myaux["REQUIRED_USE"]
2043                 if required_use:
2044                         if not eapi_has_required_use(eapi):
2045                                 stats['EAPI.incompatible'] += 1
2046                                 fails['EAPI.incompatible'].append(
2047                                         relative_path + ": REQUIRED_USE" + \
2048                                         " not supported with EAPI='%s'" % (eapi,))
2049                         try:
2050                                 portage.dep.check_required_use(required_use, (),
2051                                         pkg.iuse.is_valid_flag, eapi=eapi)
2052                         except portage.exception.InvalidDependString as e:
2053                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2054                                 fails["REQUIRED_USE.syntax"].append(
2055                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2056                                 del e
2057
2058                 # Syntax Checks
2059                 relative_path = os.path.join(x, y + ".ebuild")
2060                 full_path = os.path.join(repodir, relative_path)
2061                 if not vcs_preserves_mtime:
2062                         if ebuild_path not in new_ebuilds and \
2063                                 ebuild_path not in modified_ebuilds:
2064                                 pkg.mtime = None
2065                 try:
2066                         # All ebuilds should have utf_8 encoding.
2067                         f = io.open(_unicode_encode(full_path,
2068                                 encoding=_encodings['fs'], errors='strict'),
2069                                 mode='r', encoding=_encodings['repo.content'])
2070                         try:
2071                                 for check_name, e in run_checks(f, pkg):
2072                                         stats[check_name] += 1
2073                                         fails[check_name].append(relative_path + ': %s' % e)
2074                         finally:
2075                                 f.close()
2076                 except UnicodeDecodeError:
2077                         # A file.UTF8 failure will have already been recorded above.
2078                         pass
2079
2080                 if options.force:
2081                         # The dep_check() calls are the most expensive QA test. If --force
2082                         # is enabled, there's no point in wasting time on these since the
2083                         # user is intent on forcing the commit anyway.
2084                         continue
2085
2086                 for keyword,arch,groups in arches:
2087
2088                         if arch not in profiles:
2089                                 # A missing profile will create an error further down
2090                                 # during the KEYWORDS verification.
2091                                 continue
2092
2093                         for prof in profiles[arch]:
2094
2095                                 if prof.status not in ("stable", "dev") or \
2096                                         prof.status == "dev" and not options.include_dev:
2097                                         continue
2098
2099                                 dep_settings = arch_caches.get(prof.sub_path)
2100                                 if dep_settings is None:
2101                                         dep_settings = portage.config(
2102                                                 config_profile_path=prof.abs_path,
2103                                                 config_incrementals=repoman_incrementals,
2104                                                 config_root=config_root,
2105                                                 local_config=False,
2106                                                 _unmatched_removal=options.unmatched_removal,
2107                                                 env=env)
2108                                         dep_settings.categories = repoman_settings.categories
2109                                         if options.without_mask:
2110                                                 dep_settings._mask_manager_obj = \
2111                                                         copy.deepcopy(dep_settings._mask_manager)
2112                                                 dep_settings._mask_manager._pmaskdict.clear()
2113                                         arch_caches[prof.sub_path] = dep_settings
2114
2115                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2116                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2117                                 if xcache is None:
2118                                         portdb.melt()
2119                                         portdb.freeze()
2120                                         xcache = portdb.xcache
2121                                         xcache.update(shared_xmatch_caches)
2122                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2123
2124                                 trees[root]["porttree"].settings = dep_settings
2125                                 portdb.settings = dep_settings
2126                                 portdb.xcache = xcache
2127                                 # for package.use.mask support inside dep_check
2128                                 dep_settings.setcpv(pkg)
2129                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2130                                 # just in case, prevent config.reset() from nuking these.
2131                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2132
2133                                 if not baddepsyntax:
2134                                         ismasked = not ebuild_archs or \
2135                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2136                                         if ismasked:
2137                                                 if not have_pmasked:
2138                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2139                                                                 pkg.cpv, pkg.metadata))
2140                                                 if options.ignore_masked:
2141                                                         continue
2142                                                 #we are testing deps for a masked package; give it some lee-way
2143                                                 suffix="masked"
2144                                                 matchmode = "minimum-all"
2145                                         else:
2146                                                 suffix=""
2147                                                 matchmode = "minimum-visible"
2148
2149                                         if not have_dev_keywords:
2150                                                 have_dev_keywords = \
2151                                                         bool(dev_keywords.intersection(keywords))
2152
2153                                         if prof.status == "dev":
2154                                                 suffix=suffix+"indev"
2155
2156                                         for mytype in Package._dep_keys:
2157
2158                                                 mykey = "dependency.bad" + suffix
2159                                                 myvalue = myaux[mytype]
2160                                                 if not myvalue:
2161                                                         continue
2162
2163                                                 success, atoms = portage.dep_check(myvalue, portdb,
2164                                                         dep_settings, use="all", mode=matchmode,
2165                                                         trees=trees)
2166
2167                                                 if success:
2168                                                         if atoms:
2169
2170                                                                 # Don't bother with dependency.unknown for
2171                                                                 # cases in which *DEPEND.bad is triggered.
2172                                                                 for atom in atoms:
2173                                                                         # dep_check returns all blockers and they
2174                                                                         # aren't counted for *DEPEND.bad, so we
2175                                                                         # ignore them here.
2176                                                                         if not atom.blocker:
2177                                                                                 unknown_pkgs.discard(
2178                                                                                         (mytype, atom.unevaluated_atom))
2179
2180                                                                 if not prof.sub_path:
2181                                                                         # old-style virtuals currently aren't
2182                                                                         # resolvable with empty profile, since
2183                                                                         # 'virtuals' mappings are unavailable
2184                                                                         # (it would be expensive to search
2185                                                                         # for PROVIDE in all ebuilds)
2186                                                                         atoms = [atom for atom in atoms if not \
2187                                                                                 (atom.cp.startswith('virtual/') and \
2188                                                                                 not portdb.cp_list(atom.cp))]
2189
2190                                                                 #we have some unsolvable deps
2191                                                                 #remove ! deps, which always show up as unsatisfiable
2192                                                                 atoms = [str(atom.unevaluated_atom) \
2193                                                                         for atom in atoms if not atom.blocker]
2194
2195                                                                 #if we emptied out our list, continue:
2196                                                                 if not atoms:
2197                                                                         continue
2198                                                                 stats[mykey]=stats[mykey]+1
2199                                                                 fails[mykey].append("%s: %s: %s(%s) %s" % \
2200                                                                         (relative_path, mytype, keyword,
2201                                                                         prof, repr(atoms)))
2202                                                 else:
2203                                                         stats[mykey]=stats[mykey]+1
2204                                                         fails[mykey].append("%s: %s: %s(%s) %s" % \
2205                                                                 (relative_path, mytype, keyword,
2206                                                                 prof, repr(atoms)))
2207
2208                 if not baddepsyntax and unknown_pkgs:
2209                         type_map = {}
2210                         for mytype, atom in unknown_pkgs:
2211                                 type_map.setdefault(mytype, set()).add(atom)
2212                         for mytype, atoms in type_map.items():
2213                                 stats["dependency.unknown"] += 1
2214                                 fails["dependency.unknown"].append("%s: %s: %s" %
2215                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2216
2217         # check if there are unused local USE-descriptions in metadata.xml
2218         # (unless there are any invalids, to avoid noise)
2219         if allvalid:
2220                 for myflag in muselist.difference(used_useflags):
2221                         stats["metadata.warning"] += 1
2222                         fails["metadata.warning"].append(
2223                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2224                                 (x, myflag))
2225
2226 if options.if_modified == "y" and len(effective_scanlist) < 1:
2227         logging.warn("--if-modified is enabled, but no modified packages were found!")
2228
2229 if options.mode == "manifest":
2230         sys.exit(dofail)
2231
2232 #dofail will be set to 1 if we have failed in at least one non-warning category
2233 dofail=0
2234 #dowarn will be set to 1 if we tripped any warnings
2235 dowarn=0
2236 #dofull will be set if we should print a "repoman full" informational message
2237 dofull = options.mode != 'full'
2238
2239 for x in qacats:
2240         if not stats[x]:
2241                 continue
2242         dowarn = 1
2243         if x not in qawarnings:
2244                 dofail = 1
2245
2246 if dofail or \
2247         (dowarn and not (options.quiet or options.mode == "scan")):
2248         dofull = 0
2249
2250 # Save QA output so that it can be conveniently displayed
2251 # in $EDITOR while the user creates a commit message.
2252 # Otherwise, the user would not be able to see this output
2253 # once the editor has taken over the screen.
2254 qa_output = io.StringIO()
2255 style_file = ConsoleStyleFile(sys.stdout)
2256 if options.mode == 'commit' and \
2257         (not commitmessage or not commitmessage.strip()):
2258         style_file.write_listener = qa_output
2259 console_writer = StyleWriter(file=style_file, maxcol=9999)
2260 console_writer.style_listener = style_file.new_styles
2261
2262 f = formatter.AbstractFormatter(console_writer)
2263
2264 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2265
2266 style_file.flush()
2267 del console_writer, f, style_file
2268 qa_output = qa_output.getvalue()
2269 qa_output = qa_output.splitlines(True)
2270
2271 def grouplist(mylist,seperator="/"):
2272         """(list,seperator="/") -- Takes a list of elements; groups them into
2273         same initial element categories. Returns a dict of {base:[sublist]}
2274         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2275         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2276         mygroups={}
2277         for x in mylist:
2278                 xs=x.split(seperator)
2279                 if xs[0]==".":
2280                         xs=xs[1:]
2281                 if xs[0] not in mygroups:
2282                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2283                 else:
2284                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2285         return mygroups
2286
2287 suggest_ignore_masked = False
2288 suggest_include_dev = False
2289
2290 if have_pmasked and not (options.without_mask or options.ignore_masked):
2291         suggest_ignore_masked = True
2292 if have_dev_keywords and not options.include_dev:
2293         suggest_include_dev = True
2294
2295 if suggest_ignore_masked or suggest_include_dev:
2296         print()
2297         if suggest_ignore_masked:
2298                 print(bold("Note: use --without-mask to check " + \
2299                         "KEYWORDS on dependencies of masked packages"))
2300
2301         if suggest_include_dev:
2302                 print(bold("Note: use --include-dev (-d) to check " + \
2303                         "dependencies for 'dev' profiles"))
2304         print()
2305
2306 if options.mode != 'commit':
2307         if dofull:
2308                 print(bold("Note: type \"repoman full\" for a complete listing."))
2309         if dowarn and not dofail:
2310                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2311         elif not dofail:
2312                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2313         elif dofail:
2314                 print(bad("Please fix these important QA issues first."))
2315                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2316                 sys.exit(1)
2317 else:
2318         if dofail and can_force and options.force and not options.pretend:
2319                 print(green("RepoMan sez:") + \
2320                         " \"You want to commit even with these QA issues?\n" + \
2321                         "              I'll take it this time, but I'm not happy.\"\n")
2322         elif dofail:
2323                 if options.force and not can_force:
2324                         print(bad("The --force option has been disabled due to extraordinary issues."))
2325                 print(bad("Please fix these important QA issues first."))
2326                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2327                 sys.exit(1)
2328
2329         if options.pretend:
2330                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2331
2332         myunadded = []
2333         if vcs == "cvs":
2334                 try:
2335                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2336                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2337                 except SystemExit as e:
2338                         raise  # TODO propagate this
2339                 except:
2340                         err("Error retrieving CVS tree; exiting.")
2341         if vcs == "svn":
2342                 try:
2343                         with repoman_popen("svn status --no-ignore") as f:
2344                                 svnstatus = f.readlines()
2345                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2346                 except SystemExit as e:
2347                         raise  # TODO propagate this
2348                 except:
2349                         err("Error retrieving SVN info; exiting.")
2350         if vcs == "git":
2351                 # get list of files not under version control or missing
2352                 myf = repoman_popen("git ls-files --others")
2353                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2354                 myf.close()
2355         if vcs == "bzr":
2356                 try:
2357                         with repoman_popen("bzr status -S .") as f:
2358                                 bzrstatus = f.readlines()
2359                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2360                 except SystemExit as e:
2361                         raise  # TODO propagate this
2362                 except:
2363                         err("Error retrieving bzr info; exiting.")
2364         if vcs == "hg":
2365                 with repoman_popen("hg status --no-status --unknown .") as f:
2366                         myunadded = f.readlines()
2367                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2368
2369                 # Mercurial doesn't handle manually deleted files as removed from
2370                 # the repository, so the user need to remove them before commit,
2371                 # using "hg remove [FILES]"
2372                 with repoman_popen("hg status --no-status --deleted .") as f:
2373                         mydeleted = f.readlines()
2374                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2375
2376
2377         myautoadd=[]
2378         if myunadded:
2379                 for x in range(len(myunadded)-1,-1,-1):
2380                         xs=myunadded[x].split("/")
2381                         if xs[-1]=="files":
2382                                 print("!!! files dir is not added! Please correct this.")
2383                                 sys.exit(-1)
2384                         elif xs[-1]=="Manifest":
2385                                 # It's a manifest... auto add
2386                                 myautoadd+=[myunadded[x]]
2387                                 del myunadded[x]
2388
2389         if myunadded:
2390                 print(red("!!! The following files are in your local tree but are not added to the master"))
2391                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2392                 for x in myunadded:
2393                         print("   ",x)
2394                 print()
2395                 print()
2396                 sys.exit(1)
2397
2398         if vcs == "hg" and mydeleted:
2399                 print(red("!!! The following files are removed manually from your local tree but are not"))
2400                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2401                 for x in mydeleted:
2402                         print("   ",x)
2403                 print()
2404                 print()
2405                 sys.exit(1)
2406
2407         if vcs == "cvs":
2408                 mycvstree = cvstree.getentries("./", recursive=1)
2409                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2410                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2411                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2412                 bin_blob_pattern = re.compile("^-kb$")
2413                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2414                         recursive=1, basedir="./"))
2415
2416
2417         if vcs == "svn":
2418                 with repoman_popen("svn status") as f:
2419                         svnstatus = f.readlines()
2420                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2421                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2422                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2423
2424                 # Subversion expands keywords specified in svn:keywords properties.
2425                 with repoman_popen("svn propget -R svn:keywords") as f:
2426                         props = f.readlines()
2427                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2428                         for prop in props if " - " in prop)
2429
2430         elif vcs == "git":
2431                 with repoman_popen("git diff-index --name-only "
2432                         "--relative --diff-filter=M HEAD") as f:
2433                         mychanged = f.readlines()
2434                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2435
2436                 with repoman_popen("git diff-index --name-only "
2437                         "--relative --diff-filter=A HEAD") as f:
2438                         mynew = f.readlines()
2439                 mynew = ["./" + elem[:-1] for elem in mynew]
2440
2441                 with repoman_popen("git diff-index --name-only "
2442                         "--relative --diff-filter=D HEAD") as f:
2443                         myremoved = f.readlines()
2444                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2445
2446         if vcs == "bzr":
2447                 with repoman_popen("bzr status -S .") as f:
2448                         bzrstatus = f.readlines()
2449                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2450                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2451                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2452                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2453                 # Bazaar expands nothing.
2454
2455         if vcs == "hg":
2456                 with repoman_popen("hg status --no-status --modified .") as f:
2457                         mychanged = f.readlines()
2458                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2459
2460                 with repoman_popen("hg status --no-status --added .") as f:
2461                         mynew = f.readlines()
2462                 mynew = ["./" + elem.rstrip() for elem in mynew]
2463
2464                 with repoman_popen("hg status --no-status --removed .") as f:
2465                         myremoved = f.readlines()
2466                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2467
2468         if vcs:
2469                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2470                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2471                         print()
2472                         print("(Didn't find any changed files...)")
2473                         print()
2474                         sys.exit(1)
2475
2476         # Manifests need to be regenerated after all other commits, so don't commit
2477         # them now even if they have changed.
2478         mymanifests = set()
2479         myupdates = set()
2480         for f in mychanged + mynew:
2481                 if "Manifest" == os.path.basename(f):
2482                         mymanifests.add(f)
2483                 else:
2484                         myupdates.add(f)
2485         myupdates.difference_update(myremoved)
2486         myupdates = list(myupdates)
2487         mymanifests = list(mymanifests)
2488         myheaders = []
2489         mydirty = []
2490
2491         commitmessage = options.commitmsg
2492         if options.commitmsgfile:
2493                 try:
2494                         f = io.open(_unicode_encode(options.commitmsgfile,
2495                         encoding=_encodings['fs'], errors='strict'),
2496                         mode='r', encoding=_encodings['content'], errors='replace')
2497                         commitmessage = f.read()
2498                         f.close()
2499                         del f
2500                 except (IOError, OSError) as e:
2501                         if e.errno == errno.ENOENT:
2502                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2503                         else:
2504                                 raise
2505                 # We've read the content so the file is no longer needed.
2506                 commitmessagefile = None
2507         if not commitmessage or not commitmessage.strip():
2508                 try:
2509                         editor = os.environ.get("EDITOR")
2510                         if editor and utilities.editor_is_executable(editor):
2511                                 commitmessage = utilities.get_commit_message_with_editor(
2512                                         editor, message=qa_output)
2513                         else:
2514                                 commitmessage = utilities.get_commit_message_with_stdin()
2515                 except KeyboardInterrupt:
2516                         exithandler()
2517                 if not commitmessage or not commitmessage.strip():
2518                         print("* no commit message?  aborting commit.")
2519                         sys.exit(1)
2520         commitmessage = commitmessage.rstrip()
2521         changelog_msg = commitmessage
2522         portage_version = getattr(portage, "VERSION", None)
2523         if portage_version is None:
2524                 sys.stderr.write("Failed to insert portage version in message!\n")
2525                 sys.stderr.flush()
2526                 portage_version = "Unknown"
2527
2528         if vcs == "git":
2529                 # Use new footer only for git (see bug #438364).
2530                 commit_footer = "\n\nPackage-manager: portage-%s" % portage_version
2531                 if options.force:
2532                         commit_footer += "\nRepoMan-options: --force"
2533                 if sign_manifests:
2534                         commit_footer += "\nManifest-sign-key: %s" % \
2535                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2536         else:
2537                 unameout = platform.system() + " "
2538                 if platform.system() in ["Darwin", "SunOS"]:
2539                         unameout += platform.processor()
2540                 else:
2541                         unameout += platform.machine()
2542                 commit_footer += "\n\n(Portage version: %s/%s/%s" % \
2543                         (portage_version, vcs, unameout)
2544                 if options.force:
2545                         commit_footer += ", RepoMan options: --force"
2546                 if sign_manifests:
2547                         commit_footer += ", signed Manifest commit with key %s" % \
2548                                 repoman_settings.get("PORTAGE_GPG_KEY", "")
2549                 else:
2550                         commit_footer += ", unsigned Manifest commit"
2551                 commit_footer += ")"
2552
2553         commitmessage += commit_footer
2554
2555         if options.echangelog in ('y', 'force'):
2556                 logging.info("checking for unmodified ChangeLog files")
2557                 committer_name = utilities.get_committer_name(env=repoman_settings)
2558                 for x in sorted(vcs_files_to_cps(
2559                         chain(myupdates, mymanifests, myremoved))):
2560                         catdir, pkgdir = x.split("/")
2561                         checkdir = repodir + "/" + x
2562                         checkdir_relative = ""
2563                         if repolevel < 3:
2564                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2565                         if repolevel < 2:
2566                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2567                         checkdir_relative = os.path.join(".", checkdir_relative)
2568
2569                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2570                         changelog_modified = changelog_path in modified_changelogs
2571                         if changelog_modified and options.echangelog != 'force':
2572                                 continue
2573
2574                         # get changes for this package
2575                         cdrlen = len(checkdir_relative)
2576                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2577                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2578                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2579
2580                         # Skip ChangeLog generation if only the Manifest was modified,
2581                         # as discussed in bug #398009.
2582                         nontrivial_cl_files = set()
2583                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2584                         nontrivial_cl_files.difference_update(['Manifest'])
2585                         if not nontrivial_cl_files and options.echangelog != 'force':
2586                                 continue
2587
2588                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2589                                 committer_name, changelog_msg,
2590                                 os.path.join(repodir, 'skel.ChangeLog'),
2591                                 catdir, pkgdir,
2592                                 new=clnew, removed=clremoved, changed=clchanged,
2593                                 pretend=options.pretend)
2594                         if new_changelog is None:
2595                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2596                                         level=logging.ERROR, noiselevel=-1)
2597                                 sys.exit(1)
2598
2599                         # if the ChangeLog was just created, add it to vcs
2600                         if new_changelog:
2601                                 myautoadd.append(changelog_path)
2602                                 # myautoadd is appended to myupdates below
2603                         else:
2604                                 myupdates.append(changelog_path)
2605
2606                         if options.ask and not options.pretend:
2607                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2608                                 repoman_settings["O"] = checkdir
2609                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2610
2611         if myautoadd:
2612                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2613                 add_cmd = [vcs, "add"]
2614                 add_cmd += myautoadd
2615                 if options.pretend:
2616                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2617                                 noiselevel=-1)
2618                 else:
2619                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2620                                 # Python 3.1 produces the following TypeError if raw bytes are
2621                                 # passed to subprocess.call():
2622                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2623                                 #     errread, errwrite)
2624                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2625                                 #     raise child_exception
2626                                 # TypeError: expected an object with the buffer interface
2627                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2628                         retcode = subprocess.call(add_cmd)
2629                         if retcode != os.EX_OK:
2630                                 logging.error(
2631                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2632                                 sys.exit(retcode)
2633
2634                 myupdates += myautoadd
2635
2636         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2637
2638         if vcs not in ('cvs', 'svn'):
2639                 # With git, bzr and hg, there's never any keyword expansion, so
2640                 # there's no need to regenerate manifests and all files will be
2641                 # committed in one big commit at the end.
2642                 print()
2643         elif not repo_config.thin_manifest:
2644                 if vcs == 'cvs':
2645                         headerstring = "'\$(Header|Id).*\$'"
2646                 elif vcs == "svn":
2647                         svn_keywords = dict((k.lower(), k) for k in [
2648                                         "Rev",
2649                                         "Revision",
2650                                         "LastChangedRevision",
2651                                         "Date",
2652                                         "LastChangedDate",
2653                                         "Author",
2654                                         "LastChangedBy",
2655                                         "URL",
2656                                         "HeadURL",
2657                                         "Id",
2658                                         "Header",
2659                         ])
2660
2661                 for myfile in myupdates:
2662
2663                         # for CVS, no_expansion contains files that are excluded from expansion
2664                         if vcs == "cvs":
2665                                 if myfile in no_expansion:
2666                                         continue
2667
2668                         # for SVN, expansion contains files that are included in expansion
2669                         elif vcs == "svn":
2670                                 if myfile not in expansion:
2671                                         continue
2672
2673                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2674                                 enabled_keywords = []
2675                                 for k in expansion[myfile]:
2676                                         keyword = svn_keywords.get(k.lower())
2677                                         if keyword is not None:
2678                                                 enabled_keywords.append(keyword)
2679
2680                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2681
2682                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2683                                 portage._shell_quote(myfile))
2684                         if myout[0] == 0:
2685                                 myheaders.append(myfile)
2686
2687                 print("%s have headers that will change." % green(str(len(myheaders))))
2688                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2689
2690         logging.info("myupdates: %s", myupdates)
2691         logging.info("myheaders: %s", myheaders)
2692
2693         if options.ask and userquery('Commit changes?', True) != 'Yes':
2694                 print("* aborting commit.")
2695                 sys.exit(128 + signal.SIGINT)
2696
2697         # Handle the case where committed files have keywords which
2698         # will change and need a priming commit before the Manifest
2699         # can be committed.
2700         if (myupdates or myremoved) and myheaders:
2701                 myfiles = myupdates + myremoved
2702                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2703                 mymsg = os.fdopen(fd, "wb")
2704                 mymsg.write(_unicode_encode(commitmessage))
2705                 mymsg.close()
2706
2707                 print()
2708                 print(green("Using commit message:"))
2709                 print(green("------------------------------------------------------------------------------"))
2710                 print(commitmessage)
2711                 print(green("------------------------------------------------------------------------------"))
2712                 print()
2713
2714                 # Having a leading ./ prefix on file paths can trigger a bug in
2715                 # the cvs server when committing files to multiple directories,
2716                 # so strip the prefix.
2717                 myfiles = [f.lstrip("./") for f in myfiles]
2718
2719                 commit_cmd = [vcs]
2720                 commit_cmd.extend(vcs_global_opts)
2721                 commit_cmd.append("commit")
2722                 commit_cmd.extend(vcs_local_opts)
2723                 commit_cmd.extend(["-F", commitmessagefile])
2724                 commit_cmd.extend(myfiles)
2725
2726                 try:
2727                         if options.pretend:
2728                                 print("(%s)" % (" ".join(commit_cmd),))
2729                         else:
2730                                 retval = spawn(commit_cmd, env=os.environ)
2731                                 if retval != os.EX_OK:
2732                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2733                                                 "error code: %s\n") % (vcs, retval),
2734                                                 level=logging.ERROR, noiselevel=-1)
2735                                         sys.exit(retval)
2736                 finally:
2737                         try:
2738                                 os.unlink(commitmessagefile)
2739                         except OSError:
2740                                 pass
2741
2742         # Setup the GPG commands
2743         def gpgsign(filename):
2744                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2745                 if gpgcmd is None:
2746                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2747                                 " Is make.globals missing?")
2748                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2749                         "PORTAGE_GPG_KEY" not in repoman_settings:
2750                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2751                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2752                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2753                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2754                                         os.path.expanduser("~/.gnupg")
2755                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2756                                         % repoman_settings["PORTAGE_GPG_DIR"])
2757                         else:
2758                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2759                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2760                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2761                                 raise portage.exception.InvalidLocation(
2762                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2763                                         repoman_settings["PORTAGE_GPG_DIR"])
2764                 gpgvars = {"FILE": filename}
2765                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2766                         v = repoman_settings.get(k)
2767                         if v is not None:
2768                                 gpgvars[k] = v
2769                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2770                 if options.pretend:
2771                         print("("+gpgcmd+")")
2772                 else:
2773                         # Encode unicode manually for bug #310789.
2774                         gpgcmd = portage.util.shlex_split(gpgcmd)
2775                         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
2776                                 # Python 3.1 does not support bytes in Popen args.
2777                                 gpgcmd = [_unicode_encode(arg,
2778                                         encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
2779                         rValue = subprocess.call(gpgcmd)
2780                         if rValue == os.EX_OK:
2781                                 os.rename(filename+".asc", filename)
2782                         else:
2783                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2784
2785         def need_signature(filename):
2786                 try:
2787                         with open(_unicode_encode(filename,
2788                                 encoding=_encodings['fs'], errors='strict'), 'rb') as f:
2789                                 return b"BEGIN PGP SIGNED MESSAGE" not in f.readline()
2790                 except IOError as e:
2791                         if e.errno in (errno.ENOENT, errno.ESTALE):
2792                                 return False
2793                         raise
2794
2795         # When files are removed and re-added, the cvs server will put /Attic/
2796         # inside the $Header path. This code detects the problem and corrects it
2797         # so that the Manifest will generate correctly. See bug #169500.
2798         # Use binary mode in order to avoid potential character encoding issues.
2799         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2800         attic_str = b'/Attic/'
2801         attic_replace = b'/'
2802         for x in myheaders:
2803                 f = open(_unicode_encode(x,
2804                         encoding=_encodings['fs'], errors='strict'),
2805                         mode='rb')
2806                 mylines = f.readlines()
2807                 f.close()
2808                 modified = False
2809                 for i, line in enumerate(mylines):
2810                         if cvs_header_re.match(line) is not None and \
2811                                 attic_str in line:
2812                                 mylines[i] = line.replace(attic_str, attic_replace)
2813                                 modified = True
2814                 if modified:
2815                         portage.util.write_atomic(x, b''.join(mylines),
2816                                 mode='wb')
2817
2818         if repolevel == 1:
2819                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2820                         "doing the entire repository.\"\n")
2821
2822         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2823
2824                 for x in sorted(vcs_files_to_cps(
2825                         chain(myupdates, myremoved, mymanifests))):
2826                         repoman_settings["O"] = os.path.join(repodir, x)
2827                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2828
2829         signed = False
2830         if sign_manifests:
2831                 signed = True
2832                 try:
2833                         for x in sorted(vcs_files_to_cps(
2834                                 chain(myupdates, myremoved, mymanifests))):
2835                                 repoman_settings["O"] = os.path.join(repodir, x)
2836                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2837                                 if not need_signature(manifest_path):
2838                                         continue
2839                                 gpgsign(manifest_path)
2840                 except portage.exception.PortageException as e:
2841                         portage.writemsg("!!! %s\n" % str(e))
2842                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2843                         signed = False
2844
2845         if vcs == 'git':
2846                 # It's not safe to use the git commit -a option since there might
2847                 # be some modified files elsewhere in the working tree that the
2848                 # user doesn't want to commit. Therefore, call git update-index
2849                 # in order to ensure that the index is updated with the latest
2850                 # versions of all new and modified files in the relevant portion
2851                 # of the working tree.
2852                 myfiles = mymanifests + myupdates
2853                 myfiles.sort()
2854                 update_index_cmd = ["git", "update-index"]
2855                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2856                 if options.pretend:
2857                         print("(%s)" % (" ".join(update_index_cmd),))
2858                 else:
2859                         retval = spawn(update_index_cmd, env=os.environ)
2860                         if retval != os.EX_OK:
2861                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2862                                         "error code: %s\n") % (vcs, retval),
2863                                         level=logging.ERROR, noiselevel=-1)
2864                                 sys.exit(retval)
2865
2866         if True:
2867
2868                 myfiles = mymanifests[:]
2869                 # If there are no header (SVN/CVS keywords) changes in
2870                 # the files, this Manifest commit must include the
2871                 # other (yet uncommitted) files.
2872                 if not myheaders:
2873                         myfiles += myupdates
2874                         myfiles += myremoved
2875                 myfiles.sort()
2876
2877                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2878                 mymsg = os.fdopen(fd, "wb")
2879                 mymsg.write(_unicode_encode(commitmessage))
2880                 mymsg.close()
2881
2882                 commit_cmd = []
2883                 if options.pretend and vcs is None:
2884                         # substitute a bogus value for pretend output
2885                         commit_cmd.append("cvs")
2886                 else:
2887                         commit_cmd.append(vcs)
2888                 commit_cmd.extend(vcs_global_opts)
2889                 commit_cmd.append("commit")
2890                 commit_cmd.extend(vcs_local_opts)
2891                 if vcs == "hg":
2892                         commit_cmd.extend(["--logfile", commitmessagefile])
2893                         commit_cmd.extend(myfiles)
2894                 else:
2895                         commit_cmd.extend(["-F", commitmessagefile])
2896                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2897
2898                 try:
2899                         if options.pretend:
2900                                 print("(%s)" % (" ".join(commit_cmd),))
2901                         else:
2902                                 retval = spawn(commit_cmd, env=os.environ)
2903                                 if retval != os.EX_OK:
2904
2905                                         if repo_config.sign_commit and vcs == 'git' and \
2906                                                 not git_supports_gpg_sign():
2907                                                 # Inform user that newer git is needed (bug #403323).
2908                                                 logging.error(
2909                                                         "Git >=1.7.9 is required for signed commits!")
2910
2911                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2912                                                 "error code: %s\n") % (vcs, retval),
2913                                                 level=logging.ERROR, noiselevel=-1)
2914                                         sys.exit(retval)
2915                 finally:
2916                         try:
2917                                 os.unlink(commitmessagefile)
2918                         except OSError:
2919                                 pass
2920
2921         print()
2922         if vcs:
2923                 print("Commit complete.")
2924         else:
2925                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2926         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2927 sys.exit(0)
2928