bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2012 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27
  28 try:
  29         from urllib.request import urlopen as urllib_request_urlopen
  30 except ImportError:
  31         from urllib import urlopen as urllib_request_urlopen
  32
  33 from itertools import chain
  34 from stat import S_ISDIR
  35
  36 try:
  37         import portage
  38 except ImportError:
  39         from os import path as osp
  40         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  41         import portage
  42 portage._disable_legacy_globals()
  43 portage.dep._internal_warnings = True
  44
  45 try:
  46         import xml.etree.ElementTree
  47         from xml.parsers.expat import ExpatError
  48 except (SystemExit, KeyboardInterrupt):
  49         raise
  50 except (ImportError, SystemError, RuntimeError, Exception):
  51         # broken or missing xml support
  52         # http://bugs.python.org/issue14988
  53         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  54         from portage.output import EOutput
  55         out = EOutput()
  56         for line in msg:
  57                 out.eerror(line)
  58         sys.exit(1)
  59
  60 from portage import os
  61 from portage import subprocess_getstatusoutput
  62 from portage import _encodings
  63 from portage import _unicode_encode
  64 from repoman.checks import run_checks
  65 from repoman import utilities
  66 from repoman.herdbase import make_herd_base
  67 from _emerge.Package import Package
  68 from _emerge.RootConfig import RootConfig
  69 from _emerge.userquery import userquery
  70 import portage.checksum
  71 import portage.const
  72 from portage import cvstree, normalize_path
  73 from portage import util
  74 from portage.exception import (FileNotFound, MissingParameter,
  75         ParseError, PermissionDenied)
  76 from portage.manifest import _prohibited_filename_chars_re as \
  77         disallowed_filename_chars_re
  78 from portage.process import find_binary, spawn
  79 from portage.output import bold, create_color_func, \
  80         green, nocolor, red
  81 from portage.output import ConsoleStyleFile, StyleWriter
  82 from portage.util import writemsg_level
  83 from portage.util._desktop_entry import validate_desktop_entry
  84 from portage.package.ebuild.digestgen import digestgen
  85 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  86
  87 if sys.hexversion >= 0x3000000:
  88         basestring = str
  89
  90 util.initialize_logger()
  91
  92 # 14 is the length of DESCRIPTION=""
  93 max_desc_len = 100
  94 allowed_filename_chars="a-zA-Z0-9._-+:"
  95 pv_toolong_re = re.compile(r'[0-9]{19,}')
  96 bad = create_color_func("BAD")
  97
  98 # A sane umask is needed for files that portage creates.
  99 os.umask(0o22)
 100 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
 101 # behave incrementally.
 102 repoman_incrementals = tuple(x for x in \
 103         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 104 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 105 repoman_settings = portage.config(config_root=config_root, local_config=False)
 106
 107 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 108         repoman_settings.get('TERM') == 'dumb' or \
 109         not sys.stdout.isatty():
 110         nocolor()
 111
 112 def warn(txt):
 113         print("repoman: " + txt)
 114
 115 def err(txt):
 116         warn(txt)
 117         sys.exit(1)
 118
 119 def exithandler(signum=None, frame=None):
 120         logging.fatal("Interrupted; exiting...")
 121         if signum is None:
 122                 sys.exit(1)
 123         else:
 124                 sys.exit(128 + signum)
 125
 126 signal.signal(signal.SIGINT,exithandler)
 127
 128 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 129         """Repoman needs it's own HelpFormatter for now, because the default ones
 130         murder the help text."""
 131
 132         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 133                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 134
 135         def format_description(self, description):
 136                 return description
 137
 138 class RepomanOptionParser(optparse.OptionParser):
 139         """Add the on_tail function, ruby has it, optionParser should too
 140         """
 141
 142         def __init__(self, *args, **kwargs):
 143                 optparse.OptionParser.__init__(self, *args, **kwargs)
 144                 self.tail = ""
 145
 146         def on_tail(self, description):
 147                 self.tail += description
 148
 149         def format_help(self, formatter=None):
 150                 result = optparse.OptionParser.format_help(self, formatter)
 151                 result += self.tail
 152                 return result
 153
 154
 155 def ParseArgs(argv, qahelp):
 156         """This function uses a customized optionParser to parse command line arguments for repoman
 157         Args:
 158           argv - a sequence of command line arguments
 159                 qahelp - a dict of qa warning to help message
 160         Returns:
 161           (opts, args), just like a call to parser.parse_args()
 162         """
 163
 164         if argv and isinstance(argv[0], bytes):
 165                 argv = [portage._unicode_decode(x) for x in argv]
 166
 167         modes = {
 168                 'commit' : 'Run a scan then commit changes',
 169                 'ci' : 'Run a scan then commit changes',
 170                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 171                 'full' : 'Scan directory tree and print all issues (not a summary)',
 172                 'help' : 'Show this screen',
 173                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 174                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 175                 'scan' : 'Scan directory tree for QA issues'
 176         }
 177
 178         mode_keys = list(modes)
 179         mode_keys.sort()
 180
 181         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 182         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 183         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 184         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 185         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 186
 187         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 188                 help='Request a confirmation before commiting')
 189
 190         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 191                 help='specify a commit message on the command line')
 192
 193         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 194                 help='specify a path to a file that contains a commit message')
 195
 196         parser.add_option('--digest',
 197                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 198                 help='Automatically update Manifest digests for modified files')
 199
 200         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 201                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 202
 203         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 204                 help='do not print unnecessary messages')
 205
 206         parser.add_option(
 207                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 208                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 209                 'regardless of modification if \'force\' is specified)')
 210
 211         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 212                 help='Commit with QA violations')
 213
 214         parser.add_option('--vcs', dest='vcs',
 215                 help='Force using specific VCS instead of autodetection')
 216
 217         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 218                 help='be very verbose in output', default=0)
 219
 220         parser.add_option('-V', '--version', dest='version', action='store_true',
 221                 help='show version info')
 222
 223         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 224                 default=False, help='forces the metadata.xml parse check to be carried out')
 225
 226         parser.add_option(
 227                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 228                 metavar="<y|n>",
 229                 help='only check packages that have uncommitted modifications')
 230
 231         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 232                 default=False, help='ignore arch-specific failures (where arch != host)')
 233
 234         parser.add_option("--ignore-default-opts",
 235                 action="store_true",
 236                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 237
 238         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 239                 default=False, help='ignore masked packages (not allowed with commit mode)')
 240
 241         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 242                 default=False, help='include dev profiles in dependency checks')
 243
 244         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 245                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 246
 247         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 248                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 249
 250         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 251                 help='specify which mode repoman will run in (default=full)')
 252
 253         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 254
 255         for k in mode_keys:
 256                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 257
 258         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 259
 260         sorted_qa = list(qahelp)
 261         sorted_qa.sort()
 262         for k in sorted_qa:
 263                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 264
 265         opts, args = parser.parse_args(argv[1:])
 266
 267         if not opts.ignore_default_opts:
 268                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 269                 if default_opts:
 270                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 271
 272         if opts.mode == 'help':
 273                 parser.print_help(short=False)
 274
 275         for arg in args:
 276                 if arg in modes:
 277                         if not opts.mode:
 278                                 opts.mode = arg
 279                                 break
 280                 else:
 281                         parser.error("invalid mode: %s" % arg)
 282
 283         if not opts.mode:
 284                 opts.mode = 'full'
 285
 286         if opts.mode == 'ci':
 287                 opts.mode = 'commit'  # backwards compat shortcut
 288
 289         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 290                 if opts.ignore_masked:
 291                         parser.error('Commit mode and --ignore-masked are not compatible')
 292                 if opts.without_mask:
 293                         parser.error('Commit mode and --without-mask are not compatible')
 294
 295         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 296         for val in range(opts.verbosity):
 297                 logger = logging.getLogger()
 298                 logger.setLevel(logger.getEffectiveLevel() - 10)
 299
 300         for val in range(opts.quiet):
 301                 logger = logging.getLogger()
 302                 logger.setLevel(logger.getEffectiveLevel() + 10)
 303
 304         return (opts, args)
 305
 306 qahelp={
 307         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 308         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 309         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 310         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 311         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 312         "changelog.missing":"Missing ChangeLog files",
 313         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 314         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 315         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 316         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 317         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 318         "file.size":"Files in the files directory must be under 20 KiB",
 319         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 320         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 321         "file.UTF8":"File is not UTF8 compliant",
 322         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 323         "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it",
 324         "inherit.unused":"Ebuild inherits an eclass but does not use it",
 325         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 326         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 327         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 328         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 329         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 330         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 331         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 332         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 333         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 334         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 335         "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 336         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 337         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 338         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 339         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 340         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 341         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 342         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 343         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 344         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 345         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 346         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 347         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 348         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 349         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 350         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 351         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 352         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 353         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 354         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 355         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 356         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 357         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 358         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 359         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 360         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 361         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 362         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 363         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 364         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 365         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 366         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 367         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 368         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 369         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 370         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 371         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 372         "variable.readonly":"Assigning a readonly variable",
 373         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 374         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 375         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 376         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 377         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 378         "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
 379         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 380         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 381         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 382         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 383         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 384         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 385         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 386         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 387         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 388         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 389         "ebuild.badheader":"This ebuild has a malformed header",
 390         "manifest.bad":"Manifest has missing or incorrect digests",
 391         "metadata.missing":"Missing metadata.xml files",
 392         "metadata.bad":"Bad metadata.xml files",
 393         "metadata.warning":"Warnings in metadata.xml files",
 394         "portage.internal":"The ebuild uses an internal Portage function",
 395         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 396         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 397         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 398 }
 399
 400 qacats = list(qahelp)
 401 qacats.sort()
 402
 403 qawarnings = set((
 404 "changelog.missing",
 405 "changelog.notadded",
 406 "dependency.unknown",
 407 "digest.assumed",
 408 "digest.unused",
 409 "ebuild.notadded",
 410 "ebuild.nesteddie",
 411 "desktop.invalid",
 412 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 413 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 414 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 415 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 416 "DESCRIPTION.toolong",
 417 "EAPI.deprecated",
 418 "HOMEPAGE.virtual",
 419 "LICENSE.virtual",
 420 "KEYWORDS.dropped",
 421 "KEYWORDS.stupid",
 422 "KEYWORDS.missing",
 423 "IUSE.undefined",
 424 "PDEPEND.suspect",
 425 "RDEPEND.implicit",
 426 "RDEPEND.suspect",
 427 "RESTRICT.invalid",
 428 "ebuild.minorsyn",
 429 "ebuild.badheader",
 430 "ebuild.patches",
 431 "file.size",
 432 "inherit.unused",
 433 "inherit.deprecated",
 434 "java.eclassesnotused",
 435 "wxwidgets.eclassnotused",
 436 "metadata.warning",
 437 "portage.internal",
 438 "usage.obsolete",
 439 "upstream.workaround",
 440 "LIVEVCS.stable",
 441 "LIVEVCS.unmasked",
 442 ))
 443
 444 if portage.const._ENABLE_INHERIT_CHECK:
 445         # This is experimental, so it's non-fatal.
 446         qawarnings.add("inherit.missing")
 447
 448 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 449
 450 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 451 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 452 allvars.update(Package.metadata_keys)
 453 allvars = sorted(allvars)
 454 commitmessage=None
 455 for x in missingvars:
 456         x += ".missing"
 457         if x not in qacats:
 458                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 459                 qacats.append(x)
 460                 qawarnings.add(x)
 461
 462 valid_restrict = frozenset(["binchecks", "bindist",
 463         "fetch", "installsources", "mirror",
 464         "primaryuri", "strip", "test", "userpriv"])
 465
 466 live_eclasses = frozenset([
 467         "bzr",
 468         "cvs",
 469         "darcs",
 470         "git",
 471         "git-2",
 472         "mercurial",
 473         "subversion",
 474         "tla",
 475 ])
 476
 477 suspect_rdepend = frozenset([
 478         "app-arch/cabextract",
 479         "app-arch/rpm2targz",
 480         "app-doc/doxygen",
 481         "dev-lang/nasm",
 482         "dev-lang/swig",
 483         "dev-lang/yasm",
 484         "dev-perl/extutils-pkgconfig",
 485         "dev-util/byacc",
 486         "dev-util/cmake",
 487         "dev-util/ftjam",
 488         "dev-util/gperf",
 489         "dev-util/gtk-doc",
 490         "dev-util/gtk-doc-am",
 491         "dev-util/intltool",
 492         "dev-util/jam",
 493         "dev-util/pkgconfig",
 494         "dev-util/scons",
 495         "dev-util/unifdef",
 496         "dev-util/yacc",
 497         "media-gfx/ebdftopcf",
 498         "sys-apps/help2man",
 499         "sys-devel/autoconf",
 500         "sys-devel/automake",
 501         "sys-devel/bin86",
 502         "sys-devel/bison",
 503         "sys-devel/dev86",
 504         "sys-devel/flex",
 505         "sys-devel/m4",
 506         "sys-devel/pmake",
 507         "virtual/linux-sources",
 508         "x11-misc/bdftopcf",
 509         "x11-misc/imake",
 510 ])
 511
 512 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 513 # force refetch if the local copy creation time is older than this
 514 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 515
 516 # file.executable
 517 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 518
 519 options, arguments = ParseArgs(sys.argv, qahelp)
 520
 521 if options.version:
 522         print("Portage", portage.VERSION)
 523         sys.exit(0)
 524
 525 # Set this to False when an extraordinary issue (generally
 526 # something other than a QA issue) makes it impossible to
 527 # commit (like if Manifest generation fails).
 528 can_force = True
 529
 530 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 531 if portdir is None:
 532         sys.exit(1)
 533
 534 myreporoot = os.path.basename(portdir_overlay)
 535 myreporoot += mydir[len(portdir_overlay):]
 536
 537 if options.vcs:
 538         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 539                 vcs = options.vcs
 540         else:
 541                 vcs = None
 542 else:
 543         vcses = utilities.FindVCS()
 544         if len(vcses) > 1:
 545                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 546                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 547                 sys.exit(1)
 548         elif vcses:
 549                 vcs = vcses[0]
 550         else:
 551                 vcs = None
 552
 553 if options.if_modified == "y" and vcs is None:
 554         logging.info("Not in a version controlled repository; "
 555                 "disabling --if-modified.")
 556         options.if_modified = "n"
 557
 558 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 559 vcs_preserves_mtime = vcs in ('cvs',)
 560
 561 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 562 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 563 if vcs_global_opts is None:
 564         if vcs in ('cvs', 'svn'):
 565                 vcs_global_opts = "-q"
 566         else:
 567                 vcs_global_opts = ""
 568 vcs_global_opts = vcs_global_opts.split()
 569
 570 if options.mode == 'commit' and not options.pretend and not vcs:
 571         logging.info("Not in a version controlled repository; enabling pretend mode.")
 572         options.pretend = True
 573
 574 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 575 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 576         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 577         portage._shell_quote(portdir_overlay))
 578 # We have to call the config constructor again so
 579 # that config.repositories is initialized correctly.
 580 repoman_settings = portage.config(config_root=config_root, local_config=False,
 581         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 582
 583 root = repoman_settings['EROOT']
 584 trees = {
 585         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 586 }
 587 portdb = trees[root]['porttree'].dbapi
 588
 589 # Constrain dependency resolution to the master(s)
 590 # that are specified in layout.conf.
 591 repodir = os.path.realpath(portdir_overlay)
 592 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 593 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 594 portdir = portdb.porttrees[0]
 595
 596 if repo_config.allow_provide_virtual:
 597         qawarnings.add("virtual.oldstyle")
 598
 599 if repo_config.sign_commit:
 600         if vcs == 'git':
 601                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 602                 # the commit arguments. If key_id is unspecified, then it must be
 603                 # configured by `git config user.signingkey key_id`.
 604                 vcs_local_opts.append("--gpg-sign")
 605
 606 # In order to disable manifest signatures, repos may set
 607 # "sign-manifests = false" in metadata/layout.conf. This
 608 # can be used to prevent merge conflicts like those that
 609 # thin-manifests is designed to prevent.
 610 sign_manifests = "sign" in repoman_settings.features and \
 611         repo_config.sign_manifest
 612
 613 manifest_hashes = repo_config.manifest_hashes
 614 if manifest_hashes is None:
 615         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 616
 617 if options.mode in ("commit", "fix", "manifest"):
 618         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 619                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 620                 "metadata/layout.conf does not contain the '%s' hash which "
 621                 "is required by this portage version. You will have to "
 622                 "upgrade portage if you want to generate valid manifests for "
 623                 "this repository.") % \
 624                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 625                 for line in textwrap.wrap(msg, 70):
 626                         logging.error(line)
 627                 sys.exit(1)
 628
 629         unsupported_hashes = manifest_hashes.difference(
 630                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 631         if unsupported_hashes:
 632                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 633                 "metadata/layout.conf contains one or more hash types '%s' "
 634                 "which are not supported by this portage version. You will "
 635                 "have to upgrade portage if you want to generate valid "
 636                 "manifests for this repository.") % \
 637                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 638                 for line in textwrap.wrap(msg, 70):
 639                         logging.error(line)
 640                 sys.exit(1)
 641
 642 if "commit" == options.mode and \
 643         repo_config.name == "gentoo" and \
 644         "RMD160" in manifest_hashes and \
 645         "RMD160" not in portage.checksum.hashorigin_map:
 646         msg = "Please install " \
 647         "pycrypto or enable python's ssl USE flag in order " \
 648         "to enable RMD160 hash support. See bug #198398 for " \
 649         "more information."
 650         prefix = bad(" * ")
 651         for line in textwrap.wrap(msg, 70):
 652                 print(prefix + line)
 653         sys.exit(1)
 654
 655 if options.echangelog is None and repo_config.update_changelog:
 656         options.echangelog = 'y'
 657
 658 if vcs is None:
 659         options.echangelog = 'n'
 660
 661 # The --echangelog option causes automatic ChangeLog generation,
 662 # which invalidates changelog.ebuildadded and changelog.missing
 663 # checks.
 664 # Note: Some don't use ChangeLogs in distributed SCMs.
 665 # It will be generated on server side from scm log,
 666 # before package moves to the rsync server.
 667 # This is needed because they try to avoid merge collisions.
 668 # Gentoo's Council decided to always use the ChangeLog file.
 669 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 670 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 671
 672 if 'digest' in repoman_settings.features and options.digest != 'n':
 673         options.digest = 'y'
 674
 675 logging.debug("vcs: %s" % (vcs,))
 676 logging.debug("repo config: %s" % (repo_config,))
 677 logging.debug("options: %s" % (options,))
 678
 679 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 680 # profile-specific config constructor calls.
 681 env = os.environ.copy()
 682 env['PORTDIR'] = portdir
 683 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 684
 685 logging.info('Setting paths:')
 686 logging.info('PORTDIR = "' + portdir + '"')
 687 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 688
 689 # It's confusing if these warnings are displayed without the user
 690 # being told which profile they come from, so disable them.
 691 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 692
 693 categories = []
 694 for path in repo_config.eclass_db.porttrees:
 695         categories.extend(portage.util.grabfile(
 696                 os.path.join(path, 'profiles', 'categories')))
 697 repoman_settings.categories = frozenset(
 698         portage.util.stack_lists([categories], incremental=1))
 699 categories = repoman_settings.categories
 700
 701 portdb.settings = repoman_settings
 702 root_config = RootConfig(repoman_settings, trees[root], None)
 703 # We really only need to cache the metadata that's necessary for visibility
 704 # filtering. Anything else can be discarded to reduce memory consumption.
 705 portdb._aux_cache_keys.clear()
 706 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 707
 708 reposplit = myreporoot.split(os.path.sep)
 709 repolevel = len(reposplit)
 710
 711 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 712 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 713 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 714 if options.mode == 'commit' and repolevel not in [1,2,3]:
 715         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 716         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 717         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 718         print(red("***"))
 719         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 720
 721 # Make startdir relative to the canonical repodir, so that we can pass
 722 # it to digestgen and it won't have to be canonicalized again.
 723 if repolevel == 1:
 724         startdir = repodir
 725 else:
 726         startdir = normalize_path(mydir)
 727         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 728
 729 def caterror(mycat):
 730         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 731
 732 class ProfileDesc(object):
 733         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 734         def __init__(self, arch, status, sub_path, tree_path):
 735                 self.arch = arch
 736                 self.status = status
 737                 if sub_path:
 738                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 739                 self.sub_path = sub_path
 740                 self.tree_path = tree_path
 741                 if tree_path:
 742                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 743                 else:
 744                         self.abs_path = tree_path
 745
 746         def __str__(self):
 747                 if self.sub_path:
 748                         return self.sub_path
 749                 return 'empty profile'
 750
 751 profile_list = []
 752 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 753
 754 # get lists of valid keywords, licenses, and use
 755 kwlist = set()
 756 liclist = set()
 757 uselist = set()
 758 global_pmasklines = []
 759
 760 for path in portdb.porttrees:
 761         try:
 762                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 763         except OSError:
 764                 pass
 765         kwlist.update(portage.grabfile(os.path.join(path,
 766                 "profiles", "arch.list")))
 767
 768         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 769         for x in use_desc:
 770                 x = x.split()
 771                 if x:
 772                         uselist.add(x[0])
 773
 774         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 775         try:
 776                 expand_list = os.listdir(expand_desc_dir)
 777         except OSError:
 778                 pass
 779         else:
 780                 for fn in expand_list:
 781                         if not fn[-5:] == '.desc':
 782                                 continue
 783                         use_prefix = fn[:-5].lower() + '_'
 784                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 785                                 x = x.split()
 786                                 if x:
 787                                         uselist.add(use_prefix + x[0])
 788
 789         global_pmasklines.append(portage.util.grabfile_package(
 790                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 791
 792         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 793         try:
 794                 desc_file = io.open(_unicode_encode(desc_path,
 795                         encoding=_encodings['fs'], errors='strict'),
 796                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 797         except EnvironmentError:
 798                 pass
 799         else:
 800                 for i, x in enumerate(desc_file):
 801                         if x[0] == "#":
 802                                 continue
 803                         arch = x.split()
 804                         if len(arch) == 0:
 805                                 continue
 806                         if len(arch) != 3:
 807                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 808                                         desc_path + " line %d" % (i+1, ))
 809                         elif arch[0] not in kwlist:
 810                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 811                                         desc_path + " line %d" % (i+1, ))
 812                         elif arch[2] not in valid_profile_types:
 813                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 814                                         desc_path + " line %d" % (i+1, ))
 815                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 816                         if not os.path.isdir(profile_desc.abs_path):
 817                                 logging.error(
 818                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 819                                         arch[2], arch[1], arch[0], desc_path, i+1)
 820                                 continue
 821                         if os.path.exists(
 822                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 823                                 continue
 824                         profile_list.append(profile_desc)
 825                 desc_file.close()
 826
 827 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 828 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 829
 830 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 831 global_pmaskdict = {}
 832 for x in global_pmasklines:
 833         global_pmaskdict.setdefault(x.cp, []).append(x)
 834 del global_pmasklines
 835
 836 def has_global_mask(pkg):
 837         mask_atoms = global_pmaskdict.get(pkg.cp)
 838         if mask_atoms:
 839                 pkg_list = [pkg]
 840                 for x in mask_atoms:
 841                         if portage.dep.match_from_list(x, pkg_list):
 842                                 return x
 843         return None
 844
 845 # Ensure that profile sub_path attributes are unique. Process in reverse order
 846 # so that profiles with duplicate sub_path from overlays will override
 847 # profiles with the same sub_path from parent repos.
 848 profiles = {}
 849 profile_list.reverse()
 850 profile_sub_paths = set()
 851 for prof in profile_list:
 852         if prof.sub_path in profile_sub_paths:
 853                 continue
 854         profile_sub_paths.add(prof.sub_path)
 855         profiles.setdefault(prof.arch, []).append(prof)
 856
 857 # Use an empty profile for checking dependencies of
 858 # packages that have empty KEYWORDS.
 859 prof = ProfileDesc('**', 'stable', '', '')
 860 profiles.setdefault(prof.arch, []).append(prof)
 861
 862 for x in repoman_settings.archlist():
 863         if x[0] == "~":
 864                 continue
 865         if x not in profiles:
 866                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 867                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 868                 print(red("up with the "+x+" team."))
 869                 print()
 870
 871 if not liclist:
 872         logging.fatal("Couldn't find licenses?")
 873         sys.exit(1)
 874
 875 if not kwlist:
 876         logging.fatal("Couldn't read KEYWORDS from arch.list")
 877         sys.exit(1)
 878
 879 if not uselist:
 880         logging.fatal("Couldn't find use.desc?")
 881         sys.exit(1)
 882
 883 scanlist=[]
 884 if repolevel==2:
 885         #we are inside a category directory
 886         catdir=reposplit[-1]
 887         if catdir not in categories:
 888                 caterror(catdir)
 889         mydirlist=os.listdir(startdir)
 890         for x in mydirlist:
 891                 if x == "CVS" or x.startswith("."):
 892                         continue
 893                 if os.path.isdir(startdir+"/"+x):
 894                         scanlist.append(catdir+"/"+x)
 895         repo_subdir = catdir + os.sep
 896 elif repolevel==1:
 897         for x in categories:
 898                 if not os.path.isdir(startdir+"/"+x):
 899                         continue
 900                 for y in os.listdir(startdir+"/"+x):
 901                         if y == "CVS" or y.startswith("."):
 902                                 continue
 903                         if os.path.isdir(startdir+"/"+x+"/"+y):
 904                                 scanlist.append(x+"/"+y)
 905         repo_subdir = ""
 906 elif repolevel==3:
 907         catdir = reposplit[-2]
 908         if catdir not in categories:
 909                 caterror(catdir)
 910         scanlist.append(catdir+"/"+reposplit[-1])
 911         repo_subdir = scanlist[-1] + os.sep
 912 else:
 913         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 914                 ' from the current working directory'
 915         logging.critical(msg)
 916         sys.exit(1)
 917
 918 repo_subdir_len = len(repo_subdir)
 919 scanlist.sort()
 920
 921 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 922
 923 def vcs_files_to_cps(vcs_file_iter):
 924         """
 925         Iterate over the given modified file paths returned from the vcs,
 926         and return a frozenset containing category/pn strings for each
 927         modified package.
 928         """
 929
 930         modified_cps = []
 931
 932         if repolevel == 3:
 933                 if reposplit[-2] in categories and \
 934                         next(vcs_file_iter, None) is not None:
 935                         modified_cps.append("/".join(reposplit[-2:]))
 936
 937         elif repolevel == 2:
 938                 category = reposplit[-1]
 939                 if category in categories:
 940                         for filename in vcs_file_iter:
 941                                 f_split = filename.split(os.sep)
 942                                 # ['.', pn,...]
 943                                 if len(f_split) > 2:
 944                                         modified_cps.append(category + "/" + f_split[1])
 945
 946         else:
 947                 # repolevel == 1
 948                 for filename in vcs_file_iter:
 949                         f_split = filename.split(os.sep)
 950                         # ['.', category, pn,...]
 951                         if len(f_split) > 3 and f_split[1] in categories:
 952                                 modified_cps.append("/".join(f_split[1:3]))
 953
 954         return frozenset(modified_cps)
 955
 956 def git_supports_gpg_sign():
 957         status, cmd_output = \
 958                 subprocess_getstatusoutput("git --version")
 959         cmd_output = cmd_output.split()
 960         if cmd_output:
 961                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
 962                 if version is not None:
 963                         version = [int(x) for x in version.groups()[1:]]
 964                         if version[0] > 1 or \
 965                                 (version[0] == 1 and version[1] > 7) or \
 966                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
 967                                 return True
 968         return False
 969
 970 def dev_keywords(profiles):
 971         """
 972         Create a set of KEYWORDS values that exist in 'dev'
 973         profiles. These are used
 974         to trigger a message notifying the user when they might
 975         want to add the --include-dev option.
 976         """
 977         type_arch_map = {}
 978         for arch, arch_profiles in profiles.items():
 979                 for prof in arch_profiles:
 980                         arch_set = type_arch_map.get(prof.status)
 981                         if arch_set is None:
 982                                 arch_set = set()
 983                                 type_arch_map[prof.status] = arch_set
 984                         arch_set.add(arch)
 985
 986         dev_keywords = type_arch_map.get('dev', set())
 987         dev_keywords.update(['~' + arch for arch in dev_keywords])
 988         return frozenset(dev_keywords)
 989
 990 dev_keywords = dev_keywords(profiles)
 991
 992 stats={}
 993 fails={}
 994
 995 # provided by the desktop-file-utils package
 996 desktop_file_validate = find_binary("desktop-file-validate")
 997 desktop_pattern = re.compile(r'.*\.desktop$')
 998
 999 for x in qacats:
1000         stats[x]=0
1001         fails[x]=[]
1002
1003 xmllint_capable = False
1004 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1005
1006 def parsedate(s):
1007         """Parse a RFC 822 date and time string.
1008         This is required for python3 compatibility, since the
1009         rfc822.parsedate() function is not available."""
1010
1011         s_split = []
1012         for x in s.upper().split():
1013                 for y in x.split(','):
1014                         if y:
1015                                 s_split.append(y)
1016
1017         if len(s_split) != 6:
1018                 return None
1019
1020         # %a, %d %b %Y %H:%M:%S %Z
1021         a, d, b, Y, H_M_S, Z = s_split
1022
1023         # Convert month to integer, since strptime %w is locale-dependent.
1024         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1025                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1026         m = month_map.get(b)
1027         if m is None:
1028                 return None
1029         m = str(m).rjust(2, '0')
1030
1031         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1032
1033 def fetch_metadata_dtd():
1034         """
1035         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1036         metadata_dtd_ctime_interval.
1037         @rtype: bool
1038         @return: True if successful, otherwise False
1039         """
1040
1041         must_fetch = True
1042         metadata_dtd_st = None
1043         current_time = int(time.time())
1044         try:
1045                 metadata_dtd_st = os.stat(metadata_dtd)
1046         except EnvironmentError as e:
1047                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1048                         raise
1049                 del e
1050         else:
1051                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1052                 if abs(current_time - metadata_dtd_st.st_ctime) \
1053                         < metadata_dtd_ctime_interval:
1054                         must_fetch = False
1055
1056         if must_fetch:
1057                 print()
1058                 print(green("***") + " the local copy of metadata.dtd " + \
1059                         "needs to be refetched, doing that now")
1060                 print()
1061                 try:
1062                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1063                         msg_info = url_f.info()
1064                         last_modified = msg_info.get('last-modified')
1065                         if last_modified is not None:
1066                                 last_modified = parsedate(last_modified)
1067                                 if last_modified is not None:
1068                                         last_modified = calendar.timegm(last_modified)
1069
1070                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1071                         try:
1072                                 local_f = open(metadata_dtd_tmp, mode='wb')
1073                                 local_f.write(url_f.read())
1074                                 local_f.close()
1075                                 if last_modified is not None:
1076                                         try:
1077                                                 os.utime(metadata_dtd_tmp,
1078                                                         (int(last_modified), int(last_modified)))
1079                                         except OSError:
1080                                                 # This fails on some odd non-unix-like filesystems.
1081                                                 # We don't really need the mtime to be preserved
1082                                                 # anyway here (currently we use ctime to trigger
1083                                                 # fetch), so just ignore it.
1084                                                 pass
1085                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1086                         finally:
1087                                 try:
1088                                         os.unlink(metadata_dtd_tmp)
1089                                 except OSError:
1090                                         pass
1091
1092                         url_f.close()
1093
1094                 except EnvironmentError as e:
1095                         print()
1096                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1097                         print(red("!!!")+" exception '%s' though." % (e,))
1098                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1099                         return False
1100
1101         return True
1102
1103 if options.mode == "manifest":
1104         pass
1105 elif not find_binary('xmllint'):
1106         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1107         if options.xml_parse or repolevel==3:
1108                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1109                 sys.exit(1)
1110 else:
1111         if not fetch_metadata_dtd():
1112                 sys.exit(1)
1113         #this can be problematic if xmllint changes their output
1114         xmllint_capable=True
1115
1116 if options.mode == 'commit' and vcs:
1117         utilities.detect_vcs_conflicts(options, vcs)
1118
1119 if options.mode == "manifest":
1120         pass
1121 elif options.pretend:
1122         print(green("\nRepoMan does a once-over of the neighborhood..."))
1123 else:
1124         print(green("\nRepoMan scours the neighborhood..."))
1125
1126 new_ebuilds = set()
1127 modified_ebuilds = set()
1128 modified_changelogs = set()
1129 mychanged = []
1130 mynew = []
1131 myremoved = []
1132
1133 if vcs == "cvs":
1134         mycvstree = cvstree.getentries("./", recursive=1)
1135         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1136         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1137         if options.if_modified == "y":
1138                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1139
1140 elif vcs == "svn":
1141         with os.popen("svn status") as f:
1142                 svnstatus = f.readlines()
1143         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1144         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1145         if options.if_modified == "y":
1146                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1147
1148 elif vcs == "git":
1149         with os.popen("git diff-index --name-only "
1150                 "--relative --diff-filter=M HEAD") as f:
1151                 mychanged = f.readlines()
1152         mychanged = ["./" + elem[:-1] for elem in mychanged]
1153
1154         with os.popen("git diff-index --name-only "
1155                 "--relative --diff-filter=A HEAD") as f:
1156                 mynew = f.readlines()
1157         mynew = ["./" + elem[:-1] for elem in mynew]
1158         if options.if_modified == "y":
1159                 with os.popen("git diff-index --name-only "
1160                         "--relative --diff-filter=D HEAD") as f:
1161                         myremoved = f.readlines()
1162                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1163
1164 elif vcs == "bzr":
1165         with os.popen("bzr status -S .") as f:
1166                 bzrstatus = f.readlines()
1167         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1168         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1169         if options.if_modified == "y":
1170                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1171
1172 elif vcs == "hg":
1173         with os.popen("hg status --no-status --modified .") as f:
1174                 mychanged = f.readlines()
1175         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1176         mynew = os.popen("hg status --no-status --added .").readlines()
1177         mynew = ["./" + elem.rstrip() for elem in mynew]
1178         if options.if_modified == "y":
1179                 with os.popen("hg status --no-status --removed .") as f:
1180                         myremoved = f.readlines()
1181                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1182
1183 if vcs:
1184         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1185         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1186         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1187                 if os.path.basename(x) == "ChangeLog")
1188
1189 def vcs_new_changed(relative_path):
1190         for x in chain(mychanged, mynew):
1191                 if x == relative_path:
1192                         return True
1193         return False
1194
1195 have_pmasked = False
1196 have_dev_keywords = False
1197 dofail = 0
1198
1199 # NOTE: match-all caches are not shared due to potential
1200 # differences between profiles in _get_implicit_iuse.
1201 arch_caches={}
1202 arch_xmatch_caches = {}
1203 shared_xmatch_caches = {"cp-list":{}}
1204
1205 # Disable the "ebuild.notadded" check when not in commit mode and
1206 # running `svn status` in every package dir will be too expensive.
1207
1208 check_ebuild_notadded = not \
1209         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1210
1211 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1212 thirdpartymirrors = []
1213 for v in repoman_settings.thirdpartymirrors().values():
1214         for v in v:
1215                 if not v.endswith("/"):
1216                         v += "/"
1217                 thirdpartymirrors.append(v)
1218
1219 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1220         """
1221         Implements doctype() as required to avoid deprecation warnings with
1222         >=python-2.7.
1223         """
1224         def doctype(self, name, pubid, system):
1225                 pass
1226
1227 try:
1228         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1229 except (EnvironmentError, ParseError, PermissionDenied) as e:
1230         err(str(e))
1231 except FileNotFound:
1232         # TODO: Download as we do for metadata.dtd, but add a way to
1233         # disable for non-gentoo repoman users who may not have herds.
1234         herd_base = None
1235
1236 effective_scanlist = scanlist
1237 if options.if_modified == "y":
1238         effective_scanlist = sorted(vcs_files_to_cps(
1239                 chain(mychanged, mynew, myremoved)))
1240
1241 for x in effective_scanlist:
1242         #ebuilds and digests added to cvs respectively.
1243         logging.info("checking package %s" % x)
1244         # save memory by discarding xmatch caches from previous package(s)
1245         arch_xmatch_caches.clear()
1246         eadded=[]
1247         catdir,pkgdir=x.split("/")
1248         checkdir=repodir+"/"+x
1249         checkdir_relative = ""
1250         if repolevel < 3:
1251                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1252         if repolevel < 2:
1253                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1254         checkdir_relative = os.path.join(".", checkdir_relative)
1255         generated_manifest = False
1256
1257         if options.mode == "manifest" or \
1258           (options.mode != 'manifest-check' and options.digest == 'y') or \
1259           options.mode in ('commit', 'fix') and not options.pretend:
1260                 auto_assumed = set()
1261                 fetchlist_dict = portage.FetchlistDict(checkdir,
1262                         repoman_settings, portdb)
1263                 if options.mode == 'manifest' and options.force:
1264                         portage._doebuild_manifest_exempt_depend += 1
1265                         try:
1266                                 distdir = repoman_settings['DISTDIR']
1267                                 mf = repoman_settings.repositories.get_repo_for_location(
1268                                         os.path.dirname(os.path.dirname(checkdir)))
1269                                 mf = mf.load_manifest(checkdir, distdir,
1270                                         fetchlist_dict=fetchlist_dict)
1271                                 mf.create(requiredDistfiles=None,
1272                                         assumeDistHashesAlways=True)
1273                                 for distfiles in fetchlist_dict.values():
1274                                         for distfile in distfiles:
1275                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1276                                                         mf.fhashdict['DIST'].pop(distfile, None)
1277                                                 else:
1278                                                         auto_assumed.add(distfile)
1279                                 mf.write()
1280                         finally:
1281                                 portage._doebuild_manifest_exempt_depend -= 1
1282
1283                 repoman_settings["O"] = checkdir
1284                 try:
1285                         generated_manifest = digestgen(
1286                                 mysettings=repoman_settings, myportdb=portdb)
1287                 except portage.exception.PermissionDenied as e:
1288                         generated_manifest = False
1289                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1290                                 level=logging.ERROR, noiselevel=-1)
1291
1292                 if not generated_manifest:
1293                         print("Unable to generate manifest.")
1294                         dofail = 1
1295
1296                 if options.mode == "manifest":
1297                         if not dofail and options.force and auto_assumed and \
1298                                 'assume-digests' in repoman_settings.features:
1299                                 # Show which digests were assumed despite the --force option
1300                                 # being given. This output will already have been shown by
1301                                 # digestgen() if assume-digests is not enabled, so only show
1302                                 # it here if assume-digests is enabled.
1303                                 pkgs = list(fetchlist_dict)
1304                                 pkgs.sort()
1305                                 portage.writemsg_stdout("  digest.assumed" + \
1306                                         portage.output.colorize("WARN",
1307                                         str(len(auto_assumed)).rjust(18)) + "\n")
1308                                 for cpv in pkgs:
1309                                         fetchmap = fetchlist_dict[cpv]
1310                                         pf = portage.catsplit(cpv)[1]
1311                                         for distfile in sorted(fetchmap):
1312                                                 if distfile in auto_assumed:
1313                                                         portage.writemsg_stdout(
1314                                                                 "   %s::%s\n" % (pf, distfile))
1315                         continue
1316                 elif dofail:
1317                         sys.exit(1)
1318
1319         if not generated_manifest:
1320                 repoman_settings['O'] = checkdir
1321                 repoman_settings['PORTAGE_QUIET'] = '1'
1322                 if not portage.digestcheck([], repoman_settings, strict=1):
1323                         stats["manifest.bad"] += 1
1324                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1325                 repoman_settings.pop('PORTAGE_QUIET', None)
1326
1327         if options.mode == 'manifest-check':
1328                 continue
1329
1330         checkdirlist=os.listdir(checkdir)
1331         ebuildlist=[]
1332         pkgs = {}
1333         allvalid = True
1334         for y in checkdirlist:
1335                 if (y in no_exec or y.endswith(".ebuild")) and \
1336                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1337                                 stats["file.executable"] += 1
1338                                 fails["file.executable"].append(os.path.join(checkdir, y))
1339                 if y.endswith(".ebuild"):
1340                         pf = y[:-7]
1341                         ebuildlist.append(pf)
1342                         cpv = "%s/%s" % (catdir, pf)
1343                         try:
1344                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1345                         except KeyError:
1346                                 allvalid = False
1347                                 stats["ebuild.syntax"] += 1
1348                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1349                                 continue
1350                         except IOError:
1351                                 allvalid = False
1352                                 stats["ebuild.output"] += 1
1353                                 fails["ebuild.output"].append(os.path.join(x, y))
1354                                 continue
1355                         if not portage.eapi_is_supported(myaux["EAPI"]):
1356                                 allvalid = False
1357                                 stats["EAPI.unsupported"] += 1
1358                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1359                                 continue
1360                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1361                                 root_config=root_config, type_name="ebuild")
1362
1363         slot_keywords = {}
1364
1365         if len(pkgs) != len(ebuildlist):
1366                 # If we can't access all the metadata then it's totally unsafe to
1367                 # commit since there's no way to generate a correct Manifest.
1368                 # Do not try to do any more QA checks on this package since missing
1369                 # metadata leads to false positives for several checks, and false
1370                 # positives confuse users.
1371                 can_force = False
1372                 continue
1373
1374         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1375         ebuildlist = sorted(pkgs.values())
1376         ebuildlist = [pkg.pf for pkg in ebuildlist]
1377
1378         for y in checkdirlist:
1379                 m = disallowed_filename_chars_re.search(y.strip(os.sep))
1380                 if m is not None:
1381                         y_relative = os.path.join(checkdir_relative, y)
1382                         if vcs is not None and not vcs_new_changed(y_relative):
1383                                 # If the file isn't in the VCS new or changed set, then
1384                                 # assume that it's an irrelevant temporary file (Manifest
1385                                 # entries are not generated for file names containing
1386                                 # prohibited characters). See bug #406877.
1387                                 m = None
1388                 if m is not None:
1389                         stats["file.name"] += 1
1390                         fails["file.name"].append("%s/%s: char '%s'" % \
1391                                 (checkdir, y, m.group(0)))
1392
1393                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1394                         continue
1395                 f = None
1396                 try:
1397                         line = 1
1398                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1399                                 encoding=_encodings['fs'], errors='strict'),
1400                                 mode='r', encoding=_encodings['repo.content'])
1401                         for l in f:
1402                                 line +=1
1403                 except UnicodeDecodeError as ue:
1404                         stats["file.UTF8"] += 1
1405                         s = ue.object[:ue.start]
1406                         l2 = s.count("\n")
1407                         line += l2
1408                         if l2 != 0:
1409                                 s = s[s.rfind("\n") + 1:]
1410                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1411                 finally:
1412                         if f is not None:
1413                                 f.close()
1414
1415         if vcs in ("git", "hg") and check_ebuild_notadded:
1416                 if vcs == "git":
1417                         myf = os.popen("git ls-files --others %s" % \
1418                                 (portage._shell_quote(checkdir_relative),))
1419                 if vcs == "hg":
1420                         myf = os.popen("hg status --no-status --unknown %s" % \
1421                                 (portage._shell_quote(checkdir_relative),))
1422                 for l in myf:
1423                         if l[:-1][-7:] == ".ebuild":
1424                                 stats["ebuild.notadded"] += 1
1425                                 fails["ebuild.notadded"].append(
1426                                         os.path.join(x, os.path.basename(l[:-1])))
1427                 myf.close()
1428
1429         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1430                 try:
1431                         if vcs == "cvs":
1432                                 myf=open(checkdir+"/CVS/Entries","r")
1433                         if vcs == "svn":
1434                                 myf = os.popen("svn status --depth=files --verbose " + checkdir)
1435                         if vcs == "bzr":
1436                                 myf = os.popen("bzr ls -v --kind=file " + checkdir)
1437                         myl = myf.readlines()
1438                         myf.close()
1439                         for l in myl:
1440                                 if vcs == "cvs":
1441                                         if l[0]!="/":
1442                                                 continue
1443                                         splitl=l[1:].split("/")
1444                                         if not len(splitl):
1445                                                 continue
1446                                         if splitl[0][-7:]==".ebuild":
1447                                                 eadded.append(splitl[0][:-7])
1448                                 if vcs == "svn":
1449                                         if l[:1] == "?":
1450                                                 continue
1451                                         if l[:7] == '      >':
1452                                                 # tree conflict, new in subversion 1.6
1453                                                 continue
1454                                         l = l.split()[-1]
1455                                         if l[-7:] == ".ebuild":
1456                                                 eadded.append(os.path.basename(l[:-7]))
1457                                 if vcs == "bzr":
1458                                         if l[1:2] == "?":
1459                                                 continue
1460                                         l = l.split()[-1]
1461                                         if l[-7:] == ".ebuild":
1462                                                 eadded.append(os.path.basename(l[:-7]))
1463                         if vcs == "svn":
1464                                 myf = os.popen("svn status " + checkdir)
1465                                 myl=myf.readlines()
1466                                 myf.close()
1467                                 for l in myl:
1468                                         if l[0] == "A":
1469                                                 l = l.rstrip().split(' ')[-1]
1470                                                 if l[-7:] == ".ebuild":
1471                                                         eadded.append(os.path.basename(l[:-7]))
1472                 except IOError:
1473                         if vcs == "cvs":
1474                                 stats["CVS/Entries.IO_error"] += 1
1475                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1476                         else:
1477                                 raise
1478                         continue
1479
1480         mf = repoman_settings.repositories.get_repo_for_location(
1481                 os.path.dirname(os.path.dirname(checkdir)))
1482         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1483         mydigests=mf.getTypeDigests("DIST")
1484
1485         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1486         myfiles_all = []
1487         src_uri_error = False
1488         for mykey in fetchlist_dict:
1489                 try:
1490                         myfiles_all.extend(fetchlist_dict[mykey])
1491                 except portage.exception.InvalidDependString as e:
1492                         src_uri_error = True
1493                         try:
1494                                 portdb.aux_get(mykey, ["SRC_URI"])
1495                         except KeyError:
1496                                 # This will be reported as an "ebuild.syntax" error.
1497                                 pass
1498                         else:
1499                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1500                                 fails["SRC_URI.syntax"].append(
1501                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1502         del fetchlist_dict
1503         if not src_uri_error:
1504                 # This test can produce false positives if SRC_URI could not
1505                 # be parsed for one or more ebuilds. There's no point in
1506                 # producing a false error here since the root cause will
1507                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1508                 # or "ebuild.sytax".
1509                 myfiles_all = set(myfiles_all)
1510                 for entry in mydigests:
1511                         if entry not in myfiles_all:
1512                                 stats["digest.unused"] += 1
1513                                 fails["digest.unused"].append(checkdir+"::"+entry)
1514                 for entry in myfiles_all:
1515                         if entry not in mydigests:
1516                                 stats["digest.missing"] += 1
1517                                 fails["digest.missing"].append(checkdir+"::"+entry)
1518         del myfiles_all
1519
1520         if os.path.exists(checkdir+"/files"):
1521                 filesdirlist=os.listdir(checkdir+"/files")
1522
1523                 # recurse through files directory
1524                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1525                 while filesdirlist:
1526                         y = filesdirlist.pop(0)
1527                         relative_path = os.path.join(x, "files", y)
1528                         full_path = os.path.join(repodir, relative_path)
1529                         try:
1530                                 mystat = os.stat(full_path)
1531                         except OSError as oe:
1532                                 if oe.errno == 2:
1533                                         # don't worry about it.  it likely was removed via fix above.
1534                                         continue
1535                                 else:
1536                                         raise oe
1537                         if S_ISDIR(mystat.st_mode):
1538                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1539                                 if y == "CVS" or y == ".svn":
1540                                         continue
1541                                 for z in os.listdir(checkdir+"/files/"+y):
1542                                         if z == "CVS" or z == ".svn":
1543                                                 continue
1544                                         filesdirlist.append(y+"/"+z)
1545                         # Current policy is no files over 20 KiB, these are the checks. File size between
1546                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1547                         elif mystat.st_size > 61440:
1548                                 stats["file.size.fatal"] += 1
1549                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1550                         elif mystat.st_size > 20480:
1551                                 stats["file.size"] += 1
1552                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1553
1554                         m = disallowed_filename_chars_re.search(
1555                                 os.path.basename(y.rstrip(os.sep)))
1556                         if m is not None:
1557                                 y_relative = os.path.join(checkdir_relative, "files", y)
1558                                 if vcs is not None and not vcs_new_changed(y_relative):
1559                                         # If the file isn't in the VCS new or changed set, then
1560                                         # assume that it's an irrelevant temporary file (Manifest
1561                                         # entries are not generated for file names containing
1562                                         # prohibited characters). See bug #406877.
1563                                         m = None
1564                         if m is not None:
1565                                 stats["file.name"] += 1
1566                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1567                                         (checkdir, y, m.group(0)))
1568
1569                         if desktop_file_validate and desktop_pattern.match(y):
1570                                 cmd_output = validate_desktop_entry(full_path)
1571                                 if cmd_output:
1572                                         # Note: in the future we may want to grab the
1573                                         # warnings in addition to the errors. We're
1574                                         # just doing errors now since we don't want
1575                                         # to generate too much noise at first.
1576                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1577                                         for line in cmd_output:
1578                                                 error_match = error_re.match(line)
1579                                                 if error_match is None:
1580                                                         continue
1581                                                 stats["desktop.invalid"] += 1
1582                                                 fails["desktop.invalid"].append(
1583                                                         relative_path + ': %s' % error_match.group(1))
1584
1585         del mydigests
1586
1587         if check_changelog and "ChangeLog" not in checkdirlist:
1588                 stats["changelog.missing"]+=1
1589                 fails["changelog.missing"].append(x+"/ChangeLog")
1590
1591         musedict = {}
1592         #metadata.xml file check
1593         if "metadata.xml" not in checkdirlist:
1594                 stats["metadata.missing"]+=1
1595                 fails["metadata.missing"].append(x+"/metadata.xml")
1596         #metadata.xml parse check
1597         else:
1598                 metadata_bad = False
1599
1600                 # read metadata.xml into memory
1601                 try:
1602                         _metadata_xml = xml.etree.ElementTree.parse(
1603                                 os.path.join(checkdir, "metadata.xml"),
1604                                 parser=xml.etree.ElementTree.XMLParser(
1605                                         target=_MetadataTreeBuilder()))
1606                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1607                         metadata_bad = True
1608                         stats["metadata.bad"] += 1
1609                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1610                         del e
1611                 else:
1612                         # load USE flags from metadata.xml
1613                         try:
1614                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1615                         except portage.exception.ParseError as e:
1616                                 metadata_bad = True
1617                                 stats["metadata.bad"] += 1
1618                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1619
1620                         # Run other metadata.xml checkers
1621                         try:
1622                                 utilities.check_metadata(_metadata_xml, herd_base)
1623                         except (utilities.UnknownHerdsError, ) as e:
1624                                 metadata_bad = True
1625                                 stats["metadata.bad"] += 1
1626                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1627                                 del e
1628
1629                 #Only carry out if in package directory or check forced
1630                 if xmllint_capable and not metadata_bad:
1631                         # xmlint can produce garbage output even on success, so only dump
1632                         # the ouput when it fails.
1633                         st, out = subprocess_getstatusoutput(
1634                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1635                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1636                         if st != os.EX_OK:
1637                                 print(red("!!!") + " metadata.xml is invalid:")
1638                                 for z in out.splitlines():
1639                                         print(red("!!! ")+z)
1640                                 stats["metadata.bad"]+=1
1641                                 fails["metadata.bad"].append(x+"/metadata.xml")
1642
1643                 del metadata_bad
1644         muselist = frozenset(musedict)
1645
1646         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1647         changelog_modified = changelog_path in modified_changelogs
1648
1649         # detect unused local USE-descriptions
1650         used_useflags = set()
1651
1652         for y in ebuildlist:
1653                 relative_path = os.path.join(x, y + ".ebuild")
1654                 full_path = os.path.join(repodir, relative_path)
1655                 ebuild_path = y + ".ebuild"
1656                 if repolevel < 3:
1657                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1658                 if repolevel < 2:
1659                         ebuild_path = os.path.join(catdir, ebuild_path)
1660                 ebuild_path = os.path.join(".", ebuild_path)
1661                 if check_changelog and not changelog_modified \
1662                         and ebuild_path in new_ebuilds:
1663                         stats['changelog.ebuildadded'] += 1
1664                         fails['changelog.ebuildadded'].append(relative_path)
1665
1666                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1667                         #ebuild not added to vcs
1668                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1669                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1670                 myesplit=portage.pkgsplit(y)
1671                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1672                             or pv_toolong_re.search(myesplit[1]) \
1673                             or pv_toolong_re.search(myesplit[2]):
1674                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1675                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1676                         continue
1677                 elif myesplit[0]!=pkgdir:
1678                         print(pkgdir,myesplit[0])
1679                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1680                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1681                         continue
1682
1683                 pkg = pkgs[y]
1684
1685                 if pkg.invalid:
1686                         allvalid = False
1687                         for k, msgs in pkg.invalid.items():
1688                                 for msg in msgs:
1689                                         stats[k] = stats[k] + 1
1690                                         fails[k].append("%s %s" % (relative_path, msg))
1691                         continue
1692
1693                 myaux = pkg.metadata
1694                 eapi = myaux["EAPI"]
1695                 inherited = pkg.inherited
1696                 live_ebuild = live_eclasses.intersection(inherited)
1697
1698                 for k, v in myaux.items():
1699                         if not isinstance(v, basestring):
1700                                 continue
1701                         m = non_ascii_re.search(v)
1702                         if m is not None:
1703                                 stats["variable.invalidchar"] += 1
1704                                 fails["variable.invalidchar"].append(
1705                                         ("%s: %s variable contains non-ASCII " + \
1706                                         "character at position %s") % \
1707                                         (relative_path, k, m.start() + 1))
1708
1709                 if not src_uri_error:
1710                         # Check that URIs don't reference a server from thirdpartymirrors.
1711                         for uri in portage.dep.use_reduce( \
1712                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1713                                 contains_mirror = False
1714                                 for mirror in thirdpartymirrors:
1715                                         if uri.startswith(mirror):
1716                                                 contains_mirror = True
1717                                                 break
1718                                 if not contains_mirror:
1719                                         continue
1720
1721                                 stats["SRC_URI.mirror"] += 1
1722                                 fails["SRC_URI.mirror"].append(
1723                                         "%s: '%s' found in thirdpartymirrors" % \
1724                                         (relative_path, mirror))
1725
1726                 if myaux.get("PROVIDE"):
1727                         stats["virtual.oldstyle"]+=1
1728                         fails["virtual.oldstyle"].append(relative_path)
1729
1730                 for pos, missing_var in enumerate(missingvars):
1731                         if not myaux.get(missing_var):
1732                                 if catdir == "virtual" and \
1733                                         missing_var in ("HOMEPAGE", "LICENSE"):
1734                                         continue
1735                                 if live_ebuild and missing_var == "KEYWORDS":
1736                                         continue
1737                                 myqakey=missingvars[pos]+".missing"
1738                                 stats[myqakey]=stats[myqakey]+1
1739                                 fails[myqakey].append(x+"/"+y+".ebuild")
1740
1741                 if catdir == "virtual":
1742                         for var in ("HOMEPAGE", "LICENSE"):
1743                                 if myaux.get(var):
1744                                         myqakey = var + ".virtual"
1745                                         stats[myqakey] = stats[myqakey] + 1
1746                                         fails[myqakey].append(relative_path)
1747
1748                 # 14 is the length of DESCRIPTION=""
1749                 if len(myaux['DESCRIPTION']) > max_desc_len:
1750                         stats['DESCRIPTION.toolong'] += 1
1751                         fails['DESCRIPTION.toolong'].append(
1752                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1753                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1754
1755                 keywords = myaux["KEYWORDS"].split()
1756                 stable_keywords = []
1757                 for keyword in keywords:
1758                         if not keyword.startswith("~") and \
1759                                 not keyword.startswith("-"):
1760                                 stable_keywords.append(keyword)
1761                 if stable_keywords:
1762                         if ebuild_path in new_ebuilds:
1763                                 stable_keywords.sort()
1764                                 stats["KEYWORDS.stable"] += 1
1765                                 fails["KEYWORDS.stable"].append(
1766                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1767                                                 " ".join(stable_keywords))
1768
1769                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1770                         if not kw.startswith("-"))
1771
1772                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1773                 if previous_keywords is None:
1774                         slot_keywords[myaux["SLOT"]] = set()
1775                 elif ebuild_archs and not live_ebuild:
1776                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1777                         if dropped_keywords:
1778                                 stats["KEYWORDS.dropped"] += 1
1779                                 fails["KEYWORDS.dropped"].append(
1780                                         relative_path + ": %s" % \
1781                                         " ".join(sorted(dropped_keywords)))
1782
1783                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1784
1785                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1786                 if "-*" in keywords:
1787                         haskeyword = False
1788                         for kw in keywords:
1789                                 if kw[0] == "~":
1790                                         kw = kw[1:]
1791                                 if kw in kwlist:
1792                                         haskeyword = True
1793                         if not haskeyword:
1794                                 stats["KEYWORDS.stupid"] += 1
1795                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1796
1797                 """
1798                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1799                 not be allowed to be marked stable
1800                 """
1801                 if live_ebuild and repo_config.name == "gentoo":
1802                         bad_stable_keywords = []
1803                         for keyword in keywords:
1804                                 if not keyword.startswith("~") and \
1805                                         not keyword.startswith("-"):
1806                                         bad_stable_keywords.append(keyword)
1807                                 del keyword
1808                         if bad_stable_keywords:
1809                                 stats["LIVEVCS.stable"] += 1
1810                                 fails["LIVEVCS.stable"].append(
1811                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1812                                                 bad_stable_keywords)
1813                         del bad_stable_keywords
1814
1815                         if keywords and not has_global_mask(pkg):
1816                                 stats["LIVEVCS.unmasked"] += 1
1817                                 fails["LIVEVCS.unmasked"].append(relative_path)
1818
1819                 if options.ignore_arches:
1820                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1821                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1822                 else:
1823                         arches=[]
1824                         for keyword in myaux["KEYWORDS"].split():
1825                                 if (keyword[0]=="-"):
1826                                         continue
1827                                 elif (keyword[0]=="~"):
1828                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1829                                 else:
1830                                         arches.append([keyword, keyword, [keyword]])
1831                         if not arches:
1832                                 # Use an empty profile for checking dependencies of
1833                                 # packages that have empty KEYWORDS.
1834                                 arches.append(['**', '**', ['**']])
1835
1836                 unknown_pkgs = set()
1837                 baddepsyntax = False
1838                 badlicsyntax = False
1839                 badprovsyntax = False
1840                 catpkg = catdir+"/"+y
1841
1842                 inherited_java_eclass = "java-pkg-2" in inherited or \
1843                         "java-pkg-opt-2" in inherited
1844                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1845                 operator_tokens = set(["||", "(", ")"])
1846                 type_list, badsyntax = [], []
1847                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1848                         "LICENSE", "PROPERTIES", "PROVIDE"):
1849                         mydepstr = myaux[mytype]
1850
1851                         token_class = None
1852                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1853                                 token_class=portage.dep.Atom
1854
1855                         try:
1856                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1857                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1858                         except portage.exception.InvalidDependString as e:
1859                                 atoms = None
1860                                 badsyntax.append(str(e))
1861
1862                         if atoms and mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1863                                 if mytype in ("RDEPEND", "PDEPEND") and \
1864                                         "test?" in mydepstr.split():
1865                                         stats[mytype + '.suspect'] += 1
1866                                         fails[mytype + '.suspect'].append(relative_path + \
1867                                                 ": 'test?' USE conditional in %s" % mytype)
1868
1869                                 for atom in atoms:
1870                                         if atom == "||":
1871                                                 continue
1872
1873                                         # Skip dependency.unknown for blockers, so that we
1874                                         # don't encourage people to remove necessary blockers,
1875                                         # as discussed in bug #382407.
1876                                         if atom.blocker is None and \
1877                                                 not portdb.xmatch("match-all", atom) and \
1878                                                 not atom.cp.startswith("virtual/"):
1879                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1880
1881                                         is_blocker = atom.blocker
1882
1883                                         if mytype == "DEPEND" and \
1884                                                 not is_blocker and \
1885                                                 not inherited_java_eclass and \
1886                                                 atom.cp == "virtual/jdk":
1887                                                 stats['java.eclassesnotused'] += 1
1888                                                 fails['java.eclassesnotused'].append(relative_path)
1889                                         elif mytype == "DEPEND" and \
1890                                                 not is_blocker and \
1891                                                 not inherited_wxwidgets_eclass and \
1892                                                 atom.cp == "x11-libs/wxGTK":
1893                                                 stats['wxwidgets.eclassnotused'] += 1
1894                                                 fails['wxwidgets.eclassnotused'].append(
1895                                                         relative_path + ": DEPENDs on x11-libs/wxGTK"
1896                                                         " without inheriting wxwidgets.eclass")
1897                                         elif mytype in ("PDEPEND", "RDEPEND"):
1898                                                 if not is_blocker and \
1899                                                         atom.cp in suspect_rdepend:
1900                                                         stats[mytype + '.suspect'] += 1
1901                                                         fails[mytype + '.suspect'].append(
1902                                                                 relative_path + ": '%s'" % atom)
1903
1904                                         if atom.operator == "~" and \
1905                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1906                                                 stats[mytype + '.badtilde'] += 1
1907                                                 fails[mytype + '.badtilde'].append(
1908                                                         (relative_path + ": %s uses the ~ operator"
1909                                                          " with a non-zero revision:" + \
1910                                                          " '%s'") % (mytype, atom))
1911
1912                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1913
1914                 for m,b in zip(type_list, badsyntax):
1915                         stats[m+".syntax"] += 1
1916                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1917
1918                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1919                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1920                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1921                 badlicsyntax = badlicsyntax > 0
1922                 badprovsyntax = badprovsyntax > 0
1923
1924                 # uselist checks - global
1925                 myuse = []
1926                 default_use = []
1927                 for myflag in myaux["IUSE"].split():
1928                         flag_name = myflag.lstrip("+-")
1929                         used_useflags.add(flag_name)
1930                         if myflag != flag_name:
1931                                 default_use.append(myflag)
1932                         if flag_name not in uselist:
1933                                 myuse.append(flag_name)
1934
1935                 # uselist checks - metadata
1936                 for mypos in range(len(myuse)-1,-1,-1):
1937                         if myuse[mypos] and (myuse[mypos] in muselist):
1938                                 del myuse[mypos]
1939
1940                 if default_use and not eapi_has_iuse_defaults(eapi):
1941                         for myflag in default_use:
1942                                 stats['EAPI.incompatible'] += 1
1943                                 fails['EAPI.incompatible'].append(
1944                                         (relative_path + ": IUSE defaults" + \
1945                                         " not supported with EAPI='%s':" + \
1946                                         " '%s'") % (eapi, myflag))
1947
1948                 for mypos in range(len(myuse)):
1949                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1950                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1951
1952                 # license checks
1953                 if not badlicsyntax:
1954                         # Parse the LICENSE variable, remove USE conditions and
1955                         # flatten it.
1956                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
1957                         # Check each entry to ensure that it exists in PORTDIR's
1958                         # license directory.
1959                         for lic in licenses:
1960                                 # Need to check for "||" manually as no portage
1961                                 # function will remove it without removing values.
1962                                 if lic not in liclist and lic != "||":
1963                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1964                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
1965
1966                 #keyword checks
1967                 myuse = myaux["KEYWORDS"].split()
1968                 for mykey in myuse:
1969                         if mykey not in ("-*", "*", "~*"):
1970                                 myskey = mykey
1971                                 if myskey[:1] == "-":
1972                                         myskey = myskey[1:]
1973                                 if myskey[:1] == "~":
1974                                         myskey = myskey[1:]
1975                                 if myskey not in kwlist:
1976                                         stats["KEYWORDS.invalid"] += 1
1977                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1978                                 elif myskey not in profiles:
1979                                         stats["KEYWORDS.invalid"] += 1
1980                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1981
1982                 #restrict checks
1983                 myrestrict = None
1984                 try:
1985                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
1986                 except portage.exception.InvalidDependString as e:
1987                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1988                         fails["RESTRICT.syntax"].append(
1989                                 "%s: RESTRICT: %s" % (relative_path, e))
1990                         del e
1991                 if myrestrict:
1992                         myrestrict = set(myrestrict)
1993                         mybadrestrict = myrestrict.difference(valid_restrict)
1994                         if mybadrestrict:
1995                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
1996                                 for mybad in mybadrestrict:
1997                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1998                 #REQUIRED_USE check
1999                 required_use = myaux["REQUIRED_USE"]
2000                 if required_use:
2001                         if not eapi_has_required_use(eapi):
2002                                 stats['EAPI.incompatible'] += 1
2003                                 fails['EAPI.incompatible'].append(
2004                                         relative_path + ": REQUIRED_USE" + \
2005                                         " not supported with EAPI='%s'" % (eapi,))
2006                         try:
2007                                 portage.dep.check_required_use(required_use, (),
2008                                         pkg.iuse.is_valid_flag)
2009                         except portage.exception.InvalidDependString as e:
2010                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2011                                 fails["REQUIRED_USE.syntax"].append(
2012                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2013                                 del e
2014
2015                 # Syntax Checks
2016                 relative_path = os.path.join(x, y + ".ebuild")
2017                 full_path = os.path.join(repodir, relative_path)
2018                 if not vcs_preserves_mtime:
2019                         if ebuild_path not in new_ebuilds and \
2020                                 ebuild_path not in modified_ebuilds:
2021                                 pkg.mtime = None
2022                 try:
2023                         # All ebuilds should have utf_8 encoding.
2024                         f = io.open(_unicode_encode(full_path,
2025                                 encoding=_encodings['fs'], errors='strict'),
2026                                 mode='r', encoding=_encodings['repo.content'])
2027                         try:
2028                                 for check_name, e in run_checks(f, pkg):
2029                                         stats[check_name] += 1
2030                                         fails[check_name].append(relative_path + ': %s' % e)
2031                         finally:
2032                                 f.close()
2033                 except UnicodeDecodeError:
2034                         # A file.UTF8 failure will have already been recorded above.
2035                         pass
2036
2037                 if options.force:
2038                         # The dep_check() calls are the most expensive QA test. If --force
2039                         # is enabled, there's no point in wasting time on these since the
2040                         # user is intent on forcing the commit anyway.
2041                         continue
2042
2043                 for keyword,arch,groups in arches:
2044
2045                         if arch not in profiles:
2046                                 # A missing profile will create an error further down
2047                                 # during the KEYWORDS verification.
2048                                 continue
2049
2050                         for prof in profiles[arch]:
2051
2052                                 if prof.status not in ("stable", "dev") or \
2053                                         prof.status == "dev" and not options.include_dev:
2054                                         continue
2055
2056                                 dep_settings = arch_caches.get(prof.sub_path)
2057                                 if dep_settings is None:
2058                                         dep_settings = portage.config(
2059                                                 config_profile_path=prof.abs_path,
2060                                                 config_incrementals=repoman_incrementals,
2061                                                 config_root=config_root,
2062                                                 local_config=False,
2063                                                 _unmatched_removal=options.unmatched_removal,
2064                                                 env=env)
2065                                         dep_settings.categories = repoman_settings.categories
2066                                         if options.without_mask:
2067                                                 dep_settings._mask_manager_obj = \
2068                                                         copy.deepcopy(dep_settings._mask_manager)
2069                                                 dep_settings._mask_manager._pmaskdict.clear()
2070                                         arch_caches[prof.sub_path] = dep_settings
2071
2072                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2073                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2074                                 if xcache is None:
2075                                         portdb.melt()
2076                                         portdb.freeze()
2077                                         xcache = portdb.xcache
2078                                         xcache.update(shared_xmatch_caches)
2079                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2080
2081                                 trees[root]["porttree"].settings = dep_settings
2082                                 portdb.settings = dep_settings
2083                                 portdb.xcache = xcache
2084                                 # for package.use.mask support inside dep_check
2085                                 dep_settings.setcpv(pkg)
2086                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2087                                 # just in case, prevent config.reset() from nuking these.
2088                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2089
2090                                 if not baddepsyntax:
2091                                         ismasked = not ebuild_archs or \
2092                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2093                                         if ismasked:
2094                                                 if not have_pmasked:
2095                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2096                                                                 pkg.cpv, pkg.metadata))
2097                                                 if options.ignore_masked:
2098                                                         continue
2099                                                 #we are testing deps for a masked package; give it some lee-way
2100                                                 suffix="masked"
2101                                                 matchmode = "minimum-all"
2102                                         else:
2103                                                 suffix=""
2104                                                 matchmode = "minimum-visible"
2105
2106                                         if not have_dev_keywords:
2107                                                 have_dev_keywords = \
2108                                                         bool(dev_keywords.intersection(keywords))
2109
2110                                         if prof.status == "dev":
2111                                                 suffix=suffix+"indev"
2112
2113                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
2114
2115                                                 mykey=mytype+".bad"+suffix
2116                                                 myvalue = myaux[mytype]
2117                                                 if not myvalue:
2118                                                         continue
2119
2120                                                 success, atoms = portage.dep_check(myvalue, portdb,
2121                                                         dep_settings, use="all", mode=matchmode,
2122                                                         trees=trees)
2123
2124                                                 if success:
2125                                                         if atoms:
2126
2127                                                                 # Don't bother with dependency.unknown for
2128                                                                 # cases in which *DEPEND.bad is triggered.
2129                                                                 for atom in atoms:
2130                                                                         # dep_check returns all blockers and they
2131                                                                         # aren't counted for *DEPEND.bad, so we
2132                                                                         # ignore them here.
2133                                                                         if not atom.blocker:
2134                                                                                 unknown_pkgs.discard(
2135                                                                                         (mytype, atom.unevaluated_atom))
2136
2137                                                                 if not prof.sub_path:
2138                                                                         # old-style virtuals currently aren't
2139                                                                         # resolvable with empty profile, since
2140                                                                         # 'virtuals' mappings are unavailable
2141                                                                         # (it would be expensive to search
2142                                                                         # for PROVIDE in all ebuilds)
2143                                                                         atoms = [atom for atom in atoms if not \
2144                                                                                 (atom.cp.startswith('virtual/') and \
2145                                                                                 not portdb.cp_list(atom.cp))]
2146
2147                                                                 #we have some unsolvable deps
2148                                                                 #remove ! deps, which always show up as unsatisfiable
2149                                                                 atoms = [str(atom.unevaluated_atom) \
2150                                                                         for atom in atoms if not atom.blocker]
2151
2152                                                                 #if we emptied out our list, continue:
2153                                                                 if not atoms:
2154                                                                         continue
2155                                                                 stats[mykey]=stats[mykey]+1
2156                                                                 fails[mykey].append("%s: %s(%s) %s" % \
2157                                                                         (relative_path, keyword,
2158                                                                         prof, repr(atoms)))
2159                                                 else:
2160                                                         stats[mykey]=stats[mykey]+1
2161                                                         fails[mykey].append("%s: %s(%s) %s" % \
2162                                                                 (relative_path, keyword,
2163                                                                 prof, repr(atoms)))
2164
2165                 if not baddepsyntax and unknown_pkgs:
2166                         type_map = {}
2167                         for mytype, atom in unknown_pkgs:
2168                                 type_map.setdefault(mytype, set()).add(atom)
2169                         for mytype, atoms in type_map.items():
2170                                 stats["dependency.unknown"] += 1
2171                                 fails["dependency.unknown"].append("%s: %s: %s" %
2172                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2173
2174         # check if there are unused local USE-descriptions in metadata.xml
2175         # (unless there are any invalids, to avoid noise)
2176         if allvalid:
2177                 for myflag in muselist.difference(used_useflags):
2178                         stats["metadata.warning"] += 1
2179                         fails["metadata.warning"].append(
2180                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2181                                 (x, myflag))
2182
2183 if options.if_modified == "y" and len(effective_scanlist) < 1:
2184         logging.warn("--if-modified is enabled, but no modified packages were found!")
2185
2186 if options.mode == "manifest":
2187         sys.exit(dofail)
2188
2189 #dofail will be set to 1 if we have failed in at least one non-warning category
2190 dofail=0
2191 #dowarn will be set to 1 if we tripped any warnings
2192 dowarn=0
2193 #dofull will be set if we should print a "repoman full" informational message
2194 dofull = options.mode != 'full'
2195
2196 for x in qacats:
2197         if not stats[x]:
2198                 continue
2199         dowarn = 1
2200         if x not in qawarnings:
2201                 dofail = 1
2202
2203 if dofail or \
2204         (dowarn and not (options.quiet or options.mode == "scan")):
2205         dofull = 0
2206
2207 # Save QA output so that it can be conveniently displayed
2208 # in $EDITOR while the user creates a commit message.
2209 # Otherwise, the user would not be able to see this output
2210 # once the editor has taken over the screen.
2211 qa_output = io.StringIO()
2212 style_file = ConsoleStyleFile(sys.stdout)
2213 if options.mode == 'commit' and \
2214         (not commitmessage or not commitmessage.strip()):
2215         style_file.write_listener = qa_output
2216 console_writer = StyleWriter(file=style_file, maxcol=9999)
2217 console_writer.style_listener = style_file.new_styles
2218
2219 f = formatter.AbstractFormatter(console_writer)
2220
2221 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2222
2223 style_file.flush()
2224 del console_writer, f, style_file
2225 qa_output = qa_output.getvalue()
2226 qa_output = qa_output.splitlines(True)
2227
2228 def grouplist(mylist,seperator="/"):
2229         """(list,seperator="/") -- Takes a list of elements; groups them into
2230         same initial element categories. Returns a dict of {base:[sublist]}
2231         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2232         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2233         mygroups={}
2234         for x in mylist:
2235                 xs=x.split(seperator)
2236                 if xs[0]==".":
2237                         xs=xs[1:]
2238                 if xs[0] not in mygroups:
2239                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2240                 else:
2241                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2242         return mygroups
2243
2244 suggest_ignore_masked = False
2245 suggest_include_dev = False
2246
2247 if have_pmasked and not (options.without_mask or options.ignore_masked):
2248         suggest_ignore_masked = True
2249 if have_dev_keywords and not options.include_dev:
2250         suggest_include_dev = True
2251
2252 if suggest_ignore_masked or suggest_include_dev:
2253         print()
2254         if suggest_ignore_masked:
2255                 print(bold("Note: use --without-mask to check " + \
2256                         "KEYWORDS on dependencies of masked packages"))
2257
2258         if suggest_include_dev:
2259                 print(bold("Note: use --include-dev (-d) to check " + \
2260                         "dependencies for 'dev' profiles"))
2261         print()
2262
2263 if options.mode != 'commit':
2264         if dofull:
2265                 print(bold("Note: type \"repoman full\" for a complete listing."))
2266         if dowarn and not dofail:
2267                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2268         elif not dofail:
2269                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2270         elif dofail:
2271                 print(bad("Please fix these important QA issues first."))
2272                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2273                 sys.exit(1)
2274 else:
2275         if dofail and can_force and options.force and not options.pretend:
2276                 print(green("RepoMan sez:") + \
2277                         " \"You want to commit even with these QA issues?\n" + \
2278                         "              I'll take it this time, but I'm not happy.\"\n")
2279         elif dofail:
2280                 if options.force and not can_force:
2281                         print(bad("The --force option has been disabled due to extraordinary issues."))
2282                 print(bad("Please fix these important QA issues first."))
2283                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2284                 sys.exit(1)
2285
2286         if options.pretend:
2287                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2288
2289         myunadded = []
2290         if vcs == "cvs":
2291                 try:
2292                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2293                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2294                 except SystemExit as e:
2295                         raise  # TODO propagate this
2296                 except:
2297                         err("Error retrieving CVS tree; exiting.")
2298         if vcs == "svn":
2299                 try:
2300                         with os.popen("svn status --no-ignore") as f:
2301                                 svnstatus = f.readlines()
2302                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2303                 except SystemExit as e:
2304                         raise  # TODO propagate this
2305                 except:
2306                         err("Error retrieving SVN info; exiting.")
2307         if vcs == "git":
2308                 # get list of files not under version control or missing
2309                 myf = os.popen("git ls-files --others")
2310                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2311                 myf.close()
2312         if vcs == "bzr":
2313                 try:
2314                         with os.popen("bzr status -S .") as f:
2315                                 bzrstatus = f.readlines()
2316                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2317                 except SystemExit as e:
2318                         raise  # TODO propagate this
2319                 except:
2320                         err("Error retrieving bzr info; exiting.")
2321         if vcs == "hg":
2322                 with os.popen("hg status --no-status --unknown .") as f:
2323                         myunadded = f.readlines()
2324                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2325
2326                 # Mercurial doesn't handle manually deleted files as removed from
2327                 # the repository, so the user need to remove them before commit,
2328                 # using "hg remove [FILES]"
2329                 with os.popen("hg status --no-status --deleted .") as f:
2330                         mydeleted = f.readlines()
2331                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2332
2333
2334         myautoadd=[]
2335         if myunadded:
2336                 for x in range(len(myunadded)-1,-1,-1):
2337                         xs=myunadded[x].split("/")
2338                         if xs[-1]=="files":
2339                                 print("!!! files dir is not added! Please correct this.")
2340                                 sys.exit(-1)
2341                         elif xs[-1]=="Manifest":
2342                                 # It's a manifest... auto add
2343                                 myautoadd+=[myunadded[x]]
2344                                 del myunadded[x]
2345
2346         if myunadded:
2347                 print(red("!!! The following files are in your local tree but are not added to the master"))
2348                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2349                 for x in myunadded:
2350                         print("   ",x)
2351                 print()
2352                 print()
2353                 sys.exit(1)
2354
2355         if vcs == "hg" and mydeleted:
2356                 print(red("!!! The following files are removed manually from your local tree but are not"))
2357                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2358                 for x in mydeleted:
2359                         print("   ",x)
2360                 print()
2361                 print()
2362                 sys.exit(1)
2363
2364         if vcs == "cvs":
2365                 mycvstree = cvstree.getentries("./", recursive=1)
2366                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2367                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2368                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2369                 bin_blob_pattern = re.compile("^-kb$")
2370                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2371                         recursive=1, basedir="./"))
2372
2373
2374         if vcs == "svn":
2375                 with os.popen("svn status") as f:
2376                         svnstatus = f.readlines()
2377                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2378                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2379                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2380
2381                 # Subversion expands keywords specified in svn:keywords properties.
2382                 with os.popen("svn propget -R svn:keywords") as f:
2383                         props = f.readlines()
2384                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2385                         for prop in props if " - " in prop)
2386
2387         elif vcs == "git":
2388                 with os.popen("git diff-index --name-only "
2389                         "--relative --diff-filter=M HEAD") as f:
2390                         mychanged = f.readlines()
2391                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2392
2393                 with os.popen("git diff-index --name-only "
2394                         "--relative --diff-filter=A HEAD") as f:
2395                         mynew = f.readlines()
2396                 mynew = ["./" + elem[:-1] for elem in mynew]
2397
2398                 with os.popen("git diff-index --name-only "
2399                         "--relative --diff-filter=D HEAD") as f:
2400                         myremoved = f.readlines()
2401                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2402
2403         if vcs == "bzr":
2404                 with os.popen("bzr status -S .") as f:
2405                         bzrstatus = f.readlines()
2406                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2407                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2408                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2409                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2410                 # Bazaar expands nothing.
2411
2412         if vcs == "hg":
2413                 with os.popen("hg status --no-status --modified .") as f:
2414                         mychanged = f.readlines()
2415                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2416
2417                 with os.popen("hg status --no-status --added .") as f:
2418                         mynew = f.readlines()
2419                 mynew = ["./" + elem.rstrip() for elem in mynew]
2420
2421                 with os.popen("hg status --no-status --removed .") as f:
2422                         myremoved = f.readlines()
2423                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2424
2425         if vcs:
2426                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2427                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2428                         print()
2429                         print("(Didn't find any changed files...)")
2430                         print()
2431                         sys.exit(1)
2432
2433         # Manifests need to be regenerated after all other commits, so don't commit
2434         # them now even if they have changed.
2435         mymanifests = set()
2436         myupdates = set()
2437         for f in mychanged + mynew:
2438                 if "Manifest" == os.path.basename(f):
2439                         mymanifests.add(f)
2440                 else:
2441                         myupdates.add(f)
2442         myupdates.difference_update(myremoved)
2443         myupdates = list(myupdates)
2444         mymanifests = list(mymanifests)
2445         myheaders = []
2446         mydirty = []
2447
2448         commitmessage = options.commitmsg
2449         if options.commitmsgfile:
2450                 try:
2451                         f = io.open(_unicode_encode(options.commitmsgfile,
2452                         encoding=_encodings['fs'], errors='strict'),
2453                         mode='r', encoding=_encodings['content'], errors='replace')
2454                         commitmessage = f.read()
2455                         f.close()
2456                         del f
2457                 except (IOError, OSError) as e:
2458                         if e.errno == errno.ENOENT:
2459                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2460                         else:
2461                                 raise
2462                 # We've read the content so the file is no longer needed.
2463                 commitmessagefile = None
2464         if not commitmessage or not commitmessage.strip():
2465                 try:
2466                         editor = os.environ.get("EDITOR")
2467                         if editor and utilities.editor_is_executable(editor):
2468                                 commitmessage = utilities.get_commit_message_with_editor(
2469                                         editor, message=qa_output)
2470                         else:
2471                                 commitmessage = utilities.get_commit_message_with_stdin()
2472                 except KeyboardInterrupt:
2473                         exithandler()
2474                 if not commitmessage or not commitmessage.strip():
2475                         print("* no commit message?  aborting commit.")
2476                         sys.exit(1)
2477         commitmessage = commitmessage.rstrip()
2478         changelog_msg = commitmessage
2479         portage_version = getattr(portage, "VERSION", None)
2480         if portage_version is None:
2481                 sys.stderr.write("Failed to insert portage version in message!\n")
2482                 sys.stderr.flush()
2483                 portage_version = "Unknown"
2484         unameout = platform.system() + " "
2485         if platform.system() in ["Darwin", "SunOS"]:
2486                 unameout += platform.processor()
2487         else:
2488                 unameout += platform.machine()
2489         commitmessage += "\n\n(Portage version: %s/%s/%s" % \
2490                 (portage_version, vcs, unameout)
2491         if options.force:
2492                 commitmessage += ", RepoMan options: --force"
2493         commitmessage += ")"
2494
2495         if options.echangelog in ('y', 'force'):
2496                 logging.info("checking for unmodified ChangeLog files")
2497                 committer_name = utilities.get_committer_name(env=repoman_settings)
2498                 for x in sorted(vcs_files_to_cps(
2499                         chain(myupdates, mymanifests, myremoved))):
2500                         catdir, pkgdir = x.split("/")
2501                         checkdir = repodir + "/" + x
2502                         checkdir_relative = ""
2503                         if repolevel < 3:
2504                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2505                         if repolevel < 2:
2506                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2507                         checkdir_relative = os.path.join(".", checkdir_relative)
2508
2509                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2510                         changelog_modified = changelog_path in modified_changelogs
2511                         if changelog_modified and options.echangelog != 'force':
2512                                 continue
2513
2514                         # get changes for this package
2515                         cdrlen = len(checkdir_relative)
2516                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2517                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2518                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2519
2520                         # Skip ChangeLog generation if only the Manifest was modified,
2521                         # as discussed in bug #398009.
2522                         nontrivial_cl_files = set()
2523                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2524                         nontrivial_cl_files.difference_update(['Manifest'])
2525                         if not nontrivial_cl_files and options.echangelog != 'force':
2526                                 continue
2527
2528                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2529                                 committer_name, changelog_msg,
2530                                 os.path.join(repodir, 'skel.ChangeLog'),
2531                                 catdir, pkgdir,
2532                                 new=clnew, removed=clremoved, changed=clchanged,
2533                                 pretend=options.pretend)
2534                         if new_changelog is None:
2535                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2536                                         level=logging.ERROR, noiselevel=-1)
2537                                 sys.exit(1)
2538
2539                         # if the ChangeLog was just created, add it to vcs
2540                         if new_changelog:
2541                                 myautoadd.append(changelog_path)
2542                                 # myautoadd is appended to myupdates below
2543                         else:
2544                                 myupdates.append(changelog_path)
2545
2546                         if not options.pretend:
2547                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2548                                 repoman_settings["O"] = checkdir
2549                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2550
2551         if myautoadd:
2552                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2553                 add_cmd = [vcs, "add"]
2554                 add_cmd += myautoadd
2555                 if options.pretend:
2556                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2557                                 noiselevel=-1)
2558                 else:
2559                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2560                                 # Python 3.1 produces the following TypeError if raw bytes are
2561                                 # passed to subprocess.call():
2562                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2563                                 #     errread, errwrite)
2564                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2565                                 #     raise child_exception
2566                                 # TypeError: expected an object with the buffer interface
2567                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2568                         retcode = subprocess.call(add_cmd)
2569                         if retcode != os.EX_OK:
2570                                 logging.error(
2571                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2572                                 sys.exit(retcode)
2573
2574                 myupdates += myautoadd
2575
2576         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2577
2578         if vcs not in ('cvs', 'svn'):
2579                 # With git, bzr and hg, there's never any keyword expansion, so
2580                 # there's no need to regenerate manifests and all files will be
2581                 # committed in one big commit at the end.
2582                 print()
2583         elif not repo_config.thin_manifest:
2584                 if vcs == 'cvs':
2585                         headerstring = "'\$(Header|Id).*\$'"
2586                 elif vcs == "svn":
2587                         svn_keywords = dict((k.lower(), k) for k in [
2588                                         "Rev",
2589                                         "Revision",
2590                                         "LastChangedRevision",
2591                                         "Date",
2592                                         "LastChangedDate",
2593                                         "Author",
2594                                         "LastChangedBy",
2595                                         "URL",
2596                                         "HeadURL",
2597                                         "Id",
2598                                         "Header",
2599                         ])
2600
2601                 for myfile in myupdates:
2602
2603                         # for CVS, no_expansion contains files that are excluded from expansion
2604                         if vcs == "cvs":
2605                                 if myfile in no_expansion:
2606                                         continue
2607
2608                         # for SVN, expansion contains files that are included in expansion
2609                         elif vcs == "svn":
2610                                 if myfile not in expansion:
2611                                         continue
2612
2613                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2614                                 enabled_keywords = []
2615                                 for k in expansion[myfile]:
2616                                         keyword = svn_keywords.get(k.lower())
2617                                         if keyword is not None:
2618                                                 enabled_keywords.append(keyword)
2619
2620                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2621
2622                         myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile)
2623                         if myout[0] == 0:
2624                                 myheaders.append(myfile)
2625
2626                 print("%s have headers that will change." % green(str(len(myheaders))))
2627                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2628
2629         logging.info("myupdates: %s", myupdates)
2630         logging.info("myheaders: %s", myheaders)
2631
2632         if options.ask and userquery('Commit changes?', True) != 'Yes':
2633                 print("* aborting commit.")
2634                 sys.exit(128 + signal.SIGINT)
2635
2636         # Handle the case where committed files have keywords which
2637         # will change and need a priming commit before the Manifest
2638         # can be committed.
2639         if (myupdates or myremoved) and myheaders:
2640                 myfiles = myupdates + myremoved
2641                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2642                 mymsg = os.fdopen(fd, "wb")
2643                 mymsg.write(_unicode_encode(commitmessage))
2644                 mymsg.close()
2645
2646                 print()
2647                 print(green("Using commit message:"))
2648                 print(green("------------------------------------------------------------------------------"))
2649                 print(commitmessage)
2650                 print(green("------------------------------------------------------------------------------"))
2651                 print()
2652
2653                 # Having a leading ./ prefix on file paths can trigger a bug in
2654                 # the cvs server when committing files to multiple directories,
2655                 # so strip the prefix.
2656                 myfiles = [f.lstrip("./") for f in myfiles]
2657
2658                 commit_cmd = [vcs]
2659                 commit_cmd.extend(vcs_global_opts)
2660                 commit_cmd.append("commit")
2661                 commit_cmd.extend(vcs_local_opts)
2662                 commit_cmd.extend(["-F", commitmessagefile])
2663                 commit_cmd.extend(myfiles)
2664
2665                 try:
2666                         if options.pretend:
2667                                 print("(%s)" % (" ".join(commit_cmd),))
2668                         else:
2669                                 retval = spawn(commit_cmd, env=os.environ)
2670                                 if retval != os.EX_OK:
2671                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2672                                                 "error code: %s\n") % (vcs, retval),
2673                                                 level=logging.ERROR, noiselevel=-1)
2674                                         sys.exit(retval)
2675                 finally:
2676                         try:
2677                                 os.unlink(commitmessagefile)
2678                         except OSError:
2679                                 pass
2680
2681         # Setup the GPG commands
2682         def gpgsign(filename):
2683                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2684                 if gpgcmd is None:
2685                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2686                                 " Is make.globals missing?")
2687                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2688                         "PORTAGE_GPG_KEY" not in repoman_settings:
2689                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2690                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2691                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2692                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2693                                         os.path.expanduser("~/.gnupg")
2694                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2695                                         % repoman_settings["PORTAGE_GPG_DIR"])
2696                         else:
2697                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2698                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2699                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2700                                 raise portage.exception.InvalidLocation(
2701                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2702                                         repoman_settings["PORTAGE_GPG_DIR"])
2703                 gpgvars = {"FILE": filename}
2704                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2705                         v = repoman_settings.get(k)
2706                         if v is not None:
2707                                 gpgvars[k] = v
2708                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2709                 if options.pretend:
2710                         print("("+gpgcmd+")")
2711                 else:
2712                         rValue = os.system(gpgcmd)
2713                         if rValue == os.EX_OK:
2714                                 os.rename(filename+".asc", filename)
2715                         else:
2716                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2717
2718         # When files are removed and re-added, the cvs server will put /Attic/
2719         # inside the $Header path. This code detects the problem and corrects it
2720         # so that the Manifest will generate correctly. See bug #169500.
2721         # Use binary mode in order to avoid potential character encoding issues.
2722         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2723         attic_str = b'/Attic/'
2724         attic_replace = b'/'
2725         for x in myheaders:
2726                 f = open(_unicode_encode(x,
2727                         encoding=_encodings['fs'], errors='strict'),
2728                         mode='rb')
2729                 mylines = f.readlines()
2730                 f.close()
2731                 modified = False
2732                 for i, line in enumerate(mylines):
2733                         if cvs_header_re.match(line) is not None and \
2734                                 attic_str in line:
2735                                 mylines[i] = line.replace(attic_str, attic_replace)
2736                                 modified = True
2737                 if modified:
2738                         portage.util.write_atomic(x, b''.join(mylines),
2739                                 mode='wb')
2740
2741         if repolevel == 1:
2742                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2743                         "doing the entire repository.\"\n")
2744
2745         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2746
2747                 for x in sorted(vcs_files_to_cps(
2748                         chain(myupdates, myremoved, mymanifests))):
2749                         repoman_settings["O"] = os.path.join(repodir, x)
2750                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2751
2752         signed = False
2753         if sign_manifests:
2754                 signed = True
2755                 try:
2756                         for x in sorted(vcs_files_to_cps(
2757                                 chain(myupdates, myremoved, mymanifests))):
2758                                 repoman_settings["O"] = os.path.join(repodir, x)
2759                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2760                                 if not os.path.exists(manifest_path):
2761                                         continue
2762                                 gpgsign(manifest_path)
2763                 except portage.exception.PortageException as e:
2764                         portage.writemsg("!!! %s\n" % str(e))
2765                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2766                         signed = False
2767
2768         if vcs == 'git':
2769                 # It's not safe to use the git commit -a option since there might
2770                 # be some modified files elsewhere in the working tree that the
2771                 # user doesn't want to commit. Therefore, call git update-index
2772                 # in order to ensure that the index is updated with the latest
2773                 # versions of all new and modified files in the relevant portion
2774                 # of the working tree.
2775                 myfiles = mymanifests + myupdates
2776                 myfiles.sort()
2777                 update_index_cmd = ["git", "update-index"]
2778                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2779                 if options.pretend:
2780                         print("(%s)" % (" ".join(update_index_cmd),))
2781                 else:
2782                         retval = spawn(update_index_cmd, env=os.environ)
2783                         if retval != os.EX_OK:
2784                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2785                                         "error code: %s\n") % (vcs, retval),
2786                                         level=logging.ERROR, noiselevel=-1)
2787                                 sys.exit(retval)
2788
2789         if True:
2790
2791                 myfiles = mymanifests[:]
2792                 # If there are no header (SVN/CVS keywords) changes in
2793                 # the files, this Manifest commit must include the
2794                 # other (yet uncommitted) files.
2795                 if not myheaders:
2796                         myfiles += myupdates
2797                         myfiles += myremoved
2798                 myfiles.sort()
2799
2800                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2801                 mymsg = os.fdopen(fd, "wb")
2802                 # strip the closing parenthesis
2803                 mymsg.write(_unicode_encode(commitmessage[:-1]))
2804                 if signed:
2805                         mymsg.write(_unicode_encode(
2806                                 ", signed Manifest commit with key %s)" % \
2807                                 repoman_settings["PORTAGE_GPG_KEY"]))
2808                 else:
2809                         mymsg.write(b", unsigned Manifest commit)")
2810                 mymsg.close()
2811
2812                 commit_cmd = []
2813                 if options.pretend and vcs is None:
2814                         # substitute a bogus value for pretend output
2815                         commit_cmd.append("cvs")
2816                 else:
2817                         commit_cmd.append(vcs)
2818                 commit_cmd.extend(vcs_global_opts)
2819                 commit_cmd.append("commit")
2820                 commit_cmd.extend(vcs_local_opts)
2821                 if vcs == "hg":
2822                         commit_cmd.extend(["--logfile", commitmessagefile])
2823                         commit_cmd.extend(myfiles)
2824                 else:
2825                         commit_cmd.extend(["-F", commitmessagefile])
2826                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2827
2828                 try:
2829                         if options.pretend:
2830                                 print("(%s)" % (" ".join(commit_cmd),))
2831                         else:
2832                                 retval = spawn(commit_cmd, env=os.environ)
2833                                 if retval != os.EX_OK:
2834
2835                                         if repo_config.sign_commit and vcs == 'git' and \
2836                                                 not git_supports_gpg_sign():
2837                                                 # Inform user that newer git is needed (bug #403323).
2838                                                 logging.error(
2839                                                         "Git >=1.7.9 is required for signed commits!")
2840
2841                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2842                                                 "error code: %s\n") % (vcs, retval),
2843                                                 level=logging.ERROR, noiselevel=-1)
2844                                         sys.exit(retval)
2845                 finally:
2846                         try:
2847                                 os.unlink(commitmessagefile)
2848                         except OSError:
2849                                 pass
2850
2851         print()
2852         if vcs:
2853                 print("Commit complete.")
2854         else:
2855                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2856         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2857 sys.exit(0)
2858