bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2012 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import io
  17 import logging
  18 import optparse
  19 import re
  20 import signal
  21 import stat
  22 import subprocess
  23 import sys
  24 import tempfile
  25 import textwrap
  26 import time
  27 import platform
  28
  29 try:
  30         from urllib.request import urlopen as urllib_request_urlopen
  31 except ImportError:
  32         from urllib import urlopen as urllib_request_urlopen
  33
  34 from itertools import chain
  35 from stat import S_ISDIR
  36
  37 from os import path as osp
  38 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  39 sys.path.insert(0, pym_path)
  40 import portage
  41 portage._disable_legacy_globals()
  42 portage.dep._internal_warnings = True
  43
  44 try:
  45         import xml.etree.ElementTree
  46         from xml.parsers.expat import ExpatError
  47 except (SystemExit, KeyboardInterrupt):
  48         raise
  49 except (ImportError, SystemError, RuntimeError, Exception):
  50         # broken or missing xml support
  51         # http://bugs.python.org/issue14988
  52         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  53         from portage.output import EOutput
  54         out = EOutput()
  55         for line in msg:
  56                 out.eerror(line)
  57         sys.exit(1)
  58
  59 from portage import os
  60 from portage import _encodings
  61 from portage import _unicode_encode
  62 from repoman.checks import run_checks
  63 from repoman import utilities
  64 from repoman.herdbase import make_herd_base
  65 from _emerge.Package import Package
  66 from _emerge.RootConfig import RootConfig
  67 from _emerge.userquery import userquery
  68 import portage.checksum
  69 import portage.const
  70 from portage import cvstree, normalize_path
  71 from portage import util
  72 from portage.exception import (FileNotFound, MissingParameter,
  73         ParseError, PermissionDenied)
  74 from portage.manifest import _prohibited_filename_chars_re as \
  75         disallowed_filename_chars_re
  76 from portage.process import find_binary, spawn
  77 from portage.output import bold, create_color_func, \
  78         green, nocolor, red
  79 from portage.output import ConsoleStyleFile, StyleWriter
  80 from portage.util import writemsg_level
  81 from portage.util._desktop_entry import validate_desktop_entry
  82 from portage.package.ebuild.digestgen import digestgen
  83 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  84
  85 if sys.hexversion >= 0x3000000:
  86         basestring = str
  87
  88 util.initialize_logger()
  89
  90 # 14 is the length of DESCRIPTION=""
  91 max_desc_len = 100
  92 allowed_filename_chars="a-zA-Z0-9._-+:"
  93 pv_toolong_re = re.compile(r'[0-9]{19,}')
  94 bad = create_color_func("BAD")
  95
  96 # A sane umask is needed for files that portage creates.
  97 os.umask(0o22)
  98 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  99 # behave incrementally.
 100 repoman_incrementals = tuple(x for x in \
 101         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 102 config_root = os.environ.get("PORTAGE_CONFIGROOT")
 103 repoman_settings = portage.config(config_root=config_root, local_config=False)
 104
 105 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 106         repoman_settings.get('TERM') == 'dumb' or \
 107         not sys.stdout.isatty():
 108         nocolor()
 109
 110 def warn(txt):
 111         print("repoman: " + txt)
 112
 113 def err(txt):
 114         warn(txt)
 115         sys.exit(1)
 116
 117 def exithandler(signum=None, frame=None):
 118         logging.fatal("Interrupted; exiting...")
 119         if signum is None:
 120                 sys.exit(1)
 121         else:
 122                 sys.exit(128 + signum)
 123
 124 signal.signal(signal.SIGINT,exithandler)
 125
 126 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 127         """Repoman needs it's own HelpFormatter for now, because the default ones
 128         murder the help text."""
 129
 130         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 131                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 132
 133         def format_description(self, description):
 134                 return description
 135
 136 class RepomanOptionParser(optparse.OptionParser):
 137         """Add the on_tail function, ruby has it, optionParser should too
 138         """
 139
 140         def __init__(self, *args, **kwargs):
 141                 optparse.OptionParser.__init__(self, *args, **kwargs)
 142                 self.tail = ""
 143
 144         def on_tail(self, description):
 145                 self.tail += description
 146
 147         def format_help(self, formatter=None):
 148                 result = optparse.OptionParser.format_help(self, formatter)
 149                 result += self.tail
 150                 return result
 151
 152
 153 def ParseArgs(argv, qahelp):
 154         """This function uses a customized optionParser to parse command line arguments for repoman
 155         Args:
 156           argv - a sequence of command line arguments
 157                 qahelp - a dict of qa warning to help message
 158         Returns:
 159           (opts, args), just like a call to parser.parse_args()
 160         """
 161
 162         if argv and isinstance(argv[0], bytes):
 163                 argv = [portage._unicode_decode(x) for x in argv]
 164
 165         modes = {
 166                 'commit' : 'Run a scan then commit changes',
 167                 'ci' : 'Run a scan then commit changes',
 168                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 169                 'full' : 'Scan directory tree and print all issues (not a summary)',
 170                 'help' : 'Show this screen',
 171                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 172                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 173                 'scan' : 'Scan directory tree for QA issues'
 174         }
 175
 176         mode_keys = list(modes)
 177         mode_keys.sort()
 178
 179         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 180         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 181         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 182         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 183         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 184
 185         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 186                 help='Request a confirmation before commiting')
 187
 188         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 189                 help='specify a commit message on the command line')
 190
 191         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 192                 help='specify a path to a file that contains a commit message')
 193
 194         parser.add_option('--digest',
 195                 type='choice', choices=('y', 'n'), metavar='<y|n>',
 196                 help='Automatically update Manifest digests for modified files')
 197
 198         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 199                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 200
 201         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 202                 help='do not print unnecessary messages')
 203
 204         parser.add_option(
 205                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 206                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 207                 'regardless of modification if \'force\' is specified)')
 208
 209         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 210                 help='Commit with QA violations')
 211
 212         parser.add_option('--vcs', dest='vcs',
 213                 help='Force using specific VCS instead of autodetection')
 214
 215         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 216                 help='be very verbose in output', default=0)
 217
 218         parser.add_option('-V', '--version', dest='version', action='store_true',
 219                 help='show version info')
 220
 221         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 222                 default=False, help='forces the metadata.xml parse check to be carried out')
 223
 224         parser.add_option(
 225                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 226                 metavar="<y|n>",
 227                 help='only check packages that have uncommitted modifications')
 228
 229         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 230                 default=False, help='ignore arch-specific failures (where arch != host)')
 231
 232         parser.add_option("--ignore-default-opts",
 233                 action="store_true",
 234                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 235
 236         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 237                 default=False, help='ignore masked packages (not allowed with commit mode)')
 238
 239         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 240                 default=False, help='include dev profiles in dependency checks')
 241
 242         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 243                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 244
 245         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 246                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 247
 248         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 249                 help='specify which mode repoman will run in (default=full)')
 250
 251         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 252
 253         for k in mode_keys:
 254                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 255
 256         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 257
 258         sorted_qa = list(qahelp)
 259         sorted_qa.sort()
 260         for k in sorted_qa:
 261                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 262
 263         opts, args = parser.parse_args(argv[1:])
 264
 265         if not opts.ignore_default_opts:
 266                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 267                 if default_opts:
 268                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 269
 270         if opts.mode == 'help':
 271                 parser.print_help(short=False)
 272
 273         for arg in args:
 274                 if arg in modes:
 275                         if not opts.mode:
 276                                 opts.mode = arg
 277                                 break
 278                 else:
 279                         parser.error("invalid mode: %s" % arg)
 280
 281         if not opts.mode:
 282                 opts.mode = 'full'
 283
 284         if opts.mode == 'ci':
 285                 opts.mode = 'commit'  # backwards compat shortcut
 286
 287         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 288                 if opts.ignore_masked:
 289                         parser.error('Commit mode and --ignore-masked are not compatible')
 290                 if opts.without_mask:
 291                         parser.error('Commit mode and --without-mask are not compatible')
 292
 293         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 294         for val in range(opts.verbosity):
 295                 logger = logging.getLogger()
 296                 logger.setLevel(logger.getEffectiveLevel() - 10)
 297
 298         for val in range(opts.quiet):
 299                 logger = logging.getLogger()
 300                 logger.setLevel(logger.getEffectiveLevel() + 10)
 301
 302         return (opts, args)
 303
 304 qahelp={
 305         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 306         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 307         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 308         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 309         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 310         "changelog.missing":"Missing ChangeLog files",
 311         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 312         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 313         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 314         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 315         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 316         "file.size":"Files in the files directory must be under 20 KiB",
 317         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 318         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 319         "file.UTF8":"File is not UTF8 compliant",
 320         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 321         "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it",
 322         "inherit.unused":"Ebuild inherits an eclass but does not use it",
 323         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 324         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 325         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 326         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 327         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 328         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 329         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 330         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 331         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 332         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 333         "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)",
 334         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 335         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 336         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 337         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 338         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 339         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 340         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 341         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 342         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 343         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 344         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 345         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 346         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 347         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 348         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 349         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 350         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 351         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 352         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 353         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 354         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 355         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 356         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 357         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 358         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 359         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 360         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 361         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 362         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 363         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 364         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 365         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 366         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 367         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 368         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 369         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 370         "variable.readonly":"Assigning a readonly variable",
 371         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 372         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 373         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 374         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 375         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 376         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 377         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 378         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 379         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 380         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 381         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 382         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 383         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 384         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 385         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 386         "ebuild.badheader":"This ebuild has a malformed header",
 387         "manifest.bad":"Manifest has missing or incorrect digests",
 388         "metadata.missing":"Missing metadata.xml files",
 389         "metadata.bad":"Bad metadata.xml files",
 390         "metadata.warning":"Warnings in metadata.xml files",
 391         "portage.internal":"The ebuild uses an internal Portage function",
 392         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 393         "virtual.suspect":"Ebuild contains a package that usually should be pulled via virtual/, not directly.",
 394         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 395         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 396 }
 397
 398 qacats = list(qahelp)
 399 qacats.sort()
 400
 401 qawarnings = set((
 402 "changelog.missing",
 403 "changelog.notadded",
 404 "dependency.unknown",
 405 "digest.assumed",
 406 "digest.unused",
 407 "ebuild.notadded",
 408 "ebuild.nesteddie",
 409 "desktop.invalid",
 410 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 411 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 412 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 413 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 414 "DESCRIPTION.toolong",
 415 "EAPI.deprecated",
 416 "HOMEPAGE.virtual",
 417 "LICENSE.virtual",
 418 "KEYWORDS.dropped",
 419 "KEYWORDS.stupid",
 420 "KEYWORDS.missing",
 421 "PDEPEND.suspect",
 422 "RDEPEND.implicit",
 423 "RDEPEND.suspect",
 424 "virtual.suspect",
 425 "RESTRICT.invalid",
 426 "ebuild.minorsyn",
 427 "ebuild.badheader",
 428 "ebuild.patches",
 429 "file.size",
 430 "inherit.unused",
 431 "inherit.deprecated",
 432 "java.eclassesnotused",
 433 "wxwidgets.eclassnotused",
 434 "metadata.warning",
 435 "portage.internal",
 436 "usage.obsolete",
 437 "upstream.workaround",
 438 "LIVEVCS.stable",
 439 "LIVEVCS.unmasked",
 440 ))
 441
 442 if portage.const._ENABLE_INHERIT_CHECK:
 443         # This is experimental, so it's non-fatal.
 444         qawarnings.add("inherit.missing")
 445
 446 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 447
 448 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 449 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 450 allvars.update(Package.metadata_keys)
 451 allvars = sorted(allvars)
 452 commitmessage=None
 453 for x in missingvars:
 454         x += ".missing"
 455         if x not in qacats:
 456                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 457                 qacats.append(x)
 458                 qawarnings.add(x)
 459
 460 valid_restrict = frozenset(["binchecks", "bindist",
 461         "fetch", "installsources", "mirror",
 462         "primaryuri", "strip", "test", "userpriv"])
 463
 464 live_eclasses = frozenset([
 465         "bzr",
 466         "cvs",
 467         "darcs",
 468         "git",
 469         "git-2",
 470         "mercurial",
 471         "subversion",
 472         "tla",
 473 ])
 474
 475 suspect_rdepend = frozenset([
 476         "app-arch/cabextract",
 477         "app-arch/rpm2targz",
 478         "app-doc/doxygen",
 479         "dev-lang/nasm",
 480         "dev-lang/swig",
 481         "dev-lang/yasm",
 482         "dev-perl/extutils-pkgconfig",
 483         "dev-util/byacc",
 484         "dev-util/cmake",
 485         "dev-util/ftjam",
 486         "dev-util/gperf",
 487         "dev-util/gtk-doc",
 488         "dev-util/gtk-doc-am",
 489         "dev-util/intltool",
 490         "dev-util/jam",
 491         "dev-util/pkg-config-lite",
 492         "dev-util/pkgconf",
 493         "dev-util/pkgconfig",
 494         "dev-util/pkgconfig-openbsd",
 495         "dev-util/scons",
 496         "dev-util/unifdef",
 497         "dev-util/yacc",
 498         "media-gfx/ebdftopcf",
 499         "sys-apps/help2man",
 500         "sys-devel/autoconf",
 501         "sys-devel/automake",
 502         "sys-devel/bin86",
 503         "sys-devel/bison",
 504         "sys-devel/dev86",
 505         "sys-devel/flex",
 506         "sys-devel/m4",
 507         "sys-devel/pmake",
 508         "virtual/linux-sources",
 509         "virtual/pkgconfig",
 510         "x11-misc/bdftopcf",
 511         "x11-misc/imake",
 512 ])
 513
 514 suspect_virtual = {
 515         "dev-util/pkg-config-lite":"virtual/pkgconfig",
 516         "dev-util/pkgconf":"virtual/pkgconfig",
 517         "dev-util/pkgconfig":"virtual/pkgconfig",
 518         "dev-util/pkgconfig-openbsd":"virtual/pkgconfig",
 519         "dev-libs/libusb":"virtual/libusb",
 520         "dev-libs/libusbx":"virtual/libusb",
 521         "dev-libs/libusb-compat":"virtual/libusb",
 522 }
 523
 524 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 525 # force refetch if the local copy creation time is older than this
 526 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 527
 528 # file.executable
 529 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 530
 531 options, arguments = ParseArgs(sys.argv, qahelp)
 532
 533 if options.version:
 534         print("Portage", portage.VERSION)
 535         sys.exit(0)
 536
 537 # Set this to False when an extraordinary issue (generally
 538 # something other than a QA issue) makes it impossible to
 539 # commit (like if Manifest generation fails).
 540 can_force = True
 541
 542 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 543 if portdir is None:
 544         sys.exit(1)
 545
 546 myreporoot = os.path.basename(portdir_overlay)
 547 myreporoot += mydir[len(portdir_overlay):]
 548
 549 if options.vcs:
 550         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 551                 vcs = options.vcs
 552         else:
 553                 vcs = None
 554 else:
 555         vcses = utilities.FindVCS()
 556         if len(vcses) > 1:
 557                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 558                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 559                 sys.exit(1)
 560         elif vcses:
 561                 vcs = vcses[0]
 562         else:
 563                 vcs = None
 564
 565 if options.if_modified == "y" and vcs is None:
 566         logging.info("Not in a version controlled repository; "
 567                 "disabling --if-modified.")
 568         options.if_modified = "n"
 569
 570 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 571 vcs_preserves_mtime = vcs in ('cvs',)
 572
 573 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 574 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 575 if vcs_global_opts is None:
 576         if vcs in ('cvs', 'svn'):
 577                 vcs_global_opts = "-q"
 578         else:
 579                 vcs_global_opts = ""
 580 vcs_global_opts = vcs_global_opts.split()
 581
 582 if options.mode == 'commit' and not options.pretend and not vcs:
 583         logging.info("Not in a version controlled repository; enabling pretend mode.")
 584         options.pretend = True
 585
 586 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 587 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 588         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 589         portage._shell_quote(portdir_overlay))
 590 # We have to call the config constructor again so
 591 # that config.repositories is initialized correctly.
 592 repoman_settings = portage.config(config_root=config_root, local_config=False,
 593         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 594
 595 root = repoman_settings['EROOT']
 596 trees = {
 597         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 598 }
 599 portdb = trees[root]['porttree'].dbapi
 600
 601 # Constrain dependency resolution to the master(s)
 602 # that are specified in layout.conf.
 603 repodir = os.path.realpath(portdir_overlay)
 604 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 605 portdb.porttrees = list(repo_config.eclass_db.porttrees)
 606 portdir = portdb.porttrees[0]
 607
 608 if repo_config.allow_provide_virtual:
 609         qawarnings.add("virtual.oldstyle")
 610
 611 if repo_config.sign_commit:
 612         if vcs == 'git':
 613                 # NOTE: It's possible to use --gpg-sign=key_id to specify the key in
 614                 # the commit arguments. If key_id is unspecified, then it must be
 615                 # configured by `git config user.signingkey key_id`.
 616                 vcs_local_opts.append("--gpg-sign")
 617
 618 # In order to disable manifest signatures, repos may set
 619 # "sign-manifests = false" in metadata/layout.conf. This
 620 # can be used to prevent merge conflicts like those that
 621 # thin-manifests is designed to prevent.
 622 sign_manifests = "sign" in repoman_settings.features and \
 623         repo_config.sign_manifest
 624
 625 manifest_hashes = repo_config.manifest_hashes
 626 if manifest_hashes is None:
 627         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 628
 629 if options.mode in ("commit", "fix", "manifest"):
 630         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 631                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 632                 "metadata/layout.conf does not contain the '%s' hash which "
 633                 "is required by this portage version. You will have to "
 634                 "upgrade portage if you want to generate valid manifests for "
 635                 "this repository.") % \
 636                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 637                 for line in textwrap.wrap(msg, 70):
 638                         logging.error(line)
 639                 sys.exit(1)
 640
 641         unsupported_hashes = manifest_hashes.difference(
 642                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 643         if unsupported_hashes:
 644                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 645                 "metadata/layout.conf contains one or more hash types '%s' "
 646                 "which are not supported by this portage version. You will "
 647                 "have to upgrade portage if you want to generate valid "
 648                 "manifests for this repository.") % \
 649                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 650                 for line in textwrap.wrap(msg, 70):
 651                         logging.error(line)
 652                 sys.exit(1)
 653
 654 if "commit" == options.mode and \
 655         repo_config.name == "gentoo" and \
 656         "RMD160" in manifest_hashes and \
 657         "RMD160" not in portage.checksum.hashorigin_map:
 658         msg = "Please install " \
 659         "pycrypto or enable python's ssl USE flag in order " \
 660         "to enable RMD160 hash support. See bug #198398 for " \
 661         "more information."
 662         prefix = bad(" * ")
 663         for line in textwrap.wrap(msg, 70):
 664                 print(prefix + line)
 665         sys.exit(1)
 666
 667 if options.echangelog is None and repo_config.update_changelog:
 668         options.echangelog = 'y'
 669
 670 if vcs is None:
 671         options.echangelog = 'n'
 672
 673 # The --echangelog option causes automatic ChangeLog generation,
 674 # which invalidates changelog.ebuildadded and changelog.missing
 675 # checks.
 676 # Note: Some don't use ChangeLogs in distributed SCMs.
 677 # It will be generated on server side from scm log,
 678 # before package moves to the rsync server.
 679 # This is needed because they try to avoid merge collisions.
 680 # Gentoo's Council decided to always use the ChangeLog file.
 681 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 682 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 683
 684 if 'digest' in repoman_settings.features and options.digest != 'n':
 685         options.digest = 'y'
 686
 687 logging.debug("vcs: %s" % (vcs,))
 688 logging.debug("repo config: %s" % (repo_config,))
 689 logging.debug("options: %s" % (options,))
 690
 691 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 692 # profile-specific config constructor calls.
 693 env = os.environ.copy()
 694 env['PORTDIR'] = portdir
 695 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 696
 697 logging.info('Setting paths:')
 698 logging.info('PORTDIR = "' + portdir + '"')
 699 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 700
 701 # It's confusing if these warnings are displayed without the user
 702 # being told which profile they come from, so disable them.
 703 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 704
 705 categories = []
 706 for path in repo_config.eclass_db.porttrees:
 707         categories.extend(portage.util.grabfile(
 708                 os.path.join(path, 'profiles', 'categories')))
 709 repoman_settings.categories = frozenset(
 710         portage.util.stack_lists([categories], incremental=1))
 711 categories = repoman_settings.categories
 712
 713 portdb.settings = repoman_settings
 714 root_config = RootConfig(repoman_settings, trees[root], None)
 715 # We really only need to cache the metadata that's necessary for visibility
 716 # filtering. Anything else can be discarded to reduce memory consumption.
 717 portdb._aux_cache_keys.clear()
 718 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 719
 720 reposplit = myreporoot.split(os.path.sep)
 721 repolevel = len(reposplit)
 722
 723 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 724 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 725 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 726 if options.mode == 'commit' and repolevel not in [1,2,3]:
 727         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 728         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 729         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 730         print(red("***"))
 731         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 732
 733 # Make startdir relative to the canonical repodir, so that we can pass
 734 # it to digestgen and it won't have to be canonicalized again.
 735 if repolevel == 1:
 736         startdir = repodir
 737 else:
 738         startdir = normalize_path(mydir)
 739         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 740
 741 def caterror(mycat):
 742         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 743
 744 def repoman_getstatusoutput(cmd):
 745         """
 746         Implements an interface similar to getstatusoutput(), but with
 747         customized unicode handling (see bug #310789) and without the shell.
 748         """
 749         args = portage.util.shlex_split(cmd)
 750         encoding = _encodings['fs']
 751         if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 752                 # Python 3.1 does not support bytes in Popen args.
 753                 args = [_unicode_encode(x,
 754                         encoding=encoding, errors='strict') for x in args]
 755         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
 756                 stderr=subprocess.STDOUT)
 757         output = portage._unicode_decode(proc.communicate()[0],
 758                 encoding=encoding, errors='strict')
 759         if output and output[-1] == "\n":
 760                 # getstatusoutput strips one newline
 761                 output = output[:-1]
 762         return (proc.wait(), output)
 763
 764 class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 765         """
 766         Implements an interface similar to os.popen(), but with customized
 767         unicode handling (see bug #310789) and without the shell.
 768         """
 769
 770         __slots__ = ('_proc', '_stdout')
 771
 772         def __init__(self, cmd):
 773                 args = portage.util.shlex_split(cmd)
 774                 encoding = _encodings['fs']
 775                 if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
 776                         # Python 3.1 does not support bytes in Popen args.
 777                         args = [_unicode_encode(x,
 778                                 encoding=encoding, errors='strict') for x in args]
 779                 proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 780                 object.__setattr__(self, '_proc', proc)
 781                 object.__setattr__(self, '_stdout',
 782                         codecs.getreader(encoding)(proc.stdout, 'strict'))
 783
 784         def _get_target(self):
 785                 return object.__getattribute__(self, '_stdout')
 786
 787         __enter__ = _get_target
 788
 789         def __exit__(self, exc_type, exc_value, traceback):
 790                 proc = object.__getattribute__(self, '_proc')
 791                 proc.wait()
 792                 proc.stdout.close()
 793
 794 class ProfileDesc(object):
 795         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 796         def __init__(self, arch, status, sub_path, tree_path):
 797                 self.arch = arch
 798                 self.status = status
 799                 if sub_path:
 800                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 801                 self.sub_path = sub_path
 802                 self.tree_path = tree_path
 803                 if tree_path:
 804                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 805                 else:
 806                         self.abs_path = tree_path
 807
 808         def __str__(self):
 809                 if self.sub_path:
 810                         return self.sub_path
 811                 return 'empty profile'
 812
 813 profile_list = []
 814 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 815
 816 # get lists of valid keywords, licenses, and use
 817 kwlist = set()
 818 liclist = set()
 819 uselist = set()
 820 global_pmasklines = []
 821
 822 for path in portdb.porttrees:
 823         try:
 824                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 825         except OSError:
 826                 pass
 827         kwlist.update(portage.grabfile(os.path.join(path,
 828                 "profiles", "arch.list")))
 829
 830         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 831         for x in use_desc:
 832                 x = x.split()
 833                 if x:
 834                         uselist.add(x[0])
 835
 836         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 837         try:
 838                 expand_list = os.listdir(expand_desc_dir)
 839         except OSError:
 840                 pass
 841         else:
 842                 for fn in expand_list:
 843                         if not fn[-5:] == '.desc':
 844                                 continue
 845                         use_prefix = fn[:-5].lower() + '_'
 846                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 847                                 x = x.split()
 848                                 if x:
 849                                         uselist.add(use_prefix + x[0])
 850
 851         global_pmasklines.append(portage.util.grabfile_package(
 852                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 853
 854         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 855         try:
 856                 desc_file = io.open(_unicode_encode(desc_path,
 857                         encoding=_encodings['fs'], errors='strict'),
 858                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 859         except EnvironmentError:
 860                 pass
 861         else:
 862                 for i, x in enumerate(desc_file):
 863                         if x[0] == "#":
 864                                 continue
 865                         arch = x.split()
 866                         if len(arch) == 0:
 867                                 continue
 868                         if len(arch) != 3:
 869                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 870                                         desc_path + " line %d" % (i+1, ))
 871                         elif arch[0] not in kwlist:
 872                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 873                                         desc_path + " line %d" % (i+1, ))
 874                         elif arch[2] not in valid_profile_types:
 875                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 876                                         desc_path + " line %d" % (i+1, ))
 877                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 878                         if not os.path.isdir(profile_desc.abs_path):
 879                                 logging.error(
 880                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 881                                         arch[2], arch[1], arch[0], desc_path, i+1)
 882                                 continue
 883                         if os.path.exists(
 884                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 885                                 continue
 886                         profile_list.append(profile_desc)
 887                 desc_file.close()
 888
 889 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 890 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 891
 892 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 893 global_pmaskdict = {}
 894 for x in global_pmasklines:
 895         global_pmaskdict.setdefault(x.cp, []).append(x)
 896 del global_pmasklines
 897
 898 def has_global_mask(pkg):
 899         mask_atoms = global_pmaskdict.get(pkg.cp)
 900         if mask_atoms:
 901                 pkg_list = [pkg]
 902                 for x in mask_atoms:
 903                         if portage.dep.match_from_list(x, pkg_list):
 904                                 return x
 905         return None
 906
 907 # Ensure that profile sub_path attributes are unique. Process in reverse order
 908 # so that profiles with duplicate sub_path from overlays will override
 909 # profiles with the same sub_path from parent repos.
 910 profiles = {}
 911 profile_list.reverse()
 912 profile_sub_paths = set()
 913 for prof in profile_list:
 914         if prof.sub_path in profile_sub_paths:
 915                 continue
 916         profile_sub_paths.add(prof.sub_path)
 917         profiles.setdefault(prof.arch, []).append(prof)
 918
 919 # Use an empty profile for checking dependencies of
 920 # packages that have empty KEYWORDS.
 921 prof = ProfileDesc('**', 'stable', '', '')
 922 profiles.setdefault(prof.arch, []).append(prof)
 923
 924 for x in repoman_settings.archlist():
 925         if x[0] == "~":
 926                 continue
 927         if x not in profiles:
 928                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 929                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 930                 print(red("up with the "+x+" team."))
 931                 print()
 932
 933 if not liclist:
 934         logging.fatal("Couldn't find licenses?")
 935         sys.exit(1)
 936
 937 if not kwlist:
 938         logging.fatal("Couldn't read KEYWORDS from arch.list")
 939         sys.exit(1)
 940
 941 if not uselist:
 942         logging.fatal("Couldn't find use.desc?")
 943         sys.exit(1)
 944
 945 scanlist=[]
 946 if repolevel==2:
 947         #we are inside a category directory
 948         catdir=reposplit[-1]
 949         if catdir not in categories:
 950                 caterror(catdir)
 951         mydirlist=os.listdir(startdir)
 952         for x in mydirlist:
 953                 if x == "CVS" or x.startswith("."):
 954                         continue
 955                 if os.path.isdir(startdir+"/"+x):
 956                         scanlist.append(catdir+"/"+x)
 957         repo_subdir = catdir + os.sep
 958 elif repolevel==1:
 959         for x in categories:
 960                 if not os.path.isdir(startdir+"/"+x):
 961                         continue
 962                 for y in os.listdir(startdir+"/"+x):
 963                         if y == "CVS" or y.startswith("."):
 964                                 continue
 965                         if os.path.isdir(startdir+"/"+x+"/"+y):
 966                                 scanlist.append(x+"/"+y)
 967         repo_subdir = ""
 968 elif repolevel==3:
 969         catdir = reposplit[-2]
 970         if catdir not in categories:
 971                 caterror(catdir)
 972         scanlist.append(catdir+"/"+reposplit[-1])
 973         repo_subdir = scanlist[-1] + os.sep
 974 else:
 975         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 976                 ' from the current working directory'
 977         logging.critical(msg)
 978         sys.exit(1)
 979
 980 repo_subdir_len = len(repo_subdir)
 981 scanlist.sort()
 982
 983 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 984
 985 def vcs_files_to_cps(vcs_file_iter):
 986         """
 987         Iterate over the given modified file paths returned from the vcs,
 988         and return a frozenset containing category/pn strings for each
 989         modified package.
 990         """
 991
 992         modified_cps = []
 993
 994         if repolevel == 3:
 995                 if reposplit[-2] in categories and \
 996                         next(vcs_file_iter, None) is not None:
 997                         modified_cps.append("/".join(reposplit[-2:]))
 998
 999         elif repolevel == 2:
1000                 category = reposplit[-1]
1001                 if category in categories:
1002                         for filename in vcs_file_iter:
1003                                 f_split = filename.split(os.sep)
1004                                 # ['.', pn,...]
1005                                 if len(f_split) > 2:
1006                                         modified_cps.append(category + "/" + f_split[1])
1007
1008         else:
1009                 # repolevel == 1
1010                 for filename in vcs_file_iter:
1011                         f_split = filename.split(os.sep)
1012                         # ['.', category, pn,...]
1013                         if len(f_split) > 3 and f_split[1] in categories:
1014                                 modified_cps.append("/".join(f_split[1:3]))
1015
1016         return frozenset(modified_cps)
1017
1018 def git_supports_gpg_sign():
1019         status, cmd_output = \
1020                 repoman_getstatusoutput("git --version")
1021         cmd_output = cmd_output.split()
1022         if cmd_output:
1023                 version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1])
1024                 if version is not None:
1025                         version = [int(x) for x in version.groups()[1:]]
1026                         if version[0] > 1 or \
1027                                 (version[0] == 1 and version[1] > 7) or \
1028                                 (version[0] == 1 and version[1] == 7 and version[2] >= 9):
1029                                 return True
1030         return False
1031
1032 def dev_keywords(profiles):
1033         """
1034         Create a set of KEYWORDS values that exist in 'dev'
1035         profiles. These are used
1036         to trigger a message notifying the user when they might
1037         want to add the --include-dev option.
1038         """
1039         type_arch_map = {}
1040         for arch, arch_profiles in profiles.items():
1041                 for prof in arch_profiles:
1042                         arch_set = type_arch_map.get(prof.status)
1043                         if arch_set is None:
1044                                 arch_set = set()
1045                                 type_arch_map[prof.status] = arch_set
1046                         arch_set.add(arch)
1047
1048         dev_keywords = type_arch_map.get('dev', set())
1049         dev_keywords.update(['~' + arch for arch in dev_keywords])
1050         return frozenset(dev_keywords)
1051
1052 dev_keywords = dev_keywords(profiles)
1053
1054 stats={}
1055 fails={}
1056
1057 # provided by the desktop-file-utils package
1058 desktop_file_validate = find_binary("desktop-file-validate")
1059 desktop_pattern = re.compile(r'.*\.desktop$')
1060
1061 for x in qacats:
1062         stats[x]=0
1063         fails[x]=[]
1064
1065 xmllint_capable = False
1066 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
1067
1068 def parsedate(s):
1069         """Parse a RFC 822 date and time string.
1070         This is required for python3 compatibility, since the
1071         rfc822.parsedate() function is not available."""
1072
1073         s_split = []
1074         for x in s.upper().split():
1075                 for y in x.split(','):
1076                         if y:
1077                                 s_split.append(y)
1078
1079         if len(s_split) != 6:
1080                 return None
1081
1082         # %a, %d %b %Y %H:%M:%S %Z
1083         a, d, b, Y, H_M_S, Z = s_split
1084
1085         # Convert month to integer, since strptime %w is locale-dependent.
1086         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
1087                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
1088         m = month_map.get(b)
1089         if m is None:
1090                 return None
1091         m = str(m).rjust(2, '0')
1092
1093         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
1094
1095 def fetch_metadata_dtd():
1096         """
1097         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1098         metadata_dtd_ctime_interval.
1099         @rtype: bool
1100         @return: True if successful, otherwise False
1101         """
1102
1103         must_fetch = True
1104         metadata_dtd_st = None
1105         current_time = int(time.time())
1106         try:
1107                 metadata_dtd_st = os.stat(metadata_dtd)
1108         except EnvironmentError as e:
1109                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1110                         raise
1111                 del e
1112         else:
1113                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1114                 if abs(current_time - metadata_dtd_st.st_ctime) \
1115                         < metadata_dtd_ctime_interval:
1116                         must_fetch = False
1117
1118         if must_fetch:
1119                 print()
1120                 print(green("***") + " the local copy of metadata.dtd " + \
1121                         "needs to be refetched, doing that now")
1122                 print()
1123                 try:
1124                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1125                         msg_info = url_f.info()
1126                         last_modified = msg_info.get('last-modified')
1127                         if last_modified is not None:
1128                                 last_modified = parsedate(last_modified)
1129                                 if last_modified is not None:
1130                                         last_modified = calendar.timegm(last_modified)
1131
1132                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1133                         try:
1134                                 local_f = open(metadata_dtd_tmp, mode='wb')
1135                                 local_f.write(url_f.read())
1136                                 local_f.close()
1137                                 if last_modified is not None:
1138                                         try:
1139                                                 os.utime(metadata_dtd_tmp,
1140                                                         (int(last_modified), int(last_modified)))
1141                                         except OSError:
1142                                                 # This fails on some odd non-unix-like filesystems.
1143                                                 # We don't really need the mtime to be preserved
1144                                                 # anyway here (currently we use ctime to trigger
1145                                                 # fetch), so just ignore it.
1146                                                 pass
1147                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1148                         finally:
1149                                 try:
1150                                         os.unlink(metadata_dtd_tmp)
1151                                 except OSError:
1152                                         pass
1153
1154                         url_f.close()
1155
1156                 except EnvironmentError as e:
1157                         print()
1158                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1159                         print(red("!!!")+" exception '%s' though." % (e,))
1160                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1161                         return False
1162
1163         return True
1164
1165 if options.mode == "manifest":
1166         pass
1167 elif not find_binary('xmllint'):
1168         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1169         if options.xml_parse or repolevel==3:
1170                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1171                 sys.exit(1)
1172 else:
1173         if not fetch_metadata_dtd():
1174                 sys.exit(1)
1175         #this can be problematic if xmllint changes their output
1176         xmllint_capable=True
1177
1178 if options.mode == 'commit' and vcs:
1179         utilities.detect_vcs_conflicts(options, vcs)
1180
1181 if options.mode == "manifest":
1182         pass
1183 elif options.pretend:
1184         print(green("\nRepoMan does a once-over of the neighborhood..."))
1185 else:
1186         print(green("\nRepoMan scours the neighborhood..."))
1187
1188 new_ebuilds = set()
1189 modified_ebuilds = set()
1190 modified_changelogs = set()
1191 mychanged = []
1192 mynew = []
1193 myremoved = []
1194
1195 if vcs == "cvs":
1196         mycvstree = cvstree.getentries("./", recursive=1)
1197         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1198         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1199         if options.if_modified == "y":
1200                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1201
1202 elif vcs == "svn":
1203         with repoman_popen("svn status") as f:
1204                 svnstatus = f.readlines()
1205         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1206         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1207         if options.if_modified == "y":
1208                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1209
1210 elif vcs == "git":
1211         with repoman_popen("git diff-index --name-only "
1212                 "--relative --diff-filter=M HEAD") as f:
1213                 mychanged = f.readlines()
1214         mychanged = ["./" + elem[:-1] for elem in mychanged]
1215
1216         with repoman_popen("git diff-index --name-only "
1217                 "--relative --diff-filter=A HEAD") as f:
1218                 mynew = f.readlines()
1219         mynew = ["./" + elem[:-1] for elem in mynew]
1220         if options.if_modified == "y":
1221                 with repoman_popen("git diff-index --name-only "
1222                         "--relative --diff-filter=D HEAD") as f:
1223                         myremoved = f.readlines()
1224                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1225
1226 elif vcs == "bzr":
1227         with repoman_popen("bzr status -S .") as f:
1228                 bzrstatus = f.readlines()
1229         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1230         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1231         if options.if_modified == "y":
1232                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1233
1234 elif vcs == "hg":
1235         with repoman_popen("hg status --no-status --modified .") as f:
1236                 mychanged = f.readlines()
1237         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1238         mynew = repoman_popen("hg status --no-status --added .").readlines()
1239         mynew = ["./" + elem.rstrip() for elem in mynew]
1240         if options.if_modified == "y":
1241                 with repoman_popen("hg status --no-status --removed .") as f:
1242                         myremoved = f.readlines()
1243                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1244
1245 if vcs:
1246         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1247         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1248         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1249                 if os.path.basename(x) == "ChangeLog")
1250
1251 def vcs_new_changed(relative_path):
1252         for x in chain(mychanged, mynew):
1253                 if x == relative_path:
1254                         return True
1255         return False
1256
1257 have_pmasked = False
1258 have_dev_keywords = False
1259 dofail = 0
1260
1261 # NOTE: match-all caches are not shared due to potential
1262 # differences between profiles in _get_implicit_iuse.
1263 arch_caches={}
1264 arch_xmatch_caches = {}
1265 shared_xmatch_caches = {"cp-list":{}}
1266
1267 # Disable the "ebuild.notadded" check when not in commit mode and
1268 # running `svn status` in every package dir will be too expensive.
1269
1270 check_ebuild_notadded = not \
1271         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1272
1273 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1274 thirdpartymirrors = []
1275 for v in repoman_settings.thirdpartymirrors().values():
1276         for v in v:
1277                 if not v.endswith("/"):
1278                         v += "/"
1279                 thirdpartymirrors.append(v)
1280
1281 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1282         """
1283         Implements doctype() as required to avoid deprecation warnings with
1284         >=python-2.7.
1285         """
1286         def doctype(self, name, pubid, system):
1287                 pass
1288
1289 try:
1290         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1291 except (EnvironmentError, ParseError, PermissionDenied) as e:
1292         err(str(e))
1293 except FileNotFound:
1294         # TODO: Download as we do for metadata.dtd, but add a way to
1295         # disable for non-gentoo repoman users who may not have herds.
1296         herd_base = None
1297
1298 effective_scanlist = scanlist
1299 if options.if_modified == "y":
1300         effective_scanlist = sorted(vcs_files_to_cps(
1301                 chain(mychanged, mynew, myremoved)))
1302
1303 for x in effective_scanlist:
1304         #ebuilds and digests added to cvs respectively.
1305         logging.info("checking package %s" % x)
1306         # save memory by discarding xmatch caches from previous package(s)
1307         arch_xmatch_caches.clear()
1308         eadded=[]
1309         catdir,pkgdir=x.split("/")
1310         checkdir=repodir+"/"+x
1311         checkdir_relative = ""
1312         if repolevel < 3:
1313                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1314         if repolevel < 2:
1315                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1316         checkdir_relative = os.path.join(".", checkdir_relative)
1317         generated_manifest = False
1318
1319         if options.mode == "manifest" or \
1320           (options.mode != 'manifest-check' and options.digest == 'y') or \
1321           options.mode in ('commit', 'fix') and not options.pretend:
1322                 auto_assumed = set()
1323                 fetchlist_dict = portage.FetchlistDict(checkdir,
1324                         repoman_settings, portdb)
1325                 if options.mode == 'manifest' and options.force:
1326                         portage._doebuild_manifest_exempt_depend += 1
1327                         try:
1328                                 distdir = repoman_settings['DISTDIR']
1329                                 mf = repoman_settings.repositories.get_repo_for_location(
1330                                         os.path.dirname(os.path.dirname(checkdir)))
1331                                 mf = mf.load_manifest(checkdir, distdir,
1332                                         fetchlist_dict=fetchlist_dict)
1333                                 mf.create(requiredDistfiles=None,
1334                                         assumeDistHashesAlways=True)
1335                                 for distfiles in fetchlist_dict.values():
1336                                         for distfile in distfiles:
1337                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1338                                                         mf.fhashdict['DIST'].pop(distfile, None)
1339                                                 else:
1340                                                         auto_assumed.add(distfile)
1341                                 mf.write()
1342                         finally:
1343                                 portage._doebuild_manifest_exempt_depend -= 1
1344
1345                 repoman_settings["O"] = checkdir
1346                 try:
1347                         generated_manifest = digestgen(
1348                                 mysettings=repoman_settings, myportdb=portdb)
1349                 except portage.exception.PermissionDenied as e:
1350                         generated_manifest = False
1351                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1352                                 level=logging.ERROR, noiselevel=-1)
1353
1354                 if not generated_manifest:
1355                         print("Unable to generate manifest.")
1356                         dofail = 1
1357
1358                 if options.mode == "manifest":
1359                         if not dofail and options.force and auto_assumed and \
1360                                 'assume-digests' in repoman_settings.features:
1361                                 # Show which digests were assumed despite the --force option
1362                                 # being given. This output will already have been shown by
1363                                 # digestgen() if assume-digests is not enabled, so only show
1364                                 # it here if assume-digests is enabled.
1365                                 pkgs = list(fetchlist_dict)
1366                                 pkgs.sort()
1367                                 portage.writemsg_stdout("  digest.assumed" + \
1368                                         portage.output.colorize("WARN",
1369                                         str(len(auto_assumed)).rjust(18)) + "\n")
1370                                 for cpv in pkgs:
1371                                         fetchmap = fetchlist_dict[cpv]
1372                                         pf = portage.catsplit(cpv)[1]
1373                                         for distfile in sorted(fetchmap):
1374                                                 if distfile in auto_assumed:
1375                                                         portage.writemsg_stdout(
1376                                                                 "   %s::%s\n" % (pf, distfile))
1377                         continue
1378                 elif dofail:
1379                         sys.exit(1)
1380
1381         if not generated_manifest:
1382                 repoman_settings['O'] = checkdir
1383                 repoman_settings['PORTAGE_QUIET'] = '1'
1384                 if not portage.digestcheck([], repoman_settings, strict=1):
1385                         stats["manifest.bad"] += 1
1386                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1387                 repoman_settings.pop('PORTAGE_QUIET', None)
1388
1389         if options.mode == 'manifest-check':
1390                 continue
1391
1392         checkdirlist=os.listdir(checkdir)
1393         ebuildlist=[]
1394         pkgs = {}
1395         allvalid = True
1396         for y in checkdirlist:
1397                 if (y in no_exec or y.endswith(".ebuild")) and \
1398                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1399                                 stats["file.executable"] += 1
1400                                 fails["file.executable"].append(os.path.join(checkdir, y))
1401                 if y.endswith(".ebuild"):
1402                         pf = y[:-7]
1403                         ebuildlist.append(pf)
1404                         cpv = "%s/%s" % (catdir, pf)
1405                         try:
1406                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1407                         except KeyError:
1408                                 allvalid = False
1409                                 stats["ebuild.syntax"] += 1
1410                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1411                                 continue
1412                         except IOError:
1413                                 allvalid = False
1414                                 stats["ebuild.output"] += 1
1415                                 fails["ebuild.output"].append(os.path.join(x, y))
1416                                 continue
1417                         if not portage.eapi_is_supported(myaux["EAPI"]):
1418                                 allvalid = False
1419                                 stats["EAPI.unsupported"] += 1
1420                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1421                                 continue
1422                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1423                                 root_config=root_config, type_name="ebuild")
1424
1425         slot_keywords = {}
1426
1427         if len(pkgs) != len(ebuildlist):
1428                 # If we can't access all the metadata then it's totally unsafe to
1429                 # commit since there's no way to generate a correct Manifest.
1430                 # Do not try to do any more QA checks on this package since missing
1431                 # metadata leads to false positives for several checks, and false
1432                 # positives confuse users.
1433                 can_force = False
1434                 continue
1435
1436         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1437         ebuildlist = sorted(pkgs.values())
1438         ebuildlist = [pkg.pf for pkg in ebuildlist]
1439
1440         for y in checkdirlist:
1441                 m = disallowed_filename_chars_re.search(y.strip(os.sep))
1442                 if m is not None:
1443                         y_relative = os.path.join(checkdir_relative, y)
1444                         if vcs is not None and not vcs_new_changed(y_relative):
1445                                 # If the file isn't in the VCS new or changed set, then
1446                                 # assume that it's an irrelevant temporary file (Manifest
1447                                 # entries are not generated for file names containing
1448                                 # prohibited characters). See bug #406877.
1449                                 m = None
1450                 if m is not None:
1451                         stats["file.name"] += 1
1452                         fails["file.name"].append("%s/%s: char '%s'" % \
1453                                 (checkdir, y, m.group(0)))
1454
1455                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1456                         continue
1457                 f = None
1458                 try:
1459                         line = 1
1460                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1461                                 encoding=_encodings['fs'], errors='strict'),
1462                                 mode='r', encoding=_encodings['repo.content'])
1463                         for l in f:
1464                                 line +=1
1465                 except UnicodeDecodeError as ue:
1466                         stats["file.UTF8"] += 1
1467                         s = ue.object[:ue.start]
1468                         l2 = s.count("\n")
1469                         line += l2
1470                         if l2 != 0:
1471                                 s = s[s.rfind("\n") + 1:]
1472                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1473                 finally:
1474                         if f is not None:
1475                                 f.close()
1476
1477         if vcs in ("git", "hg") and check_ebuild_notadded:
1478                 if vcs == "git":
1479                         myf = repoman_popen("git ls-files --others %s" % \
1480                                 (portage._shell_quote(checkdir_relative),))
1481                 if vcs == "hg":
1482                         myf = repoman_popen("hg status --no-status --unknown %s" % \
1483                                 (portage._shell_quote(checkdir_relative),))
1484                 for l in myf:
1485                         if l[:-1][-7:] == ".ebuild":
1486                                 stats["ebuild.notadded"] += 1
1487                                 fails["ebuild.notadded"].append(
1488                                         os.path.join(x, os.path.basename(l[:-1])))
1489                 myf.close()
1490
1491         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1492                 try:
1493                         if vcs == "cvs":
1494                                 myf=open(checkdir+"/CVS/Entries","r")
1495                         if vcs == "svn":
1496                                 myf = repoman_popen("svn status --depth=files --verbose " +
1497                                         portage._shell_quote(checkdir))
1498                         if vcs == "bzr":
1499                                 myf = repoman_popen("bzr ls -v --kind=file " +
1500                                         portage._shell_quote(checkdir))
1501                         myl = myf.readlines()
1502                         myf.close()
1503                         for l in myl:
1504                                 if vcs == "cvs":
1505                                         if l[0]!="/":
1506                                                 continue
1507                                         splitl=l[1:].split("/")
1508                                         if not len(splitl):
1509                                                 continue
1510                                         if splitl[0][-7:]==".ebuild":
1511                                                 eadded.append(splitl[0][:-7])
1512                                 if vcs == "svn":
1513                                         if l[:1] == "?":
1514                                                 continue
1515                                         if l[:7] == '      >':
1516                                                 # tree conflict, new in subversion 1.6
1517                                                 continue
1518                                         l = l.split()[-1]
1519                                         if l[-7:] == ".ebuild":
1520                                                 eadded.append(os.path.basename(l[:-7]))
1521                                 if vcs == "bzr":
1522                                         if l[1:2] == "?":
1523                                                 continue
1524                                         l = l.split()[-1]
1525                                         if l[-7:] == ".ebuild":
1526                                                 eadded.append(os.path.basename(l[:-7]))
1527                         if vcs == "svn":
1528                                 myf = repoman_popen("svn status " +
1529                                         portage._shell_quote(checkdir))
1530                                 myl=myf.readlines()
1531                                 myf.close()
1532                                 for l in myl:
1533                                         if l[0] == "A":
1534                                                 l = l.rstrip().split(' ')[-1]
1535                                                 if l[-7:] == ".ebuild":
1536                                                         eadded.append(os.path.basename(l[:-7]))
1537                 except IOError:
1538                         if vcs == "cvs":
1539                                 stats["CVS/Entries.IO_error"] += 1
1540                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1541                         else:
1542                                 raise
1543                         continue
1544
1545         mf = repoman_settings.repositories.get_repo_for_location(
1546                 os.path.dirname(os.path.dirname(checkdir)))
1547         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1548         mydigests=mf.getTypeDigests("DIST")
1549
1550         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1551         myfiles_all = []
1552         src_uri_error = False
1553         for mykey in fetchlist_dict:
1554                 try:
1555                         myfiles_all.extend(fetchlist_dict[mykey])
1556                 except portage.exception.InvalidDependString as e:
1557                         src_uri_error = True
1558                         try:
1559                                 portdb.aux_get(mykey, ["SRC_URI"])
1560                         except KeyError:
1561                                 # This will be reported as an "ebuild.syntax" error.
1562                                 pass
1563                         else:
1564                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1565                                 fails["SRC_URI.syntax"].append(
1566                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1567         del fetchlist_dict
1568         if not src_uri_error:
1569                 # This test can produce false positives if SRC_URI could not
1570                 # be parsed for one or more ebuilds. There's no point in
1571                 # producing a false error here since the root cause will
1572                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1573                 # or "ebuild.sytax".
1574                 myfiles_all = set(myfiles_all)
1575                 for entry in mydigests:
1576                         if entry not in myfiles_all:
1577                                 stats["digest.unused"] += 1
1578                                 fails["digest.unused"].append(checkdir+"::"+entry)
1579                 for entry in myfiles_all:
1580                         if entry not in mydigests:
1581                                 stats["digest.missing"] += 1
1582                                 fails["digest.missing"].append(checkdir+"::"+entry)
1583         del myfiles_all
1584
1585         if os.path.exists(checkdir+"/files"):
1586                 filesdirlist=os.listdir(checkdir+"/files")
1587
1588                 # recurse through files directory
1589                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1590                 while filesdirlist:
1591                         y = filesdirlist.pop(0)
1592                         relative_path = os.path.join(x, "files", y)
1593                         full_path = os.path.join(repodir, relative_path)
1594                         try:
1595                                 mystat = os.stat(full_path)
1596                         except OSError as oe:
1597                                 if oe.errno == 2:
1598                                         # don't worry about it.  it likely was removed via fix above.
1599                                         continue
1600                                 else:
1601                                         raise oe
1602                         if S_ISDIR(mystat.st_mode):
1603                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1604                                 if y == "CVS" or y == ".svn":
1605                                         continue
1606                                 for z in os.listdir(checkdir+"/files/"+y):
1607                                         if z == "CVS" or z == ".svn":
1608                                                 continue
1609                                         filesdirlist.append(y+"/"+z)
1610                         # Current policy is no files over 20 KiB, these are the checks. File size between
1611                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1612                         elif mystat.st_size > 61440:
1613                                 stats["file.size.fatal"] += 1
1614                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1615                         elif mystat.st_size > 20480:
1616                                 stats["file.size"] += 1
1617                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1618
1619                         m = disallowed_filename_chars_re.search(
1620                                 os.path.basename(y.rstrip(os.sep)))
1621                         if m is not None:
1622                                 y_relative = os.path.join(checkdir_relative, "files", y)
1623                                 if vcs is not None and not vcs_new_changed(y_relative):
1624                                         # If the file isn't in the VCS new or changed set, then
1625                                         # assume that it's an irrelevant temporary file (Manifest
1626                                         # entries are not generated for file names containing
1627                                         # prohibited characters). See bug #406877.
1628                                         m = None
1629                         if m is not None:
1630                                 stats["file.name"] += 1
1631                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1632                                         (checkdir, y, m.group(0)))
1633
1634                         if desktop_file_validate and desktop_pattern.match(y):
1635                                 cmd_output = validate_desktop_entry(full_path)
1636                                 if cmd_output:
1637                                         # Note: in the future we may want to grab the
1638                                         # warnings in addition to the errors. We're
1639                                         # just doing errors now since we don't want
1640                                         # to generate too much noise at first.
1641                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1642                                         for line in cmd_output:
1643                                                 error_match = error_re.match(line)
1644                                                 if error_match is None:
1645                                                         continue
1646                                                 stats["desktop.invalid"] += 1
1647                                                 fails["desktop.invalid"].append(
1648                                                         relative_path + ': %s' % error_match.group(1))
1649
1650         del mydigests
1651
1652         if check_changelog and "ChangeLog" not in checkdirlist:
1653                 stats["changelog.missing"]+=1
1654                 fails["changelog.missing"].append(x+"/ChangeLog")
1655
1656         musedict = {}
1657         #metadata.xml file check
1658         if "metadata.xml" not in checkdirlist:
1659                 stats["metadata.missing"]+=1
1660                 fails["metadata.missing"].append(x+"/metadata.xml")
1661         #metadata.xml parse check
1662         else:
1663                 metadata_bad = False
1664
1665                 # read metadata.xml into memory
1666                 try:
1667                         _metadata_xml = xml.etree.ElementTree.parse(
1668                                 _unicode_encode(os.path.join(checkdir, "metadata.xml"),
1669                                 encoding=_encodings['fs'], errors='strict'),
1670                                 parser=xml.etree.ElementTree.XMLParser(
1671                                         target=_MetadataTreeBuilder()))
1672                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1673                         metadata_bad = True
1674                         stats["metadata.bad"] += 1
1675                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1676                         del e
1677                 else:
1678                         # load USE flags from metadata.xml
1679                         try:
1680                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1681                         except portage.exception.ParseError as e:
1682                                 metadata_bad = True
1683                                 stats["metadata.bad"] += 1
1684                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1685
1686                         # Run other metadata.xml checkers
1687                         try:
1688                                 utilities.check_metadata(_metadata_xml, herd_base)
1689                         except (utilities.UnknownHerdsError, ) as e:
1690                                 metadata_bad = True
1691                                 stats["metadata.bad"] += 1
1692                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1693                                 del e
1694
1695                 #Only carry out if in package directory or check forced
1696                 if xmllint_capable and not metadata_bad:
1697                         # xmlint can produce garbage output even on success, so only dump
1698                         # the ouput when it fails.
1699                         st, out = repoman_getstatusoutput(
1700                                 "xmllint --nonet --noout --dtdvalid %s %s" % \
1701                                  (portage._shell_quote(metadata_dtd),
1702                                  portage._shell_quote(os.path.join(checkdir, "metadata.xml"))))
1703                         if st != os.EX_OK:
1704                                 print(red("!!!") + " metadata.xml is invalid:")
1705                                 for z in out.splitlines():
1706                                         print(red("!!! ")+z)
1707                                 stats["metadata.bad"]+=1
1708                                 fails["metadata.bad"].append(x+"/metadata.xml")
1709
1710                 del metadata_bad
1711         muselist = frozenset(musedict)
1712
1713         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1714         changelog_modified = changelog_path in modified_changelogs
1715
1716         # detect unused local USE-descriptions
1717         used_useflags = set()
1718
1719         for y in ebuildlist:
1720                 relative_path = os.path.join(x, y + ".ebuild")
1721                 full_path = os.path.join(repodir, relative_path)
1722                 ebuild_path = y + ".ebuild"
1723                 if repolevel < 3:
1724                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1725                 if repolevel < 2:
1726                         ebuild_path = os.path.join(catdir, ebuild_path)
1727                 ebuild_path = os.path.join(".", ebuild_path)
1728                 if check_changelog and not changelog_modified \
1729                         and ebuild_path in new_ebuilds:
1730                         stats['changelog.ebuildadded'] += 1
1731                         fails['changelog.ebuildadded'].append(relative_path)
1732
1733                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1734                         #ebuild not added to vcs
1735                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1736                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1737                 myesplit=portage.pkgsplit(y)
1738                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1739                             or pv_toolong_re.search(myesplit[1]) \
1740                             or pv_toolong_re.search(myesplit[2]):
1741                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1742                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1743                         continue
1744                 elif myesplit[0]!=pkgdir:
1745                         print(pkgdir,myesplit[0])
1746                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1747                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1748                         continue
1749
1750                 pkg = pkgs[y]
1751
1752                 if pkg.invalid:
1753                         allvalid = False
1754                         for k, msgs in pkg.invalid.items():
1755                                 for msg in msgs:
1756                                         stats[k] = stats[k] + 1
1757                                         fails[k].append("%s %s" % (relative_path, msg))
1758                         continue
1759
1760                 myaux = pkg.metadata
1761                 eapi = myaux["EAPI"]
1762                 inherited = pkg.inherited
1763                 live_ebuild = live_eclasses.intersection(inherited)
1764
1765                 for k, v in myaux.items():
1766                         if not isinstance(v, basestring):
1767                                 continue
1768                         m = non_ascii_re.search(v)
1769                         if m is not None:
1770                                 stats["variable.invalidchar"] += 1
1771                                 fails["variable.invalidchar"].append(
1772                                         ("%s: %s variable contains non-ASCII " + \
1773                                         "character at position %s") % \
1774                                         (relative_path, k, m.start() + 1))
1775
1776                 if not src_uri_error:
1777                         # Check that URIs don't reference a server from thirdpartymirrors.
1778                         for uri in portage.dep.use_reduce( \
1779                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1780                                 contains_mirror = False
1781                                 for mirror in thirdpartymirrors:
1782                                         if uri.startswith(mirror):
1783                                                 contains_mirror = True
1784                                                 break
1785                                 if not contains_mirror:
1786                                         continue
1787
1788                                 stats["SRC_URI.mirror"] += 1
1789                                 fails["SRC_URI.mirror"].append(
1790                                         "%s: '%s' found in thirdpartymirrors" % \
1791                                         (relative_path, mirror))
1792
1793                 if myaux.get("PROVIDE"):
1794                         stats["virtual.oldstyle"]+=1
1795                         fails["virtual.oldstyle"].append(relative_path)
1796
1797                 for pos, missing_var in enumerate(missingvars):
1798                         if not myaux.get(missing_var):
1799                                 if catdir == "virtual" and \
1800                                         missing_var in ("HOMEPAGE", "LICENSE"):
1801                                         continue
1802                                 if live_ebuild and missing_var == "KEYWORDS":
1803                                         continue
1804                                 myqakey=missingvars[pos]+".missing"
1805                                 stats[myqakey]=stats[myqakey]+1
1806                                 fails[myqakey].append(x+"/"+y+".ebuild")
1807
1808                 if catdir == "virtual":
1809                         for var in ("HOMEPAGE", "LICENSE"):
1810                                 if myaux.get(var):
1811                                         myqakey = var + ".virtual"
1812                                         stats[myqakey] = stats[myqakey] + 1
1813                                         fails[myqakey].append(relative_path)
1814
1815                 # 14 is the length of DESCRIPTION=""
1816                 if len(myaux['DESCRIPTION']) > max_desc_len:
1817                         stats['DESCRIPTION.toolong'] += 1
1818                         fails['DESCRIPTION.toolong'].append(
1819                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1820                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1821
1822                 keywords = myaux["KEYWORDS"].split()
1823                 stable_keywords = []
1824                 for keyword in keywords:
1825                         if not keyword.startswith("~") and \
1826                                 not keyword.startswith("-"):
1827                                 stable_keywords.append(keyword)
1828                 if stable_keywords:
1829                         if ebuild_path in new_ebuilds:
1830                                 stable_keywords.sort()
1831                                 stats["KEYWORDS.stable"] += 1
1832                                 fails["KEYWORDS.stable"].append(
1833                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1834                                                 " ".join(stable_keywords))
1835
1836                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1837                         if not kw.startswith("-"))
1838
1839                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1840                 if previous_keywords is None:
1841                         slot_keywords[myaux["SLOT"]] = set()
1842                 elif ebuild_archs and not live_ebuild:
1843                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1844                         if dropped_keywords:
1845                                 stats["KEYWORDS.dropped"] += 1
1846                                 fails["KEYWORDS.dropped"].append(
1847                                         relative_path + ": %s" % \
1848                                         " ".join(sorted(dropped_keywords)))
1849
1850                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1851
1852                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1853                 if "-*" in keywords:
1854                         haskeyword = False
1855                         for kw in keywords:
1856                                 if kw[0] == "~":
1857                                         kw = kw[1:]
1858                                 if kw in kwlist:
1859                                         haskeyword = True
1860                         if not haskeyword:
1861                                 stats["KEYWORDS.stupid"] += 1
1862                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1863
1864                 """
1865                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1866                 not be allowed to be marked stable
1867                 """
1868                 if live_ebuild and repo_config.name == "gentoo":
1869                         bad_stable_keywords = []
1870                         for keyword in keywords:
1871                                 if not keyword.startswith("~") and \
1872                                         not keyword.startswith("-"):
1873                                         bad_stable_keywords.append(keyword)
1874                                 del keyword
1875                         if bad_stable_keywords:
1876                                 stats["LIVEVCS.stable"] += 1
1877                                 fails["LIVEVCS.stable"].append(
1878                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1879                                                 bad_stable_keywords)
1880                         del bad_stable_keywords
1881
1882                         if keywords and not has_global_mask(pkg):
1883                                 stats["LIVEVCS.unmasked"] += 1
1884                                 fails["LIVEVCS.unmasked"].append(relative_path)
1885
1886                 if options.ignore_arches:
1887                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1888                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1889                 else:
1890                         arches=[]
1891                         for keyword in myaux["KEYWORDS"].split():
1892                                 if (keyword[0]=="-"):
1893                                         continue
1894                                 elif (keyword[0]=="~"):
1895                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1896                                 else:
1897                                         arches.append([keyword, keyword, [keyword]])
1898                         if not arches:
1899                                 # Use an empty profile for checking dependencies of
1900                                 # packages that have empty KEYWORDS.
1901                                 arches.append(['**', '**', ['**']])
1902
1903                 unknown_pkgs = set()
1904                 baddepsyntax = False
1905                 badlicsyntax = False
1906                 badprovsyntax = False
1907                 catpkg = catdir+"/"+y
1908
1909                 inherited_java_eclass = "java-pkg-2" in inherited or \
1910                         "java-pkg-opt-2" in inherited
1911                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1912                 operator_tokens = set(["||", "(", ")"])
1913                 type_list, badsyntax = [], []
1914                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1915                         "LICENSE", "PROPERTIES", "PROVIDE"):
1916                         mydepstr = myaux[mytype]
1917
1918                         token_class = None
1919                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1920                                 token_class=portage.dep.Atom
1921
1922                         try:
1923                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1924                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1925                         except portage.exception.InvalidDependString as e:
1926                                 atoms = None
1927                                 badsyntax.append(str(e))
1928
1929                         if atoms and mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1930                                 if mytype in ("RDEPEND", "PDEPEND") and \
1931                                         "test?" in mydepstr.split():
1932                                         stats[mytype + '.suspect'] += 1
1933                                         fails[mytype + '.suspect'].append(relative_path + \
1934                                                 ": 'test?' USE conditional in %s" % mytype)
1935
1936                                 for atom in atoms:
1937                                         if atom == "||":
1938                                                 continue
1939
1940                                         # Skip dependency.unknown for blockers, so that we
1941                                         # don't encourage people to remove necessary blockers,
1942                                         # as discussed in bug #382407.
1943                                         if atom.blocker is None and \
1944                                                 not portdb.xmatch("match-all", atom) and \
1945                                                 not atom.cp.startswith("virtual/"):
1946                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1947
1948                                         is_blocker = atom.blocker
1949
1950                                         if catdir != "virtual":
1951                                                 if not is_blocker and \
1952                                                         atom.cp in suspect_virtual:
1953                                                         stats['virtual.suspect'] += 1
1954                                                         fails['virtual.suspect'].append(
1955                                                                 relative_path +
1956                                                                 ": %s: consider using '%s' instead of '%s'" %
1957                                                                 (mytype, suspect_virtual[atom.cp], atom))
1958
1959                                         if mytype == "DEPEND" and \
1960                                                 not is_blocker and \
1961                                                 not inherited_java_eclass and \
1962                                                 atom.cp == "virtual/jdk":
1963                                                 stats['java.eclassesnotused'] += 1
1964                                                 fails['java.eclassesnotused'].append(relative_path)
1965                                         elif mytype == "DEPEND" and \
1966                                                 not is_blocker and \
1967                                                 not inherited_wxwidgets_eclass and \
1968                                                 atom.cp == "x11-libs/wxGTK":
1969                                                 stats['wxwidgets.eclassnotused'] += 1
1970                                                 fails['wxwidgets.eclassnotused'].append(
1971                                                         relative_path + ": DEPENDs on x11-libs/wxGTK"
1972                                                         " without inheriting wxwidgets.eclass")
1973                                         elif mytype in ("PDEPEND", "RDEPEND"):
1974                                                 if not is_blocker and \
1975                                                         atom.cp in suspect_rdepend:
1976                                                         stats[mytype + '.suspect'] += 1
1977                                                         fails[mytype + '.suspect'].append(
1978                                                                 relative_path + ": '%s'" % atom)
1979
1980                                         if atom.operator == "~" and \
1981                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1982                                                 stats[mytype + '.badtilde'] += 1
1983                                                 fails[mytype + '.badtilde'].append(
1984                                                         (relative_path + ": %s uses the ~ operator"
1985                                                          " with a non-zero revision:" + \
1986                                                          " '%s'") % (mytype, atom))
1987
1988                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1989
1990                 for m,b in zip(type_list, badsyntax):
1991                         stats[m+".syntax"] += 1
1992                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1993
1994                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1995                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1996                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1997                 badlicsyntax = badlicsyntax > 0
1998                 badprovsyntax = badprovsyntax > 0
1999
2000                 # uselist checks - global
2001                 myuse = []
2002                 default_use = []
2003                 for myflag in myaux["IUSE"].split():
2004                         flag_name = myflag.lstrip("+-")
2005                         used_useflags.add(flag_name)
2006                         if myflag != flag_name:
2007                                 default_use.append(myflag)
2008                         if flag_name not in uselist:
2009                                 myuse.append(flag_name)
2010
2011                 # uselist checks - metadata
2012                 for mypos in range(len(myuse)-1,-1,-1):
2013                         if myuse[mypos] and (myuse[mypos] in muselist):
2014                                 del myuse[mypos]
2015
2016                 if default_use and not eapi_has_iuse_defaults(eapi):
2017                         for myflag in default_use:
2018                                 stats['EAPI.incompatible'] += 1
2019                                 fails['EAPI.incompatible'].append(
2020                                         (relative_path + ": IUSE defaults" + \
2021                                         " not supported with EAPI='%s':" + \
2022                                         " '%s'") % (eapi, myflag))
2023
2024                 for mypos in range(len(myuse)):
2025                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
2026                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
2027
2028                 # license checks
2029                 if not badlicsyntax:
2030                         # Parse the LICENSE variable, remove USE conditions and
2031                         # flatten it.
2032                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
2033                         # Check each entry to ensure that it exists in PORTDIR's
2034                         # license directory.
2035                         for lic in licenses:
2036                                 # Need to check for "||" manually as no portage
2037                                 # function will remove it without removing values.
2038                                 if lic not in liclist and lic != "||":
2039                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
2040                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
2041
2042                 #keyword checks
2043                 myuse = myaux["KEYWORDS"].split()
2044                 for mykey in myuse:
2045                         if mykey not in ("-*", "*", "~*"):
2046                                 myskey = mykey
2047                                 if myskey[:1] == "-":
2048                                         myskey = myskey[1:]
2049                                 if myskey[:1] == "~":
2050                                         myskey = myskey[1:]
2051                                 if myskey not in kwlist:
2052                                         stats["KEYWORDS.invalid"] += 1
2053                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
2054                                 elif myskey not in profiles:
2055                                         stats["KEYWORDS.invalid"] += 1
2056                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
2057
2058                 #restrict checks
2059                 myrestrict = None
2060                 try:
2061                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
2062                 except portage.exception.InvalidDependString as e:
2063                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
2064                         fails["RESTRICT.syntax"].append(
2065                                 "%s: RESTRICT: %s" % (relative_path, e))
2066                         del e
2067                 if myrestrict:
2068                         myrestrict = set(myrestrict)
2069                         mybadrestrict = myrestrict.difference(valid_restrict)
2070                         if mybadrestrict:
2071                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
2072                                 for mybad in mybadrestrict:
2073                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
2074                 #REQUIRED_USE check
2075                 required_use = myaux["REQUIRED_USE"]
2076                 if required_use:
2077                         if not eapi_has_required_use(eapi):
2078                                 stats['EAPI.incompatible'] += 1
2079                                 fails['EAPI.incompatible'].append(
2080                                         relative_path + ": REQUIRED_USE" + \
2081                                         " not supported with EAPI='%s'" % (eapi,))
2082                         try:
2083                                 portage.dep.check_required_use(required_use, (),
2084                                         pkg.iuse.is_valid_flag, eapi=eapi)
2085                         except portage.exception.InvalidDependString as e:
2086                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
2087                                 fails["REQUIRED_USE.syntax"].append(
2088                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
2089                                 del e
2090
2091                 # Syntax Checks
2092                 relative_path = os.path.join(x, y + ".ebuild")
2093                 full_path = os.path.join(repodir, relative_path)
2094                 if not vcs_preserves_mtime:
2095                         if ebuild_path not in new_ebuilds and \
2096                                 ebuild_path not in modified_ebuilds:
2097                                 pkg.mtime = None
2098                 try:
2099                         # All ebuilds should have utf_8 encoding.
2100                         f = io.open(_unicode_encode(full_path,
2101                                 encoding=_encodings['fs'], errors='strict'),
2102                                 mode='r', encoding=_encodings['repo.content'])
2103                         try:
2104                                 for check_name, e in run_checks(f, pkg):
2105                                         stats[check_name] += 1
2106                                         fails[check_name].append(relative_path + ': %s' % e)
2107                         finally:
2108                                 f.close()
2109                 except UnicodeDecodeError:
2110                         # A file.UTF8 failure will have already been recorded above.
2111                         pass
2112
2113                 if options.force:
2114                         # The dep_check() calls are the most expensive QA test. If --force
2115                         # is enabled, there's no point in wasting time on these since the
2116                         # user is intent on forcing the commit anyway.
2117                         continue
2118
2119                 for keyword,arch,groups in arches:
2120
2121                         if arch not in profiles:
2122                                 # A missing profile will create an error further down
2123                                 # during the KEYWORDS verification.
2124                                 continue
2125
2126                         for prof in profiles[arch]:
2127
2128                                 if prof.status not in ("stable", "dev") or \
2129                                         prof.status == "dev" and not options.include_dev:
2130                                         continue
2131
2132                                 dep_settings = arch_caches.get(prof.sub_path)
2133                                 if dep_settings is None:
2134                                         dep_settings = portage.config(
2135                                                 config_profile_path=prof.abs_path,
2136                                                 config_incrementals=repoman_incrementals,
2137                                                 config_root=config_root,
2138                                                 local_config=False,
2139                                                 _unmatched_removal=options.unmatched_removal,
2140                                                 env=env)
2141                                         dep_settings.categories = repoman_settings.categories
2142                                         if options.without_mask:
2143                                                 dep_settings._mask_manager_obj = \
2144                                                         copy.deepcopy(dep_settings._mask_manager)
2145                                                 dep_settings._mask_manager._pmaskdict.clear()
2146                                         arch_caches[prof.sub_path] = dep_settings
2147
2148                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2149                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2150                                 if xcache is None:
2151                                         portdb.melt()
2152                                         portdb.freeze()
2153                                         xcache = portdb.xcache
2154                                         xcache.update(shared_xmatch_caches)
2155                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2156
2157                                 trees[root]["porttree"].settings = dep_settings
2158                                 portdb.settings = dep_settings
2159                                 portdb.xcache = xcache
2160                                 # for package.use.mask support inside dep_check
2161                                 dep_settings.setcpv(pkg)
2162                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2163                                 # just in case, prevent config.reset() from nuking these.
2164                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2165
2166                                 if not baddepsyntax:
2167                                         ismasked = not ebuild_archs or \
2168                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2169                                         if ismasked:
2170                                                 if not have_pmasked:
2171                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2172                                                                 pkg.cpv, pkg.metadata))
2173                                                 if options.ignore_masked:
2174                                                         continue
2175                                                 #we are testing deps for a masked package; give it some lee-way
2176                                                 suffix="masked"
2177                                                 matchmode = "minimum-all"
2178                                         else:
2179                                                 suffix=""
2180                                                 matchmode = "minimum-visible"
2181
2182                                         if not have_dev_keywords:
2183                                                 have_dev_keywords = \
2184                                                         bool(dev_keywords.intersection(keywords))
2185
2186                                         if prof.status == "dev":
2187                                                 suffix=suffix+"indev"
2188
2189                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
2190
2191                                                 mykey=mytype+".bad"+suffix
2192                                                 myvalue = myaux[mytype]
2193                                                 if not myvalue:
2194                                                         continue
2195
2196                                                 success, atoms = portage.dep_check(myvalue, portdb,
2197                                                         dep_settings, use="all", mode=matchmode,
2198                                                         trees=trees)
2199
2200                                                 if success:
2201                                                         if atoms:
2202
2203                                                                 # Don't bother with dependency.unknown for
2204                                                                 # cases in which *DEPEND.bad is triggered.
2205                                                                 for atom in atoms:
2206                                                                         # dep_check returns all blockers and they
2207                                                                         # aren't counted for *DEPEND.bad, so we
2208                                                                         # ignore them here.
2209                                                                         if not atom.blocker:
2210                                                                                 unknown_pkgs.discard(
2211                                                                                         (mytype, atom.unevaluated_atom))
2212
2213                                                                 if not prof.sub_path:
2214                                                                         # old-style virtuals currently aren't
2215                                                                         # resolvable with empty profile, since
2216                                                                         # 'virtuals' mappings are unavailable
2217                                                                         # (it would be expensive to search
2218                                                                         # for PROVIDE in all ebuilds)
2219                                                                         atoms = [atom for atom in atoms if not \
2220                                                                                 (atom.cp.startswith('virtual/') and \
2221                                                                                 not portdb.cp_list(atom.cp))]
2222
2223                                                                 #we have some unsolvable deps
2224                                                                 #remove ! deps, which always show up as unsatisfiable
2225                                                                 atoms = [str(atom.unevaluated_atom) \
2226                                                                         for atom in atoms if not atom.blocker]
2227
2228                                                                 #if we emptied out our list, continue:
2229                                                                 if not atoms:
2230                                                                         continue
2231                                                                 stats[mykey]=stats[mykey]+1
2232                                                                 fails[mykey].append("%s: %s(%s) %s" % \
2233                                                                         (relative_path, keyword,
2234                                                                         prof, repr(atoms)))
2235                                                 else:
2236                                                         stats[mykey]=stats[mykey]+1
2237                                                         fails[mykey].append("%s: %s(%s) %s" % \
2238                                                                 (relative_path, keyword,
2239                                                                 prof, repr(atoms)))
2240
2241                 if not baddepsyntax and unknown_pkgs:
2242                         type_map = {}
2243                         for mytype, atom in unknown_pkgs:
2244                                 type_map.setdefault(mytype, set()).add(atom)
2245                         for mytype, atoms in type_map.items():
2246                                 stats["dependency.unknown"] += 1
2247                                 fails["dependency.unknown"].append("%s: %s: %s" %
2248                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2249
2250         # check if there are unused local USE-descriptions in metadata.xml
2251         # (unless there are any invalids, to avoid noise)
2252         if allvalid:
2253                 for myflag in muselist.difference(used_useflags):
2254                         stats["metadata.warning"] += 1
2255                         fails["metadata.warning"].append(
2256                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2257                                 (x, myflag))
2258
2259 if options.if_modified == "y" and len(effective_scanlist) < 1:
2260         logging.warn("--if-modified is enabled, but no modified packages were found!")
2261
2262 if options.mode == "manifest":
2263         sys.exit(dofail)
2264
2265 #dofail will be set to 1 if we have failed in at least one non-warning category
2266 dofail=0
2267 #dowarn will be set to 1 if we tripped any warnings
2268 dowarn=0
2269 #dofull will be set if we should print a "repoman full" informational message
2270 dofull = options.mode != 'full'
2271
2272 for x in qacats:
2273         if not stats[x]:
2274                 continue
2275         dowarn = 1
2276         if x not in qawarnings:
2277                 dofail = 1
2278
2279 if dofail or \
2280         (dowarn and not (options.quiet or options.mode == "scan")):
2281         dofull = 0
2282
2283 # Save QA output so that it can be conveniently displayed
2284 # in $EDITOR while the user creates a commit message.
2285 # Otherwise, the user would not be able to see this output
2286 # once the editor has taken over the screen.
2287 qa_output = io.StringIO()
2288 style_file = ConsoleStyleFile(sys.stdout)
2289 if options.mode == 'commit' and \
2290         (not commitmessage or not commitmessage.strip()):
2291         style_file.write_listener = qa_output
2292 console_writer = StyleWriter(file=style_file, maxcol=9999)
2293 console_writer.style_listener = style_file.new_styles
2294
2295 f = formatter.AbstractFormatter(console_writer)
2296
2297 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2298
2299 style_file.flush()
2300 del console_writer, f, style_file
2301 qa_output = qa_output.getvalue()
2302 qa_output = qa_output.splitlines(True)
2303
2304 def grouplist(mylist,seperator="/"):
2305         """(list,seperator="/") -- Takes a list of elements; groups them into
2306         same initial element categories. Returns a dict of {base:[sublist]}
2307         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2308         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2309         mygroups={}
2310         for x in mylist:
2311                 xs=x.split(seperator)
2312                 if xs[0]==".":
2313                         xs=xs[1:]
2314                 if xs[0] not in mygroups:
2315                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2316                 else:
2317                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2318         return mygroups
2319
2320 suggest_ignore_masked = False
2321 suggest_include_dev = False
2322
2323 if have_pmasked and not (options.without_mask or options.ignore_masked):
2324         suggest_ignore_masked = True
2325 if have_dev_keywords and not options.include_dev:
2326         suggest_include_dev = True
2327
2328 if suggest_ignore_masked or suggest_include_dev:
2329         print()
2330         if suggest_ignore_masked:
2331                 print(bold("Note: use --without-mask to check " + \
2332                         "KEYWORDS on dependencies of masked packages"))
2333
2334         if suggest_include_dev:
2335                 print(bold("Note: use --include-dev (-d) to check " + \
2336                         "dependencies for 'dev' profiles"))
2337         print()
2338
2339 if options.mode != 'commit':
2340         if dofull:
2341                 print(bold("Note: type \"repoman full\" for a complete listing."))
2342         if dowarn and not dofail:
2343                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2344         elif not dofail:
2345                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2346         elif dofail:
2347                 print(bad("Please fix these important QA issues first."))
2348                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2349                 sys.exit(1)
2350 else:
2351         if dofail and can_force and options.force and not options.pretend:
2352                 print(green("RepoMan sez:") + \
2353                         " \"You want to commit even with these QA issues?\n" + \
2354                         "              I'll take it this time, but I'm not happy.\"\n")
2355         elif dofail:
2356                 if options.force and not can_force:
2357                         print(bad("The --force option has been disabled due to extraordinary issues."))
2358                 print(bad("Please fix these important QA issues first."))
2359                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2360                 sys.exit(1)
2361
2362         if options.pretend:
2363                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2364
2365         myunadded = []
2366         if vcs == "cvs":
2367                 try:
2368                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2369                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2370                 except SystemExit as e:
2371                         raise  # TODO propagate this
2372                 except:
2373                         err("Error retrieving CVS tree; exiting.")
2374         if vcs == "svn":
2375                 try:
2376                         with repoman_popen("svn status --no-ignore") as f:
2377                                 svnstatus = f.readlines()
2378                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2379                 except SystemExit as e:
2380                         raise  # TODO propagate this
2381                 except:
2382                         err("Error retrieving SVN info; exiting.")
2383         if vcs == "git":
2384                 # get list of files not under version control or missing
2385                 myf = repoman_popen("git ls-files --others")
2386                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2387                 myf.close()
2388         if vcs == "bzr":
2389                 try:
2390                         with repoman_popen("bzr status -S .") as f:
2391                                 bzrstatus = f.readlines()
2392                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2393                 except SystemExit as e:
2394                         raise  # TODO propagate this
2395                 except:
2396                         err("Error retrieving bzr info; exiting.")
2397         if vcs == "hg":
2398                 with repoman_popen("hg status --no-status --unknown .") as f:
2399                         myunadded = f.readlines()
2400                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2401
2402                 # Mercurial doesn't handle manually deleted files as removed from
2403                 # the repository, so the user need to remove them before commit,
2404                 # using "hg remove [FILES]"
2405                 with repoman_popen("hg status --no-status --deleted .") as f:
2406                         mydeleted = f.readlines()
2407                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2408
2409
2410         myautoadd=[]
2411         if myunadded:
2412                 for x in range(len(myunadded)-1,-1,-1):
2413                         xs=myunadded[x].split("/")
2414                         if xs[-1]=="files":
2415                                 print("!!! files dir is not added! Please correct this.")
2416                                 sys.exit(-1)
2417                         elif xs[-1]=="Manifest":
2418                                 # It's a manifest... auto add
2419                                 myautoadd+=[myunadded[x]]
2420                                 del myunadded[x]
2421
2422         if myunadded:
2423                 print(red("!!! The following files are in your local tree but are not added to the master"))
2424                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2425                 for x in myunadded:
2426                         print("   ",x)
2427                 print()
2428                 print()
2429                 sys.exit(1)
2430
2431         if vcs == "hg" and mydeleted:
2432                 print(red("!!! The following files are removed manually from your local tree but are not"))
2433                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2434                 for x in mydeleted:
2435                         print("   ",x)
2436                 print()
2437                 print()
2438                 sys.exit(1)
2439
2440         if vcs == "cvs":
2441                 mycvstree = cvstree.getentries("./", recursive=1)
2442                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2443                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2444                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2445                 bin_blob_pattern = re.compile("^-kb$")
2446                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2447                         recursive=1, basedir="./"))
2448
2449
2450         if vcs == "svn":
2451                 with repoman_popen("svn status") as f:
2452                         svnstatus = f.readlines()
2453                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2454                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2455                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2456
2457                 # Subversion expands keywords specified in svn:keywords properties.
2458                 with repoman_popen("svn propget -R svn:keywords") as f:
2459                         props = f.readlines()
2460                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2461                         for prop in props if " - " in prop)
2462
2463         elif vcs == "git":
2464                 with repoman_popen("git diff-index --name-only "
2465                         "--relative --diff-filter=M HEAD") as f:
2466                         mychanged = f.readlines()
2467                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2468
2469                 with repoman_popen("git diff-index --name-only "
2470                         "--relative --diff-filter=A HEAD") as f:
2471                         mynew = f.readlines()
2472                 mynew = ["./" + elem[:-1] for elem in mynew]
2473
2474                 with repoman_popen("git diff-index --name-only "
2475                         "--relative --diff-filter=D HEAD") as f:
2476                         myremoved = f.readlines()
2477                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2478
2479         if vcs == "bzr":
2480                 with repoman_popen("bzr status -S .") as f:
2481                         bzrstatus = f.readlines()
2482                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2483                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2484                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2485                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2486                 # Bazaar expands nothing.
2487
2488         if vcs == "hg":
2489                 with repoman_popen("hg status --no-status --modified .") as f:
2490                         mychanged = f.readlines()
2491                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2492
2493                 with repoman_popen("hg status --no-status --added .") as f:
2494                         mynew = f.readlines()
2495                 mynew = ["./" + elem.rstrip() for elem in mynew]
2496
2497                 with repoman_popen("hg status --no-status --removed .") as f:
2498                         myremoved = f.readlines()
2499                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2500
2501         if vcs:
2502                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2503                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2504                         print()
2505                         print("(Didn't find any changed files...)")
2506                         print()
2507                         sys.exit(1)
2508
2509         # Manifests need to be regenerated after all other commits, so don't commit
2510         # them now even if they have changed.
2511         mymanifests = set()
2512         myupdates = set()
2513         for f in mychanged + mynew:
2514                 if "Manifest" == os.path.basename(f):
2515                         mymanifests.add(f)
2516                 else:
2517                         myupdates.add(f)
2518         myupdates.difference_update(myremoved)
2519         myupdates = list(myupdates)
2520         mymanifests = list(mymanifests)
2521         myheaders = []
2522         mydirty = []
2523
2524         commitmessage = options.commitmsg
2525         if options.commitmsgfile:
2526                 try:
2527                         f = io.open(_unicode_encode(options.commitmsgfile,
2528                         encoding=_encodings['fs'], errors='strict'),
2529                         mode='r', encoding=_encodings['content'], errors='replace')
2530                         commitmessage = f.read()
2531                         f.close()
2532                         del f
2533                 except (IOError, OSError) as e:
2534                         if e.errno == errno.ENOENT:
2535                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2536                         else:
2537                                 raise
2538                 # We've read the content so the file is no longer needed.
2539                 commitmessagefile = None
2540         if not commitmessage or not commitmessage.strip():
2541                 try:
2542                         editor = os.environ.get("EDITOR")
2543                         if editor and utilities.editor_is_executable(editor):
2544                                 commitmessage = utilities.get_commit_message_with_editor(
2545                                         editor, message=qa_output)
2546                         else:
2547                                 commitmessage = utilities.get_commit_message_with_stdin()
2548                 except KeyboardInterrupt:
2549                         exithandler()
2550                 if not commitmessage or not commitmessage.strip():
2551                         print("* no commit message?  aborting commit.")
2552                         sys.exit(1)
2553         commitmessage = commitmessage.rstrip()
2554         changelog_msg = commitmessage
2555         portage_version = getattr(portage, "VERSION", None)
2556         if portage_version is None:
2557                 sys.stderr.write("Failed to insert portage version in message!\n")
2558                 sys.stderr.flush()
2559                 portage_version = "Unknown"
2560         unameout = platform.system() + " "
2561         if platform.system() in ["Darwin", "SunOS"]:
2562                 unameout += platform.processor()
2563         else:
2564                 unameout += platform.machine()
2565         commitmessage += "\n\n(Portage version: %s/%s/%s" % \
2566                 (portage_version, vcs, unameout)
2567         if options.force:
2568                 commitmessage += ", RepoMan options: --force"
2569         commitmessage += ")"
2570
2571         if options.echangelog in ('y', 'force'):
2572                 logging.info("checking for unmodified ChangeLog files")
2573                 committer_name = utilities.get_committer_name(env=repoman_settings)
2574                 for x in sorted(vcs_files_to_cps(
2575                         chain(myupdates, mymanifests, myremoved))):
2576                         catdir, pkgdir = x.split("/")
2577                         checkdir = repodir + "/" + x
2578                         checkdir_relative = ""
2579                         if repolevel < 3:
2580                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2581                         if repolevel < 2:
2582                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2583                         checkdir_relative = os.path.join(".", checkdir_relative)
2584
2585                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2586                         changelog_modified = changelog_path in modified_changelogs
2587                         if changelog_modified and options.echangelog != 'force':
2588                                 continue
2589
2590                         # get changes for this package
2591                         cdrlen = len(checkdir_relative)
2592                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2593                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2594                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2595
2596                         # Skip ChangeLog generation if only the Manifest was modified,
2597                         # as discussed in bug #398009.
2598                         nontrivial_cl_files = set()
2599                         nontrivial_cl_files.update(clnew, clremoved, clchanged)
2600                         nontrivial_cl_files.difference_update(['Manifest'])
2601                         if not nontrivial_cl_files and options.echangelog != 'force':
2602                                 continue
2603
2604                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2605                                 committer_name, changelog_msg,
2606                                 os.path.join(repodir, 'skel.ChangeLog'),
2607                                 catdir, pkgdir,
2608                                 new=clnew, removed=clremoved, changed=clchanged,
2609                                 pretend=options.pretend)
2610                         if new_changelog is None:
2611                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2612                                         level=logging.ERROR, noiselevel=-1)
2613                                 sys.exit(1)
2614
2615                         # if the ChangeLog was just created, add it to vcs
2616                         if new_changelog:
2617                                 myautoadd.append(changelog_path)
2618                                 # myautoadd is appended to myupdates below
2619                         else:
2620                                 myupdates.append(changelog_path)
2621
2622                         if options.ask and not options.pretend:
2623                                 # regenerate Manifest for modified ChangeLog (bug #420735)
2624                                 repoman_settings["O"] = checkdir
2625                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2626
2627         if myautoadd:
2628                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2629                 add_cmd = [vcs, "add"]
2630                 add_cmd += myautoadd
2631                 if options.pretend:
2632                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2633                                 noiselevel=-1)
2634                 else:
2635                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2636                                 # Python 3.1 produces the following TypeError if raw bytes are
2637                                 # passed to subprocess.call():
2638                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2639                                 #     errread, errwrite)
2640                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2641                                 #     raise child_exception
2642                                 # TypeError: expected an object with the buffer interface
2643                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2644                         retcode = subprocess.call(add_cmd)
2645                         if retcode != os.EX_OK:
2646                                 logging.error(
2647                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2648                                 sys.exit(retcode)
2649
2650                 myupdates += myautoadd
2651
2652         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2653
2654         if vcs not in ('cvs', 'svn'):
2655                 # With git, bzr and hg, there's never any keyword expansion, so
2656                 # there's no need to regenerate manifests and all files will be
2657                 # committed in one big commit at the end.
2658                 print()
2659         elif not repo_config.thin_manifest:
2660                 if vcs == 'cvs':
2661                         headerstring = "'\$(Header|Id).*\$'"
2662                 elif vcs == "svn":
2663                         svn_keywords = dict((k.lower(), k) for k in [
2664                                         "Rev",
2665                                         "Revision",
2666                                         "LastChangedRevision",
2667                                         "Date",
2668                                         "LastChangedDate",
2669                                         "Author",
2670                                         "LastChangedBy",
2671                                         "URL",
2672                                         "HeadURL",
2673                                         "Id",
2674                                         "Header",
2675                         ])
2676
2677                 for myfile in myupdates:
2678
2679                         # for CVS, no_expansion contains files that are excluded from expansion
2680                         if vcs == "cvs":
2681                                 if myfile in no_expansion:
2682                                         continue
2683
2684                         # for SVN, expansion contains files that are included in expansion
2685                         elif vcs == "svn":
2686                                 if myfile not in expansion:
2687                                         continue
2688
2689                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2690                                 enabled_keywords = []
2691                                 for k in expansion[myfile]:
2692                                         keyword = svn_keywords.get(k.lower())
2693                                         if keyword is not None:
2694                                                 enabled_keywords.append(keyword)
2695
2696                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2697
2698                         myout = repoman_getstatusoutput("egrep -q " + headerstring + " " +
2699                                 portage._shell_quote(myfile))
2700                         if myout[0] == 0:
2701                                 myheaders.append(myfile)
2702
2703                 print("%s have headers that will change." % green(str(len(myheaders))))
2704                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2705
2706         logging.info("myupdates: %s", myupdates)
2707         logging.info("myheaders: %s", myheaders)
2708
2709         if options.ask and userquery('Commit changes?', True) != 'Yes':
2710                 print("* aborting commit.")
2711                 sys.exit(128 + signal.SIGINT)
2712
2713         # Handle the case where committed files have keywords which
2714         # will change and need a priming commit before the Manifest
2715         # can be committed.
2716         if (myupdates or myremoved) and myheaders:
2717                 myfiles = myupdates + myremoved
2718                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2719                 mymsg = os.fdopen(fd, "wb")
2720                 mymsg.write(_unicode_encode(commitmessage))
2721                 mymsg.close()
2722
2723                 print()
2724                 print(green("Using commit message:"))
2725                 print(green("------------------------------------------------------------------------------"))
2726                 print(commitmessage)
2727                 print(green("------------------------------------------------------------------------------"))
2728                 print()
2729
2730                 # Having a leading ./ prefix on file paths can trigger a bug in
2731                 # the cvs server when committing files to multiple directories,
2732                 # so strip the prefix.
2733                 myfiles = [f.lstrip("./") for f in myfiles]
2734
2735                 commit_cmd = [vcs]
2736                 commit_cmd.extend(vcs_global_opts)
2737                 commit_cmd.append("commit")
2738                 commit_cmd.extend(vcs_local_opts)
2739                 commit_cmd.extend(["-F", commitmessagefile])
2740                 commit_cmd.extend(myfiles)
2741
2742                 try:
2743                         if options.pretend:
2744                                 print("(%s)" % (" ".join(commit_cmd),))
2745                         else:
2746                                 retval = spawn(commit_cmd, env=os.environ)
2747                                 if retval != os.EX_OK:
2748                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2749                                                 "error code: %s\n") % (vcs, retval),
2750                                                 level=logging.ERROR, noiselevel=-1)
2751                                         sys.exit(retval)
2752                 finally:
2753                         try:
2754                                 os.unlink(commitmessagefile)
2755                         except OSError:
2756                                 pass
2757
2758         # Setup the GPG commands
2759         def gpgsign(filename):
2760                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2761                 if gpgcmd is None:
2762                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2763                                 " Is make.globals missing?")
2764                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2765                         "PORTAGE_GPG_KEY" not in repoman_settings:
2766                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2767                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2768                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2769                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2770                                         os.path.expanduser("~/.gnupg")
2771                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2772                                         % repoman_settings["PORTAGE_GPG_DIR"])
2773                         else:
2774                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2775                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2776                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2777                                 raise portage.exception.InvalidLocation(
2778                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2779                                         repoman_settings["PORTAGE_GPG_DIR"])
2780                 gpgvars = {"FILE": filename}
2781                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2782                         v = repoman_settings.get(k)
2783                         if v is not None:
2784                                 gpgvars[k] = v
2785                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2786                 if options.pretend:
2787                         print("("+gpgcmd+")")
2788                 else:
2789                         rValue = os.system(gpgcmd)
2790                         if rValue == os.EX_OK:
2791                                 os.rename(filename+".asc", filename)
2792                         else:
2793                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2794
2795         # When files are removed and re-added, the cvs server will put /Attic/
2796         # inside the $Header path. This code detects the problem and corrects it
2797         # so that the Manifest will generate correctly. See bug #169500.
2798         # Use binary mode in order to avoid potential character encoding issues.
2799         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2800         attic_str = b'/Attic/'
2801         attic_replace = b'/'
2802         for x in myheaders:
2803                 f = open(_unicode_encode(x,
2804                         encoding=_encodings['fs'], errors='strict'),
2805                         mode='rb')
2806                 mylines = f.readlines()
2807                 f.close()
2808                 modified = False
2809                 for i, line in enumerate(mylines):
2810                         if cvs_header_re.match(line) is not None and \
2811                                 attic_str in line:
2812                                 mylines[i] = line.replace(attic_str, attic_replace)
2813                                 modified = True
2814                 if modified:
2815                         portage.util.write_atomic(x, b''.join(mylines),
2816                                 mode='wb')
2817
2818         if repolevel == 1:
2819                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2820                         "doing the entire repository.\"\n")
2821
2822         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2823
2824                 for x in sorted(vcs_files_to_cps(
2825                         chain(myupdates, myremoved, mymanifests))):
2826                         repoman_settings["O"] = os.path.join(repodir, x)
2827                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2828
2829         signed = False
2830         if sign_manifests:
2831                 signed = True
2832                 try:
2833                         for x in sorted(vcs_files_to_cps(
2834                                 chain(myupdates, myremoved, mymanifests))):
2835                                 repoman_settings["O"] = os.path.join(repodir, x)
2836                                 manifest_path = os.path.join(repoman_settings["O"], "Manifest")
2837                                 if not os.path.exists(manifest_path):
2838                                         continue
2839                                 gpgsign(manifest_path)
2840                 except portage.exception.PortageException as e:
2841                         portage.writemsg("!!! %s\n" % str(e))
2842                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2843                         signed = False
2844
2845         if vcs == 'git':
2846                 # It's not safe to use the git commit -a option since there might
2847                 # be some modified files elsewhere in the working tree that the
2848                 # user doesn't want to commit. Therefore, call git update-index
2849                 # in order to ensure that the index is updated with the latest
2850                 # versions of all new and modified files in the relevant portion
2851                 # of the working tree.
2852                 myfiles = mymanifests + myupdates
2853                 myfiles.sort()
2854                 update_index_cmd = ["git", "update-index"]
2855                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2856                 if options.pretend:
2857                         print("(%s)" % (" ".join(update_index_cmd),))
2858                 else:
2859                         retval = spawn(update_index_cmd, env=os.environ)
2860                         if retval != os.EX_OK:
2861                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2862                                         "error code: %s\n") % (vcs, retval),
2863                                         level=logging.ERROR, noiselevel=-1)
2864                                 sys.exit(retval)
2865
2866         if True:
2867
2868                 myfiles = mymanifests[:]
2869                 # If there are no header (SVN/CVS keywords) changes in
2870                 # the files, this Manifest commit must include the
2871                 # other (yet uncommitted) files.
2872                 if not myheaders:
2873                         myfiles += myupdates
2874                         myfiles += myremoved
2875                 myfiles.sort()
2876
2877                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2878                 mymsg = os.fdopen(fd, "wb")
2879                 # strip the closing parenthesis
2880                 mymsg.write(_unicode_encode(commitmessage[:-1]))
2881                 if signed:
2882                         mymsg.write(_unicode_encode(
2883                                 ", signed Manifest commit with key %s)" % \
2884                                 repoman_settings["PORTAGE_GPG_KEY"]))
2885                 else:
2886                         mymsg.write(b", unsigned Manifest commit)")
2887                 mymsg.close()
2888
2889                 commit_cmd = []
2890                 if options.pretend and vcs is None:
2891                         # substitute a bogus value for pretend output
2892                         commit_cmd.append("cvs")
2893                 else:
2894                         commit_cmd.append(vcs)
2895                 commit_cmd.extend(vcs_global_opts)
2896                 commit_cmd.append("commit")
2897                 commit_cmd.extend(vcs_local_opts)
2898                 if vcs == "hg":
2899                         commit_cmd.extend(["--logfile", commitmessagefile])
2900                         commit_cmd.extend(myfiles)
2901                 else:
2902                         commit_cmd.extend(["-F", commitmessagefile])
2903                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2904
2905                 try:
2906                         if options.pretend:
2907                                 print("(%s)" % (" ".join(commit_cmd),))
2908                         else:
2909                                 retval = spawn(commit_cmd, env=os.environ)
2910                                 if retval != os.EX_OK:
2911
2912                                         if repo_config.sign_commit and vcs == 'git' and \
2913                                                 not git_supports_gpg_sign():
2914                                                 # Inform user that newer git is needed (bug #403323).
2915                                                 logging.error(
2916                                                         "Git >=1.7.9 is required for signed commits!")
2917
2918                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2919                                                 "error code: %s\n") % (vcs, retval),
2920                                                 level=logging.ERROR, noiselevel=-1)
2921                                         sys.exit(retval)
2922                 finally:
2923                         try:
2924                                 os.unlink(commitmessagefile)
2925                         except OSError:
2926                                 pass
2927
2928         print()
2929         if vcs:
2930                 print("Commit complete.")
2931         else:
2932                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2933         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2934 sys.exit(0)
2935