bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2011 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import copy
  13 import errno
  14 import formatter
  15 import io
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import textwrap
  25 import time
  26 import platform
  27
  28 try:
  29         from urllib.request import urlopen as urllib_request_urlopen
  30 except ImportError:
  31         from urllib import urlopen as urllib_request_urlopen
  32
  33 from itertools import chain
  34 from stat import S_ISDIR
  35
  36 try:
  37         import portage
  38 except ImportError:
  39         from os import path as osp
  40         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  41         import portage
  42 portage._disable_legacy_globals()
  43 portage.dep._internal_warnings = True
  44
  45 try:
  46         import xml.etree.ElementTree
  47         from xml.parsers.expat import ExpatError
  48 except ImportError:
  49         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  50         from portage.output import EOutput
  51         out = EOutput()
  52         for line in msg:
  53                 out.eerror(line)
  54         sys.exit(1)
  55
  56 from portage import os
  57 from portage import subprocess_getstatusoutput
  58 from portage import _encodings
  59 from portage import _unicode_encode
  60 from repoman.checks import run_checks
  61 from repoman import utilities
  62 from repoman.herdbase import make_herd_base
  63 from _emerge.Package import Package
  64 from _emerge.RootConfig import RootConfig
  65 from _emerge.userquery import userquery
  66 import portage.checksum
  67 import portage.const
  68 from portage import cvstree, normalize_path
  69 from portage import util
  70 from portage.exception import (FileNotFound, MissingParameter,
  71         ParseError, PermissionDenied)
  72 from portage.process import find_binary, spawn
  73 from portage.output import bold, create_color_func, \
  74         green, nocolor, red
  75 from portage.output import ConsoleStyleFile, StyleWriter
  76 from portage.util import cmp_sort_key, writemsg_level
  77 from portage.package.ebuild.digestgen import digestgen
  78 from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
  79
  80 if sys.hexversion >= 0x3000000:
  81         basestring = str
  82
  83 util.initialize_logger()
  84
  85 # 14 is the length of DESCRIPTION=""
  86 max_desc_len = 100
  87 allowed_filename_chars="a-zA-Z0-9._-+:"
  88 disallowed_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]')
  89 pv_toolong_re = re.compile(r'[0-9]{19,}')
  90 bad = create_color_func("BAD")
  91
  92 # A sane umask is needed for files that portage creates.
  93 os.umask(0o22)
  94 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  95 # behave incrementally.
  96 repoman_incrementals = tuple(x for x in \
  97         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
  98 eprefix = os.environ.get("__REPOMAN_TEST_EPREFIX")
  99 repoman_settings = portage.config(local_config=False, _eprefix=eprefix)
 100 repoman_settings.lock()
 101
 102 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 103         repoman_settings.get('TERM') == 'dumb' or \
 104         not sys.stdout.isatty():
 105         nocolor()
 106
 107 def warn(txt):
 108         print("repoman: " + txt)
 109
 110 def err(txt):
 111         warn(txt)
 112         sys.exit(1)
 113
 114 def exithandler(signum=None, frame=None):
 115         logging.fatal("Interrupted; exiting...")
 116         if signum is None:
 117                 sys.exit(1)
 118         else:
 119                 sys.exit(128 + signum)
 120
 121 signal.signal(signal.SIGINT,exithandler)
 122
 123 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 124         """Repoman needs it's own HelpFormatter for now, because the default ones
 125         murder the help text."""
 126
 127         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 128                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 129
 130         def format_description(self, description):
 131                 return description
 132
 133 class RepomanOptionParser(optparse.OptionParser):
 134         """Add the on_tail function, ruby has it, optionParser should too
 135         """
 136
 137         def __init__(self, *args, **kwargs):
 138                 optparse.OptionParser.__init__(self, *args, **kwargs)
 139                 self.tail = ""
 140
 141         def on_tail(self, description):
 142                 self.tail += description
 143
 144         def format_help(self, formatter=None):
 145                 result = optparse.OptionParser.format_help(self, formatter)
 146                 result += self.tail
 147                 return result
 148
 149
 150 def ParseArgs(argv, qahelp):
 151         """This function uses a customized optionParser to parse command line arguments for repoman
 152         Args:
 153           argv - a sequence of command line arguments
 154                 qahelp - a dict of qa warning to help message
 155         Returns:
 156           (opts, args), just like a call to parser.parse_args()
 157         """
 158
 159         if argv and sys.hexversion < 0x3000000 and not isinstance(argv[0], unicode):
 160                 argv = [portage._unicode_decode(x) for x in argv]
 161
 162         modes = {
 163                 'commit' : 'Run a scan then commit changes',
 164                 'ci' : 'Run a scan then commit changes',
 165                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 166                 'full' : 'Scan directory tree and print all issues (not a summary)',
 167                 'help' : 'Show this screen',
 168                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 169                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 170                 'scan' : 'Scan directory tree for QA issues'
 171         }
 172
 173         mode_keys = list(modes)
 174         mode_keys.sort()
 175
 176         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 177         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 178         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 179         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 180         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 181
 182         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 183                 help='Request a confirmation before commiting')
 184
 185         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 186                 help='specify a commit message on the command line')
 187
 188         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 189                 help='specify a path to a file that contains a commit message')
 190
 191         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 192                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 193
 194         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 195                 help='do not print unnecessary messages')
 196
 197         parser.add_option(
 198                 '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>",
 199                 help='for commit mode, call echangelog if ChangeLog is unmodified (or '
 200                 'regardless of modification if \'force\' is specified)')
 201
 202         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 203                 help='Commit with QA violations')
 204
 205         parser.add_option('--vcs', dest='vcs',
 206                 help='Force using specific VCS instead of autodetection')
 207
 208         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 209                 help='be very verbose in output', default=0)
 210
 211         parser.add_option('-V', '--version', dest='version', action='store_true',
 212                 help='show version info')
 213
 214         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 215                 default=False, help='forces the metadata.xml parse check to be carried out')
 216
 217         parser.add_option(
 218                 '--if-modified', type='choice', choices=('y', 'n'), default='n',
 219                 metavar="<y|n>",
 220                 help='only check packages that have uncommitted modifications')
 221
 222         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 223                 default=False, help='ignore arch-specific failures (where arch != host)')
 224
 225         parser.add_option("--ignore-default-opts",
 226                 action="store_true",
 227                 help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
 228
 229         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 230                 default=False, help='ignore masked packages (not allowed with commit mode)')
 231
 232         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 233                 default=False, help='include dev profiles in dependency checks')
 234
 235         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 236                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 237
 238         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 239                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 240
 241         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 242                 help='specify which mode repoman will run in (default=full)')
 243
 244         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 245
 246         for k in mode_keys:
 247                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 248
 249         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 250
 251         sorted_qa = list(qahelp)
 252         sorted_qa.sort()
 253         for k in sorted_qa:
 254                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 255
 256         opts, args = parser.parse_args(argv[1:])
 257
 258         if not opts.ignore_default_opts:
 259                 default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split()
 260                 if default_opts:
 261                         opts, args = parser.parse_args(default_opts + sys.argv[1:])
 262
 263         if opts.mode == 'help':
 264                 parser.print_help(short=False)
 265
 266         for arg in args:
 267                 if arg in modes:
 268                         if not opts.mode:
 269                                 opts.mode = arg
 270                                 break
 271                 else:
 272                         parser.error("invalid mode: %s" % arg)
 273
 274         if not opts.mode:
 275                 opts.mode = 'full'
 276
 277         if opts.mode == 'ci':
 278                 opts.mode = 'commit'  # backwards compat shortcut
 279
 280         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 281                 if opts.ignore_masked:
 282                         parser.error('Commit mode and --ignore-masked are not compatible')
 283                 if opts.without_mask:
 284                         parser.error('Commit mode and --without-mask are not compatible')
 285
 286         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 287         for val in range(opts.verbosity):
 288                 logger = logging.getLogger()
 289                 logger.setLevel(logger.getEffectiveLevel() - 10)
 290
 291         for val in range(opts.quiet):
 292                 logger = logging.getLogger()
 293                 logger.setLevel(logger.getEffectiveLevel() + 10)
 294
 295         return (opts, args)
 296
 297 qahelp={
 298         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 299         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 300         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 301         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 302         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 303         "changelog.missing":"Missing ChangeLog files",
 304         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 305         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 306         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 307         "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)",
 308         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit",
 309         "file.size":"Files in the files directory must be under 20 KiB",
 310         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 311         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 312         "file.UTF8":"File is not UTF8 compliant",
 313         "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf",
 314         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 315         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 316         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 317         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 318         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 319         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 320         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 321         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 322         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 323         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 324         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 325         "EAPI.definition":"EAPI is defined after an inherit call (must be defined before)",
 326         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 327         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 328         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 329         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 330         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 331         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 332         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 333         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 334         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 335         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 336         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 337         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 338         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 339         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 340         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 341         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 342         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 343         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 344         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 345         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 346         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 347         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 348         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 349         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 350         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 351         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 352         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 353         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 354         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 355         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 356         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 357         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 358         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 359         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 360         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 361         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 362         "variable.readonly":"Assigning a readonly variable",
 363         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 364         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 365         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 366         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 367         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 368         "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
 369         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 370         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 371         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 372         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 373         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 374         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 375         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 376         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 377         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 378         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 379         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 380         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 381         "ebuild.badheader":"This ebuild has a malformed header",
 382         "eprefixify.defined":"The ebuild uses eprefixify, but does not inherit the prefix eclass",
 383         "manifest.bad":"Manifest has missing or incorrect digests",
 384         "metadata.missing":"Missing metadata.xml files",
 385         "metadata.bad":"Bad metadata.xml files",
 386         "metadata.warning":"Warnings in metadata.xml files",
 387         "portage.internal":"The ebuild uses an internal Portage function",
 388         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 389         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 390         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 391 }
 392
 393 qacats = list(qahelp)
 394 qacats.sort()
 395
 396 qawarnings = set((
 397 "changelog.missing",
 398 "changelog.notadded",
 399 "dependency.unknown",
 400 "digest.assumed",
 401 "digest.unused",
 402 "ebuild.notadded",
 403 "ebuild.nostable",
 404 "ebuild.allmasked",
 405 "ebuild.nesteddie",
 406 "desktop.invalid",
 407 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 408 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 409 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 410 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 411 "DESCRIPTION.toolong",
 412 "EAPI.deprecated",
 413 "HOMEPAGE.virtual",
 414 "LICENSE.virtual",
 415 "KEYWORDS.dropped",
 416 "KEYWORDS.stupid",
 417 "KEYWORDS.missing",
 418 "IUSE.undefined",
 419 "PDEPEND.suspect",
 420 "RDEPEND.implicit",
 421 "RDEPEND.suspect",
 422 "RESTRICT.invalid",
 423 "SRC_URI.mirror",
 424 "ebuild.minorsyn",
 425 "ebuild.badheader",
 426 "ebuild.patches",
 427 "file.size",
 428 "inherit.autotools",
 429 "inherit.deprecated",
 430 "java.eclassesnotused",
 431 "wxwidgets.eclassnotused",
 432 "metadata.warning",
 433 "portage.internal",
 434 "usage.obsolete",
 435 "upstream.workaround",
 436 "virtual.oldstyle",
 437 "LIVEVCS.stable",
 438 "LIVEVCS.unmasked",
 439 ))
 440
 441 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 442
 443 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 444 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 445 allvars.update(Package.metadata_keys)
 446 allvars = sorted(allvars)
 447 commitmessage=None
 448 for x in missingvars:
 449         x += ".missing"
 450         if x not in qacats:
 451                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 452                 qacats.append(x)
 453                 qawarnings.add(x)
 454
 455 valid_restrict = frozenset(["binchecks", "bindist",
 456         "fetch", "installsources", "mirror",
 457         "primaryuri", "strip", "test", "userpriv"])
 458
 459 live_eclasses = frozenset([
 460         "bzr",
 461         "cvs",
 462         "darcs",
 463         "git",
 464         "git-2",
 465         "mercurial",
 466         "subversion",
 467         "tla",
 468 ])
 469
 470 suspect_rdepend = frozenset([
 471         "app-arch/cabextract",
 472         "app-arch/rpm2targz",
 473         "app-doc/doxygen",
 474         "dev-lang/nasm",
 475         "dev-lang/swig",
 476         "dev-lang/yasm",
 477         "dev-perl/extutils-pkgconfig",
 478         "dev-util/byacc",
 479         "dev-util/cmake",
 480         "dev-util/ftjam",
 481         "dev-util/gperf",
 482         "dev-util/gtk-doc",
 483         "dev-util/gtk-doc-am",
 484         "dev-util/intltool",
 485         "dev-util/jam",
 486         "dev-util/pkgconfig",
 487         "dev-util/scons",
 488         "dev-util/unifdef",
 489         "dev-util/yacc",
 490         "media-gfx/ebdftopcf",
 491         "sys-apps/help2man",
 492         "sys-devel/autoconf",
 493         "sys-devel/automake",
 494         "sys-devel/bin86",
 495         "sys-devel/bison",
 496         "sys-devel/dev86",
 497         "sys-devel/flex",
 498         "sys-devel/m4",
 499         "sys-devel/pmake",
 500         "virtual/linux-sources",
 501         "x11-misc/bdftopcf",
 502         "x11-misc/imake",
 503 ])
 504
 505 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 506 # force refetch if the local copy creation time is older than this
 507 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 508
 509 # file.executable
 510 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 511
 512 options, arguments = ParseArgs(sys.argv, qahelp)
 513
 514 if options.version:
 515         print("Portage", portage.VERSION)
 516         sys.exit(0)
 517
 518 # Set this to False when an extraordinary issue (generally
 519 # something other than a QA issue) makes it impossible to
 520 # commit (like if Manifest generation fails).
 521 can_force = True
 522
 523 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 524 if portdir is None:
 525         sys.exit(1)
 526
 527 myreporoot = os.path.basename(portdir_overlay)
 528 myreporoot += mydir[len(portdir_overlay):]
 529
 530 if options.vcs:
 531         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 532                 vcs = options.vcs
 533         else:
 534                 vcs = None
 535 else:
 536         vcses = utilities.FindVCS()
 537         if len(vcses) > 1:
 538                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 539                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 540                 sys.exit(1)
 541         elif vcses:
 542                 vcs = vcses[0]
 543         else:
 544                 vcs = None
 545
 546 if options.if_modified == "y" and vcs is None:
 547         logging.info("Not in a version controlled repository; "
 548                 "disabling --if-modified.")
 549         options.if_modified = "n"
 550
 551 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 552 vcs_preserves_mtime = vcs not in ('git',)
 553
 554 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 555 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 556 if vcs_global_opts is None:
 557         if vcs in ('cvs', 'svn'):
 558                 vcs_global_opts = "-q"
 559         else:
 560                 vcs_global_opts = ""
 561 vcs_global_opts = vcs_global_opts.split()
 562
 563 if options.mode == 'commit' and not options.pretend and not vcs:
 564         logging.info("Not in a version controlled repository; enabling pretend mode.")
 565         options.pretend = True
 566
 567 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 568 repoman_settings = portage.config(local_config=False, _eprefix=eprefix)
 569 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 570         (repoman_settings.get('PORTDIR_OVERLAY', ''),
 571         portage._shell_quote(portdir_overlay))
 572 # We have to call the config constructor again so
 573 # that config.repositories is initialized correctly.
 574 repoman_settings = portage.config(local_config=False, _eprefix=eprefix,
 575         env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 576
 577 root = repoman_settings['EROOT']
 578 trees = {
 579         root : {'porttree' : portage.portagetree(settings=repoman_settings)}
 580 }
 581 portdb = trees[root]['porttree'].dbapi
 582
 583 # Constrain dependency resolution to the master(s)
 584 # that are specified in layout.conf.
 585 repodir = os.path.realpath(portdir_overlay)
 586 repo_config = repoman_settings.repositories.get_repo_for_location(repodir)
 587 repo_info = portdb._repo_info[repodir]
 588 portdb.porttrees = list(repo_info.eclass_db.porttrees)
 589 portdir = portdb.porttrees[0]
 590
 591 # In order to disable manifest signatures, repos may set
 592 # "sign-manifests = false" in metadata/layout.conf. This
 593 # can be used to prevent merge conflicts like those that
 594 # thin-manifests is designed to prevent.
 595 sign_manifests = "sign" in repoman_settings.features and \
 596         repo_config.sign_manifest
 597
 598 manifest_hashes = repo_config.manifest_hashes
 599 if manifest_hashes is None:
 600         manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
 601
 602 if options.mode in ("commit", "fix", "manifest"):
 603         if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 604                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 605                 "metadata/layout.conf does not contain the '%s' hash which "
 606                 "is required by this portage version. You will have to "
 607                 "upgrade portage if you want to generate valid manifests for "
 608                 "this repository.") % \
 609                 (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
 610                 for line in textwrap.wrap(msg, 70):
 611                         logging.error(line)
 612                 sys.exit(1)
 613
 614         unsupported_hashes = manifest_hashes.difference(
 615                 portage.const.MANIFEST2_HASH_FUNCTIONS)
 616         if unsupported_hashes:
 617                 msg = ("The 'manifest-hashes' setting in the '%s' repository's "
 618                 "metadata/layout.conf contains one or more hash types '%s' "
 619                 "which are not supported by this portage version. You will "
 620                 "have to upgrade portage if you want to generate valid "
 621                 "manifests for this repository.") % \
 622                 (repo_config.name, " ".join(sorted(unsupported_hashes)))
 623                 for line in textwrap.wrap(msg, 70):
 624                         logging.error(line)
 625                 sys.exit(1)
 626
 627 if "commit" == options.mode and \
 628         repo_config.name == "gentoo" and \
 629         "RMD160" in manifest_hashes and \
 630         "RMD160" not in portage.checksum.hashorigin_map:
 631         msg = "Please install " \
 632         "pycrypto or enable python's ssl USE flag in order " \
 633         "to enable RMD160 hash support. See bug #198398 for " \
 634         "more information."
 635         prefix = bad(" * ")
 636         for line in textwrap.wrap(msg, 70):
 637                 print(prefix + line)
 638         sys.exit(1)
 639
 640 if options.echangelog is None and repo_config.update_changelog:
 641         options.echangelog = 'y'
 642
 643 if vcs is None:
 644         options.echangelog = 'n'
 645
 646 # The --echangelog option causes automatic ChangeLog generation,
 647 # which invalidates changelog.ebuildadded and changelog.missing
 648 # checks.
 649 # Note: Some don't use ChangeLogs in distributed SCMs.
 650 # It will be generated on server side from scm log,
 651 # before package moves to the rsync server.
 652 # This is needed because they try to avoid merge collisions.
 653 # Gentoo's Council decided to always use the ChangeLog file.
 654 # TODO: shouldn't this just be switched on the repo, iso the VCS?
 655 check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn')
 656
 657 logging.debug("repo config: %s" % (repo_config,))
 658 logging.debug("options: %s" % (options,))
 659
 660 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 661 # profile-specific config constructor calls.
 662 env = os.environ.copy()
 663 env['PORTDIR'] = portdir
 664 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 665
 666 logging.info('Setting paths:')
 667 logging.info('PORTDIR = "' + portdir + '"')
 668 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 669
 670 # It's confusing if these warnings are displayed without the user
 671 # being told which profile they come from, so disable them.
 672 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 673
 674 categories = []
 675 for path in repo_info.eclass_db.porttrees:
 676         categories.extend(portage.util.grabfile(
 677                 os.path.join(path, 'profiles', 'categories')))
 678 repoman_settings.categories = frozenset(
 679         portage.util.stack_lists([categories], incremental=1))
 680 categories = repoman_settings.categories
 681
 682 portdb.settings = repoman_settings
 683 root_config = RootConfig(repoman_settings, trees[root], None)
 684 # We really only need to cache the metadata that's necessary for visibility
 685 # filtering. Anything else can be discarded to reduce memory consumption.
 686 portdb._aux_cache_keys.clear()
 687 portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
 688
 689 reposplit = myreporoot.split(os.path.sep)
 690 repolevel = len(reposplit)
 691
 692 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 693 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 694 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 695 if options.mode == 'commit' and repolevel not in [1,2,3]:
 696         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 697         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 698         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 699         print(red("***"))
 700         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 701
 702 # Make startdir relative to the canonical repodir, so that we can pass
 703 # it to digestgen and it won't have to be canonicalized again.
 704 if repolevel == 1:
 705         startdir = repodir
 706 else:
 707         startdir = normalize_path(mydir)
 708         startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:])
 709
 710 def caterror(mycat):
 711         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 712
 713 class ProfileDesc(object):
 714         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 715         def __init__(self, arch, status, sub_path, tree_path):
 716                 self.arch = arch
 717                 self.status = status
 718                 if sub_path:
 719                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 720                 self.sub_path = sub_path
 721                 self.tree_path = tree_path
 722                 if tree_path:
 723                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 724                 else:
 725                         self.abs_path = tree_path
 726
 727         def __str__(self):
 728                 if self.sub_path:
 729                         return self.sub_path
 730                 return 'empty profile'
 731
 732 profile_list = []
 733 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 734
 735 # get lists of valid keywords, licenses, and use
 736 kwlist = set()
 737 liclist = set()
 738 uselist = set()
 739 global_pmasklines = []
 740
 741 for path in portdb.porttrees:
 742         try:
 743                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 744         except OSError:
 745                 pass
 746         kwlist.update(portage.grabfile(os.path.join(path,
 747                 "profiles", "arch.list")))
 748
 749         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 750         for x in use_desc:
 751                 x = x.split()
 752                 if x:
 753                         uselist.add(x[0])
 754
 755         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 756         try:
 757                 expand_list = os.listdir(expand_desc_dir)
 758         except OSError:
 759                 pass
 760         else:
 761                 for fn in expand_list:
 762                         if not fn[-5:] == '.desc':
 763                                 continue
 764                         use_prefix = fn[:-5].lower() + '_'
 765                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 766                                 x = x.split()
 767                                 if x:
 768                                         uselist.add(use_prefix + x[0])
 769
 770         global_pmasklines.append(portage.util.grabfile_package(
 771                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 772
 773         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 774         try:
 775                 desc_file = io.open(_unicode_encode(desc_path,
 776                         encoding=_encodings['fs'], errors='strict'),
 777                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 778         except EnvironmentError:
 779                 pass
 780         else:
 781                 for i, x in enumerate(desc_file):
 782                         if x[0] == "#":
 783                                 continue
 784                         arch = x.split()
 785                         if len(arch) == 0:
 786                                 continue
 787                         if len(arch) != 3:
 788                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 789                                         desc_path + " line %d" % (i+1, ))
 790                         elif arch[0] not in kwlist:
 791                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 792                                         desc_path + " line %d" % (i+1, ))
 793                         elif arch[2] not in valid_profile_types:
 794                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 795                                         desc_path + " line %d" % (i+1, ))
 796                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 797                         if not os.path.isdir(profile_desc.abs_path):
 798                                 logging.error(
 799                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 800                                         arch[2], arch[1], arch[0], desc_path, i+1)
 801                                 continue
 802                         if os.path.exists(
 803                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 804                                 continue
 805                         profile_list.append(profile_desc)
 806                 desc_file.close()
 807
 808 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 809 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 810
 811 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 812 global_pmaskdict = {}
 813 for x in global_pmasklines:
 814         global_pmaskdict.setdefault(x.cp, []).append(x)
 815 del global_pmasklines
 816
 817 def has_global_mask(pkg):
 818         mask_atoms = global_pmaskdict.get(pkg.cp)
 819         if mask_atoms:
 820                 pkg_list = [pkg]
 821                 for x in mask_atoms:
 822                         if portage.dep.match_from_list(x, pkg_list):
 823                                 return x
 824         return None
 825
 826 # Ensure that profile sub_path attributes are unique. Process in reverse order
 827 # so that profiles with duplicate sub_path from overlays will override
 828 # profiles with the same sub_path from parent repos.
 829 profiles = {}
 830 profile_list.reverse()
 831 profile_sub_paths = set()
 832 for prof in profile_list:
 833         if prof.sub_path in profile_sub_paths:
 834                 continue
 835         profile_sub_paths.add(prof.sub_path)
 836         profiles.setdefault(prof.arch, []).append(prof)
 837
 838 # Use an empty profile for checking dependencies of
 839 # packages that have empty KEYWORDS.
 840 prof = ProfileDesc('**', 'stable', '', '')
 841 profiles.setdefault(prof.arch, []).append(prof)
 842
 843 for x in repoman_settings.archlist():
 844         if x[0] == "~":
 845                 continue
 846         if x not in profiles:
 847                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 848                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 849                 print(red("up with the "+x+" team."))
 850                 print()
 851
 852 if not liclist:
 853         logging.fatal("Couldn't find licenses?")
 854         sys.exit(1)
 855
 856 if not kwlist:
 857         logging.fatal("Couldn't read KEYWORDS from arch.list")
 858         sys.exit(1)
 859
 860 if not uselist:
 861         logging.fatal("Couldn't find use.desc?")
 862         sys.exit(1)
 863
 864 scanlist=[]
 865 if repolevel==2:
 866         #we are inside a category directory
 867         catdir=reposplit[-1]
 868         if catdir not in categories:
 869                 caterror(catdir)
 870         mydirlist=os.listdir(startdir)
 871         for x in mydirlist:
 872                 if x == "CVS" or x.startswith("."):
 873                         continue
 874                 if os.path.isdir(startdir+"/"+x):
 875                         scanlist.append(catdir+"/"+x)
 876         repo_subdir = catdir + os.sep
 877 elif repolevel==1:
 878         for x in categories:
 879                 if not os.path.isdir(startdir+"/"+x):
 880                         continue
 881                 for y in os.listdir(startdir+"/"+x):
 882                         if y == "CVS" or y.startswith("."):
 883                                 continue
 884                         if os.path.isdir(startdir+"/"+x+"/"+y):
 885                                 scanlist.append(x+"/"+y)
 886         repo_subdir = ""
 887 elif repolevel==3:
 888         catdir = reposplit[-2]
 889         if catdir not in categories:
 890                 caterror(catdir)
 891         scanlist.append(catdir+"/"+reposplit[-1])
 892         repo_subdir = scanlist[-1] + os.sep
 893 else:
 894         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 895                 ' from the current working directory'
 896         logging.critical(msg)
 897         sys.exit(1)
 898
 899 repo_subdir_len = len(repo_subdir)
 900 scanlist.sort()
 901
 902 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 903
 904 def vcs_files_to_cps(vcs_file_iter):
 905         """
 906         Iterate over the given modified file paths returned from the vcs,
 907         and return a frozenset containing category/pn strings for each
 908         modified package.
 909         """
 910
 911         modified_cps = []
 912
 913         if repolevel == 3:
 914                 if reposplit[-2] in categories and \
 915                         next(vcs_file_iter, None) is not None:
 916                         modified_cps.append("/".join(reposplit[-2:]))
 917
 918         elif repolevel == 2:
 919                 category = reposplit[-1]
 920                 if category in categories:
 921                         for filename in vcs_file_iter:
 922                                 f_split = filename.split(os.sep)
 923                                 # ['.', pn,...]
 924                                 if len(f_split) > 2:
 925                                         modified_cps.append(category + "/" + f_split[1])
 926
 927         else:
 928                 # repolevel == 1
 929                 for filename in vcs_file_iter:
 930                         f_split = filename.split(os.sep)
 931                         # ['.', category, pn,...]
 932                         if len(f_split) > 3 and f_split[1] in categories:
 933                                 modified_cps.append("/".join(f_split[1:3]))
 934
 935         return frozenset(modified_cps)
 936
 937 def dev_keywords(profiles):
 938         """
 939         Create a set of KEYWORDS values that exist in 'dev'
 940         profiles. These are used
 941         to trigger a message notifying the user when they might
 942         want to add the --include-dev option.
 943         """
 944         type_arch_map = {}
 945         for arch, arch_profiles in profiles.items():
 946                 for prof in arch_profiles:
 947                         arch_set = type_arch_map.get(prof.status)
 948                         if arch_set is None:
 949                                 arch_set = set()
 950                                 type_arch_map[prof.status] = arch_set
 951                         arch_set.add(arch)
 952
 953         dev_keywords = type_arch_map.get('dev', set())
 954         dev_keywords.update(['~' + arch for arch in dev_keywords])
 955         return frozenset(dev_keywords)
 956
 957 dev_keywords = dev_keywords(profiles)
 958
 959 stats={}
 960 fails={}
 961
 962 # provided by the desktop-file-utils package
 963 desktop_file_validate = find_binary("desktop-file-validate")
 964 desktop_pattern = re.compile(r'.*\.desktop$')
 965
 966 for x in qacats:
 967         stats[x]=0
 968         fails[x]=[]
 969
 970 xmllint_capable = False
 971 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
 972
 973 def parsedate(s):
 974         """Parse a RFC 822 date and time string.
 975         This is required for python3 compatibility, since the
 976         rfc822.parsedate() function is not available."""
 977
 978         s_split = []
 979         for x in s.upper().split():
 980                 for y in x.split(','):
 981                         if y:
 982                                 s_split.append(y)
 983
 984         if len(s_split) != 6:
 985                 return None
 986
 987         # %a, %d %b %Y %H:%M:%S %Z
 988         a, d, b, Y, H_M_S, Z = s_split
 989
 990         # Convert month to integer, since strptime %w is locale-dependent.
 991         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
 992                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
 993         m = month_map.get(b)
 994         if m is None:
 995                 return None
 996         m = str(m).rjust(2, '0')
 997
 998         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
 999
1000 def fetch_metadata_dtd():
1001         """
1002         Fetch metadata.dtd if it doesn't exist or the ctime is older than
1003         metadata_dtd_ctime_interval.
1004         @rtype: bool
1005         @returns: True if successful, otherwise False
1006         """
1007
1008         must_fetch = True
1009         metadata_dtd_st = None
1010         current_time = int(time.time())
1011         try:
1012                 metadata_dtd_st = os.stat(metadata_dtd)
1013         except EnvironmentError as e:
1014                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1015                         raise
1016                 del e
1017         else:
1018                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
1019                 if abs(current_time - metadata_dtd_st.st_ctime) \
1020                         < metadata_dtd_ctime_interval:
1021                         must_fetch = False
1022
1023         if must_fetch:
1024                 print()
1025                 print(green("***") + " the local copy of metadata.dtd " + \
1026                         "needs to be refetched, doing that now")
1027                 print()
1028                 try:
1029                         url_f = urllib_request_urlopen(metadata_dtd_uri)
1030                         msg_info = url_f.info()
1031                         last_modified = msg_info.get('last-modified')
1032                         if last_modified is not None:
1033                                 last_modified = parsedate(last_modified)
1034                                 if last_modified is not None:
1035                                         last_modified = calendar.timegm(last_modified)
1036
1037                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
1038                         try:
1039                                 local_f = open(metadata_dtd_tmp, mode='wb')
1040                                 local_f.write(url_f.read())
1041                                 local_f.close()
1042                                 if last_modified is not None:
1043                                         try:
1044                                                 os.utime(metadata_dtd_tmp,
1045                                                         (int(last_modified), int(last_modified)))
1046                                         except OSError:
1047                                                 # This fails on some odd non-unix-like filesystems.
1048                                                 # We don't really need the mtime to be preserved
1049                                                 # anyway here (currently we use ctime to trigger
1050                                                 # fetch), so just ignore it.
1051                                                 pass
1052                                 os.rename(metadata_dtd_tmp, metadata_dtd)
1053                         finally:
1054                                 try:
1055                                         os.unlink(metadata_dtd_tmp)
1056                                 except OSError:
1057                                         pass
1058
1059                         url_f.close()
1060
1061                 except EnvironmentError as e:
1062                         print()
1063                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
1064                         print(red("!!!")+" exception '%s' though." % (e,))
1065                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
1066                         return False
1067
1068         return True
1069
1070 if options.mode == "manifest":
1071         pass
1072 elif not find_binary('xmllint'):
1073         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
1074         if options.xml_parse or repolevel==3:
1075                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
1076                 sys.exit(1)
1077 else:
1078         if not fetch_metadata_dtd():
1079                 sys.exit(1)
1080         #this can be problematic if xmllint changes their output
1081         xmllint_capable=True
1082
1083 if options.mode == 'commit' and vcs:
1084         utilities.detect_vcs_conflicts(options, vcs)
1085
1086 if options.mode == "manifest":
1087         pass
1088 elif options.pretend:
1089         print(green("\nRepoMan does a once-over of the neighborhood..."))
1090 else:
1091         print(green("\nRepoMan scours the neighborhood..."))
1092
1093 new_ebuilds = set()
1094 modified_ebuilds = set()
1095 modified_changelogs = set()
1096 mychanged = []
1097 mynew = []
1098 myremoved = []
1099
1100 if vcs == "cvs":
1101         mycvstree = cvstree.getentries("./", recursive=1)
1102         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1103         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1104         if options.if_modified == "y":
1105                 myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./")
1106
1107 if vcs == "svn":
1108         with os.popen("svn status") as f:
1109                 svnstatus = f.readlines()
1110         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1111         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1112         if options.if_modified == "y":
1113                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
1114
1115 elif vcs == "git":
1116         with os.popen("git diff-index --name-only "
1117                 "--relative --diff-filter=M HEAD") as f:
1118                 mychanged = f.readlines()
1119         mychanged = ["./" + elem[:-1] for elem in mychanged]
1120
1121         with os.popen("git diff-index --name-only "
1122                 "--relative --diff-filter=A HEAD") as f:
1123                 mynew = f.readlines()
1124         mynew = ["./" + elem[:-1] for elem in mynew]
1125         if options.if_modified == "y":
1126                 with os.popen("git diff-index --name-only "
1127                         "--relative --diff-filter=D HEAD") as f:
1128                         myremoved = f.readlines()
1129                 myremoved = ["./" + elem[:-1] for elem in myremoved]
1130
1131 elif vcs == "bzr":
1132         with os.popen("bzr status -S .") as f:
1133                 bzrstatus = f.readlines()
1134         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1135         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1136         if options.if_modified == "y":
1137                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
1138
1139 elif vcs == "hg":
1140         with os.popen("hg status --no-status --modified .") as f:
1141                 mychanged = f.readlines()
1142         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1143         mynew = os.popen("hg status --no-status --added .").readlines()
1144         mynew = ["./" + elem.rstrip() for elem in mynew]
1145         if options.if_modified == "y":
1146                 with os.popen("hg status --no-status --removed .") as f:
1147                         myremoved = f.readlines()
1148                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
1149
1150 if vcs:
1151         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1152         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1153         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1154                 if os.path.basename(x) == "ChangeLog")
1155
1156 have_pmasked = False
1157 have_dev_keywords = False
1158 dofail = 0
1159 arch_caches={}
1160 arch_xmatch_caches = {}
1161 shared_xmatch_caches = {"cp-list":{}}
1162
1163 # Disable the "ebuild.notadded" check when not in commit mode and
1164 # running `svn status` in every package dir will be too expensive.
1165
1166 check_ebuild_notadded = not \
1167         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1168
1169 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1170 thirdpartymirrors = []
1171 for v in repoman_settings.thirdpartymirrors().values():
1172         thirdpartymirrors.extend(v)
1173
1174 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1175         """
1176         Implements doctype() as required to avoid deprecation warnings with
1177         >=python-2.7.
1178         """
1179         def doctype(self, name, pubid, system):
1180                 pass
1181
1182 try:
1183         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1184 except (EnvironmentError, ParseError, PermissionDenied) as e:
1185         err(str(e))
1186 except FileNotFound:
1187         # TODO: Download as we do for metadata.dtd, but add a way to
1188         # disable for non-gentoo repoman users who may not have herds.
1189         herd_base = None
1190
1191 effective_scanlist = scanlist
1192 if options.if_modified == "y":
1193         effective_scanlist = sorted(vcs_files_to_cps(
1194                 chain(mychanged, mynew, myremoved)))
1195
1196 for x in effective_scanlist:
1197         #ebuilds and digests added to cvs respectively.
1198         logging.info("checking package %s" % x)
1199         eadded=[]
1200         catdir,pkgdir=x.split("/")
1201         checkdir=repodir+"/"+x
1202         checkdir_relative = ""
1203         if repolevel < 3:
1204                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1205         if repolevel < 2:
1206                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1207         checkdir_relative = os.path.join(".", checkdir_relative)
1208         generated_manifest = False
1209
1210         if options.mode == "manifest" or \
1211           (options.mode != 'manifest-check' and \
1212           'digest' in repoman_settings.features) or \
1213           options.mode in ('commit', 'fix') and not options.pretend:
1214                 auto_assumed = set()
1215                 fetchlist_dict = portage.FetchlistDict(checkdir,
1216                         repoman_settings, portdb)
1217                 if options.mode == 'manifest' and options.force:
1218                         portage._doebuild_manifest_exempt_depend += 1
1219                         try:
1220                                 distdir = repoman_settings['DISTDIR']
1221                                 mf = repoman_settings.repositories.get_repo_for_location(
1222                                         os.path.dirname(os.path.dirname(checkdir)))
1223                                 mf = mf.load_manifest(checkdir, distdir,
1224                                         fetchlist_dict=fetchlist_dict)
1225                                 mf.create(requiredDistfiles=None,
1226                                         assumeDistHashesAlways=True)
1227                                 for distfiles in fetchlist_dict.values():
1228                                         for distfile in distfiles:
1229                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1230                                                         mf.fhashdict['DIST'].pop(distfile, None)
1231                                                 else:
1232                                                         auto_assumed.add(distfile)
1233                                 mf.write()
1234                         finally:
1235                                 portage._doebuild_manifest_exempt_depend -= 1
1236
1237                 repoman_settings["O"] = checkdir
1238                 try:
1239                         generated_manifest = digestgen(
1240                                 mysettings=repoman_settings, myportdb=portdb)
1241                 except portage.exception.PermissionDenied as e:
1242                         generated_manifest = False
1243                         writemsg_level("!!! Permission denied: '%s'\n" % (e,),
1244                                 level=logging.ERROR, noiselevel=-1)
1245
1246                 if not generated_manifest:
1247                         print("Unable to generate manifest.")
1248                         dofail = 1
1249
1250                 if options.mode == "manifest":
1251                         if not dofail and options.force and auto_assumed and \
1252                                 'assume-digests' in repoman_settings.features:
1253                                 # Show which digests were assumed despite the --force option
1254                                 # being given. This output will already have been shown by
1255                                 # digestgen() if assume-digests is not enabled, so only show
1256                                 # it here if assume-digests is enabled.
1257                                 pkgs = list(fetchlist_dict)
1258                                 pkgs.sort()
1259                                 portage.writemsg_stdout("  digest.assumed" + \
1260                                         portage.output.colorize("WARN",
1261                                         str(len(auto_assumed)).rjust(18)) + "\n")
1262                                 for cpv in pkgs:
1263                                         fetchmap = fetchlist_dict[cpv]
1264                                         pf = portage.catsplit(cpv)[1]
1265                                         for distfile in sorted(fetchmap):
1266                                                 if distfile in auto_assumed:
1267                                                         portage.writemsg_stdout(
1268                                                                 "   %s::%s\n" % (pf, distfile))
1269                         continue
1270                 elif dofail:
1271                         sys.exit(1)
1272
1273         if not generated_manifest:
1274                 repoman_settings['O'] = checkdir
1275                 repoman_settings['PORTAGE_QUIET'] = '1'
1276                 if not portage.digestcheck([], repoman_settings, strict=1):
1277                         stats["manifest.bad"] += 1
1278                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1279                 repoman_settings.pop('PORTAGE_QUIET', None)
1280
1281         if options.mode == 'manifest-check':
1282                 continue
1283
1284         checkdirlist=os.listdir(checkdir)
1285         ebuildlist=[]
1286         pkgs = {}
1287         allvalid = True
1288         for y in checkdirlist:
1289                 if (y in no_exec or y.endswith(".ebuild")) and \
1290                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1291                                 stats["file.executable"] += 1
1292                                 fails["file.executable"].append(os.path.join(checkdir, y))
1293                 if y.endswith(".ebuild"):
1294                         pf = y[:-7]
1295                         ebuildlist.append(pf)
1296                         cpv = "%s/%s" % (catdir, pf)
1297                         try:
1298                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1299                         except KeyError:
1300                                 allvalid = False
1301                                 stats["ebuild.syntax"] += 1
1302                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1303                                 continue
1304                         except IOError:
1305                                 allvalid = False
1306                                 stats["ebuild.output"] += 1
1307                                 fails["ebuild.output"].append(os.path.join(x, y))
1308                                 continue
1309                         if not portage.eapi_is_supported(myaux["EAPI"]):
1310                                 allvalid = False
1311                                 stats["EAPI.unsupported"] += 1
1312                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1313                                 continue
1314                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1315                                 root_config=root_config, type_name="ebuild")
1316
1317         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1318         pkgsplits = {}
1319         for i in range(len(ebuildlist)):
1320                 ebuild_split = portage.pkgsplit(ebuildlist[i])
1321                 pkgsplits[ebuild_split] = ebuildlist[i]
1322                 ebuildlist[i] = ebuild_split
1323         ebuildlist.sort(key=cmp_sort_key(portage.pkgcmp))
1324         for i in range(len(ebuildlist)):
1325                 ebuildlist[i] = pkgsplits[ebuildlist[i]]
1326         del pkgsplits
1327
1328         slot_keywords = {}
1329
1330         if len(pkgs) != len(ebuildlist):
1331                 # If we can't access all the metadata then it's totally unsafe to
1332                 # commit since there's no way to generate a correct Manifest.
1333                 # Do not try to do any more QA checks on this package since missing
1334                 # metadata leads to false positives for several checks, and false
1335                 # positives confuse users.
1336                 can_force = False
1337                 continue
1338
1339         for y in checkdirlist:
1340                 m = disallowed_filename_chars_re.search(y.strip(os.sep))
1341                 if m is not None:
1342                         stats["file.name"] += 1
1343                         fails["file.name"].append("%s/%s: char '%s'" % \
1344                                 (checkdir, y, m.group(0)))
1345
1346                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1347                         continue
1348                 f = None
1349                 try:
1350                         line = 1
1351                         f = io.open(_unicode_encode(os.path.join(checkdir, y),
1352                                 encoding=_encodings['fs'], errors='strict'),
1353                                 mode='r', encoding=_encodings['repo.content'])
1354                         for l in f:
1355                                 line +=1
1356                 except UnicodeDecodeError as ue:
1357                         stats["file.UTF8"] += 1
1358                         s = ue.object[:ue.start]
1359                         l2 = s.count("\n")
1360                         line += l2
1361                         if l2 != 0:
1362                                 s = s[s.rfind("\n") + 1:]
1363                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1364                 finally:
1365                         if f is not None:
1366                                 f.close()
1367
1368         if vcs in ("git", "hg") and check_ebuild_notadded:
1369                 if vcs == "git":
1370                         myf = os.popen("git ls-files --others %s" % \
1371                                 (portage._shell_quote(checkdir_relative),))
1372                 if vcs == "hg":
1373                         myf = os.popen("hg status --no-status --unknown %s" % \
1374                                 (portage._shell_quote(checkdir_relative),))
1375                 for l in myf:
1376                         if l[:-1][-7:] == ".ebuild":
1377                                 stats["ebuild.notadded"] += 1
1378                                 fails["ebuild.notadded"].append(
1379                                         os.path.join(x, os.path.basename(l[:-1])))
1380                 myf.close()
1381
1382         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1383                 try:
1384                         if vcs == "cvs":
1385                                 myf=open(checkdir+"/CVS/Entries","r")
1386                         if vcs == "svn":
1387                                 myf = os.popen("svn status --depth=files --verbose " + checkdir)
1388                         if vcs == "bzr":
1389                                 myf = os.popen("bzr ls -v --kind=file " + checkdir)
1390                         myl = myf.readlines()
1391                         myf.close()
1392                         for l in myl:
1393                                 if vcs == "cvs":
1394                                         if l[0]!="/":
1395                                                 continue
1396                                         splitl=l[1:].split("/")
1397                                         if not len(splitl):
1398                                                 continue
1399                                         if splitl[0][-7:]==".ebuild":
1400                                                 eadded.append(splitl[0][:-7])
1401                                 if vcs == "svn":
1402                                         if l[:1] == "?":
1403                                                 continue
1404                                         if l[:7] == '      >':
1405                                                 # tree conflict, new in subversion 1.6
1406                                                 continue
1407                                         l = l.split()[-1]
1408                                         if l[-7:] == ".ebuild":
1409                                                 eadded.append(os.path.basename(l[:-7]))
1410                                 if vcs == "bzr":
1411                                         if l[1:2] == "?":
1412                                                 continue
1413                                         l = l.split()[-1]
1414                                         if l[-7:] == ".ebuild":
1415                                                 eadded.append(os.path.basename(l[:-7]))
1416                         if vcs == "svn":
1417                                 myf = os.popen("svn status " + checkdir)
1418                                 myl=myf.readlines()
1419                                 myf.close()
1420                                 for l in myl:
1421                                         if l[0] == "A":
1422                                                 l = l.rstrip().split(' ')[-1]
1423                                                 if l[-7:] == ".ebuild":
1424                                                         eadded.append(os.path.basename(l[:-7]))
1425                 except IOError:
1426                         if vcs == "cvs":
1427                                 stats["CVS/Entries.IO_error"] += 1
1428                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1429                         else:
1430                                 raise
1431                         continue
1432
1433         mf = repoman_settings.repositories.get_repo_for_location(
1434                 os.path.dirname(os.path.dirname(checkdir)))
1435         mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"])
1436         mydigests=mf.getTypeDigests("DIST")
1437
1438         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1439         myfiles_all = []
1440         src_uri_error = False
1441         for mykey in fetchlist_dict:
1442                 try:
1443                         myfiles_all.extend(fetchlist_dict[mykey])
1444                 except portage.exception.InvalidDependString as e:
1445                         src_uri_error = True
1446                         try:
1447                                 portdb.aux_get(mykey, ["SRC_URI"])
1448                         except KeyError:
1449                                 # This will be reported as an "ebuild.syntax" error.
1450                                 pass
1451                         else:
1452                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1453                                 fails["SRC_URI.syntax"].append(
1454                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1455         del fetchlist_dict
1456         if not src_uri_error:
1457                 # This test can produce false positives if SRC_URI could not
1458                 # be parsed for one or more ebuilds. There's no point in
1459                 # producing a false error here since the root cause will
1460                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1461                 # or "ebuild.sytax".
1462                 myfiles_all = set(myfiles_all)
1463                 for entry in mydigests:
1464                         if entry not in myfiles_all:
1465                                 stats["digest.unused"] += 1
1466                                 fails["digest.unused"].append(checkdir+"::"+entry)
1467                 for entry in myfiles_all:
1468                         if entry not in mydigests:
1469                                 stats["digest.missing"] += 1
1470                                 fails["digest.missing"].append(checkdir+"::"+entry)
1471         del myfiles_all
1472
1473         if os.path.exists(checkdir+"/files"):
1474                 filesdirlist=os.listdir(checkdir+"/files")
1475
1476                 # recurse through files directory
1477                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1478                 while filesdirlist:
1479                         y = filesdirlist.pop(0)
1480                         relative_path = os.path.join(x, "files", y)
1481                         full_path = os.path.join(repodir, relative_path)
1482                         try:
1483                                 mystat = os.stat(full_path)
1484                         except OSError as oe:
1485                                 if oe.errno == 2:
1486                                         # don't worry about it.  it likely was removed via fix above.
1487                                         continue
1488                                 else:
1489                                         raise oe
1490                         if S_ISDIR(mystat.st_mode):
1491                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1492                                 if y == "CVS" or y == ".svn":
1493                                         continue
1494                                 for z in os.listdir(checkdir+"/files/"+y):
1495                                         if z == "CVS" or z == ".svn":
1496                                                 continue
1497                                         filesdirlist.append(y+"/"+z)
1498                         # Current policy is no files over 20 KiB, these are the checks. File size between
1499                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1500                         elif mystat.st_size > 61440:
1501                                 stats["file.size.fatal"] += 1
1502                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1503                         elif mystat.st_size > 20480:
1504                                 stats["file.size"] += 1
1505                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1506
1507                         m = disallowed_filename_chars_re.search(
1508                                 os.path.basename(y.rstrip(os.sep)))
1509                         if m is not None:
1510                                 stats["file.name"] += 1
1511                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1512                                         (checkdir, y, m.group(0)))
1513
1514                         if desktop_file_validate and desktop_pattern.match(y):
1515                                 status, cmd_output = subprocess_getstatusoutput(
1516                                         "'%s' '%s'" % (desktop_file_validate, full_path))
1517                                 if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK:
1518                                         # Note: in the future we may want to grab the
1519                                         # warnings in addition to the errors. We're
1520                                         # just doing errors now since we don't want
1521                                         # to generate too much noise at first.
1522                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1523                                         for line in cmd_output.splitlines():
1524                                                 error_match = error_re.match(line)
1525                                                 if error_match is None:
1526                                                         continue
1527                                                 stats["desktop.invalid"] += 1
1528                                                 fails["desktop.invalid"].append(
1529                                                         relative_path + ': %s' % error_match.group(1))
1530
1531         del mydigests
1532
1533         if check_changelog and "ChangeLog" not in checkdirlist:
1534                 stats["changelog.missing"]+=1
1535                 fails["changelog.missing"].append(x+"/ChangeLog")
1536
1537         musedict = {}
1538         #metadata.xml file check
1539         if "metadata.xml" not in checkdirlist:
1540                 stats["metadata.missing"]+=1
1541                 fails["metadata.missing"].append(x+"/metadata.xml")
1542         #metadata.xml parse check
1543         else:
1544                 metadata_bad = False
1545
1546                 # read metadata.xml into memory
1547                 try:
1548                         _metadata_xml = xml.etree.ElementTree.parse(
1549                                 os.path.join(checkdir, "metadata.xml"),
1550                                 parser=xml.etree.ElementTree.XMLParser(
1551                                         target=_MetadataTreeBuilder()))
1552                 except (ExpatError, SyntaxError, EnvironmentError) as e:
1553                         metadata_bad = True
1554                         stats["metadata.bad"] += 1
1555                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1556                         del e
1557                 else:
1558                         # load USE flags from metadata.xml
1559                         try:
1560                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1561                         except portage.exception.ParseError as e:
1562                                 metadata_bad = True
1563                                 stats["metadata.bad"] += 1
1564                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1565
1566                         # Run other metadata.xml checkers
1567                         try:
1568                                 utilities.check_metadata(_metadata_xml, herd_base)
1569                         except (utilities.UnknownHerdsError, ) as e:
1570                                 metadata_bad = True
1571                                 stats["metadata.bad"] += 1
1572                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1573                                 del e
1574
1575                 #Only carry out if in package directory or check forced
1576                 if xmllint_capable and not metadata_bad:
1577                         # xmlint can produce garbage output even on success, so only dump
1578                         # the ouput when it fails.
1579                         st, out = subprocess_getstatusoutput(
1580                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1581                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1582                         if st != os.EX_OK:
1583                                 print(red("!!!") + " metadata.xml is invalid:")
1584                                 for z in out.splitlines():
1585                                         print(red("!!! ")+z)
1586                                 stats["metadata.bad"]+=1
1587                                 fails["metadata.bad"].append(x+"/metadata.xml")
1588
1589                 del metadata_bad
1590         muselist = frozenset(musedict)
1591
1592         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1593         changelog_modified = changelog_path in modified_changelogs
1594
1595         allmasked = True
1596         # detect unused local USE-descriptions
1597         used_useflags = set()
1598
1599         for y in ebuildlist:
1600                 relative_path = os.path.join(x, y + ".ebuild")
1601                 full_path = os.path.join(repodir, relative_path)
1602                 ebuild_path = y + ".ebuild"
1603                 if repolevel < 3:
1604                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1605                 if repolevel < 2:
1606                         ebuild_path = os.path.join(catdir, ebuild_path)
1607                 ebuild_path = os.path.join(".", ebuild_path)
1608                 if check_changelog and not changelog_modified \
1609                         and ebuild_path in new_ebuilds:
1610                         stats['changelog.ebuildadded'] += 1
1611                         fails['changelog.ebuildadded'].append(relative_path)
1612
1613                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1614                         #ebuild not added to vcs
1615                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1616                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1617                 myesplit=portage.pkgsplit(y)
1618                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1619                             or pv_toolong_re.search(myesplit[1]) \
1620                             or pv_toolong_re.search(myesplit[2]):
1621                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1622                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1623                         continue
1624                 elif myesplit[0]!=pkgdir:
1625                         print(pkgdir,myesplit[0])
1626                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1627                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1628                         continue
1629
1630                 pkg = pkgs[y]
1631
1632                 if pkg.invalid:
1633                         allvalid = False
1634                         for k, msgs in pkg.invalid.items():
1635                                 for msg in msgs:
1636                                         stats[k] = stats[k] + 1
1637                                         fails[k].append("%s %s" % (relative_path, msg))
1638                         continue
1639
1640                 myaux = pkg.metadata
1641                 eapi = myaux["EAPI"]
1642                 inherited = pkg.inherited
1643                 live_ebuild = live_eclasses.intersection(inherited)
1644
1645                 for k, v in myaux.items():
1646                         if not isinstance(v, basestring):
1647                                 continue
1648                         m = non_ascii_re.search(v)
1649                         if m is not None:
1650                                 stats["variable.invalidchar"] += 1
1651                                 fails["variable.invalidchar"].append(
1652                                         ("%s: %s variable contains non-ASCII " + \
1653                                         "character at position %s") % \
1654                                         (relative_path, k, m.start() + 1))
1655
1656                 if not src_uri_error:
1657                         # Check that URIs don't reference a server from thirdpartymirrors.
1658                         for uri in portage.dep.use_reduce( \
1659                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1660                                 contains_mirror = False
1661                                 for mirror in thirdpartymirrors:
1662                                         if uri.startswith(mirror):
1663                                                 contains_mirror = True
1664                                                 break
1665                                 if not contains_mirror:
1666                                         continue
1667
1668                                 stats["SRC_URI.mirror"] += 1
1669                                 fails["SRC_URI.mirror"].append(
1670                                         "%s: '%s' found in thirdpartymirrors" % \
1671                                         (relative_path, mirror))
1672
1673                 if myaux.get("PROVIDE"):
1674                         stats["virtual.oldstyle"]+=1
1675                         fails["virtual.oldstyle"].append(relative_path)
1676
1677                 for pos, missing_var in enumerate(missingvars):
1678                         if not myaux.get(missing_var):
1679                                 if catdir == "virtual" and \
1680                                         missing_var in ("HOMEPAGE", "LICENSE"):
1681                                         continue
1682                                 if live_ebuild and missing_var == "KEYWORDS":
1683                                         continue
1684                                 myqakey=missingvars[pos]+".missing"
1685                                 stats[myqakey]=stats[myqakey]+1
1686                                 fails[myqakey].append(x+"/"+y+".ebuild")
1687
1688                 if catdir == "virtual":
1689                         for var in ("HOMEPAGE", "LICENSE"):
1690                                 if myaux.get(var):
1691                                         myqakey = var + ".virtual"
1692                                         stats[myqakey] = stats[myqakey] + 1
1693                                         fails[myqakey].append(relative_path)
1694
1695                 # 14 is the length of DESCRIPTION=""
1696                 if len(myaux['DESCRIPTION']) > max_desc_len:
1697                         stats['DESCRIPTION.toolong'] += 1
1698                         fails['DESCRIPTION.toolong'].append(
1699                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1700                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1701
1702                 keywords = myaux["KEYWORDS"].split()
1703                 stable_keywords = []
1704                 for keyword in keywords:
1705                         if not keyword.startswith("~") and \
1706                                 not keyword.startswith("-"):
1707                                 stable_keywords.append(keyword)
1708                 if stable_keywords:
1709                         if ebuild_path in new_ebuilds:
1710                                 stable_keywords.sort()
1711                                 stats["KEYWORDS.stable"] += 1
1712                                 fails["KEYWORDS.stable"].append(
1713                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1714                                                 " ".join(stable_keywords))
1715
1716                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1717                         if not kw.startswith("-"))
1718
1719                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1720                 if previous_keywords is None:
1721                         slot_keywords[myaux["SLOT"]] = set()
1722                 elif ebuild_archs and not live_ebuild:
1723                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1724                         if dropped_keywords:
1725                                 stats["KEYWORDS.dropped"] += 1
1726                                 fails["KEYWORDS.dropped"].append(
1727                                         relative_path + ": %s" % \
1728                                         " ".join(sorted(dropped_keywords)))
1729
1730                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1731
1732                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1733                 if "-*" in keywords:
1734                         haskeyword = False
1735                         for kw in keywords:
1736                                 if kw[0] == "~":
1737                                         kw = kw[1:]
1738                                 if kw in kwlist:
1739                                         haskeyword = True
1740                         if not haskeyword:
1741                                 stats["KEYWORDS.stupid"] += 1
1742                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1743
1744                 """
1745                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1746                 not be allowed to be marked stable
1747                 """
1748                 if live_ebuild:
1749                         bad_stable_keywords = []
1750                         for keyword in keywords:
1751                                 if not keyword.startswith("~") and \
1752                                         not keyword.startswith("-"):
1753                                         bad_stable_keywords.append(keyword)
1754                                 del keyword
1755                         if bad_stable_keywords:
1756                                 stats["LIVEVCS.stable"] += 1
1757                                 fails["LIVEVCS.stable"].append(
1758                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1759                                                 bad_stable_keywords)
1760                         del bad_stable_keywords
1761
1762                         if keywords and not has_global_mask(pkg):
1763                                 stats["LIVEVCS.unmasked"] += 1
1764                                 fails["LIVEVCS.unmasked"].append(relative_path)
1765
1766                 if options.ignore_arches:
1767                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1768                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1769                 else:
1770                         arches=[]
1771                         for keyword in myaux["KEYWORDS"].split():
1772                                 if (keyword[0]=="-"):
1773                                         continue
1774                                 elif (keyword[0]=="~"):
1775                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1776                                 else:
1777                                         arches.append([keyword, keyword, [keyword]])
1778                                         allmasked = False
1779                         if not arches:
1780                                 # Use an empty profile for checking dependencies of
1781                                 # packages that have empty KEYWORDS.
1782                                 arches.append(['**', '**', ['**']])
1783
1784                 unknown_pkgs = set()
1785                 baddepsyntax = False
1786                 badlicsyntax = False
1787                 badprovsyntax = False
1788                 catpkg = catdir+"/"+y
1789
1790                 inherited_java_eclass = "java-pkg-2" in inherited or \
1791                         "java-pkg-opt-2" in inherited
1792                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1793                 operator_tokens = set(["||", "(", ")"])
1794                 type_list, badsyntax = [], []
1795                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1796                         "LICENSE", "PROPERTIES", "PROVIDE"):
1797                         mydepstr = myaux[mytype]
1798
1799                         token_class = None
1800                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1801                                 token_class=portage.dep.Atom
1802
1803                         try:
1804                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1805                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1806                         except portage.exception.InvalidDependString as e:
1807                                 atoms = None
1808                                 badsyntax.append(str(e))
1809
1810                         if atoms and mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1811                                 if mytype in ("RDEPEND", "PDEPEND") and \
1812                                         "test?" in mydepstr.split():
1813                                         stats[mytype + '.suspect'] += 1
1814                                         fails[mytype + '.suspect'].append(relative_path + \
1815                                                 ": 'test?' USE conditional in %s" % mytype)
1816
1817                                 for atom in atoms:
1818                                         if atom == "||":
1819                                                 continue
1820
1821                                         if not portdb.xmatch("match-all", atom) and \
1822                                                 not atom.cp.startswith("virtual/"):
1823                                                 unknown_pkgs.add((mytype, atom.unevaluated_atom))
1824
1825                                         is_blocker = atom.blocker
1826
1827                                         if mytype == "DEPEND" and \
1828                                                 not is_blocker and \
1829                                                 not inherited_java_eclass and \
1830                                                 atom.cp == "virtual/jdk":
1831                                                 stats['java.eclassesnotused'] += 1
1832                                                 fails['java.eclassesnotused'].append(relative_path)
1833                                         elif mytype == "DEPEND" and \
1834                                                 not is_blocker and \
1835                                                 not inherited_wxwidgets_eclass and \
1836                                                 atom.cp == "x11-libs/wxGTK":
1837                                                 stats['wxwidgets.eclassnotused'] += 1
1838                                                 fails['wxwidgets.eclassnotused'].append(
1839                                                         relative_path + ": DEPENDs on x11-libs/wxGTK"
1840                                                         " without inheriting wxwidgets.eclass")
1841                                         elif mytype in ("PDEPEND", "RDEPEND"):
1842                                                 if not is_blocker and \
1843                                                         atom.cp in suspect_rdepend:
1844                                                         stats[mytype + '.suspect'] += 1
1845                                                         fails[mytype + '.suspect'].append(
1846                                                                 relative_path + ": '%s'" % atom)
1847
1848                                         if atom.operator == "~" and \
1849                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1850                                                 stats[mytype + '.badtilde'] += 1
1851                                                 fails[mytype + '.badtilde'].append(
1852                                                         (relative_path + ": %s uses the ~ operator"
1853                                                          " with a non-zero revision:" + \
1854                                                          " '%s'") % (mytype, atom))
1855
1856                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1857
1858                 for m,b in zip(type_list, badsyntax):
1859                         stats[m+".syntax"] += 1
1860                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1861
1862                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1863                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1864                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1865                 badlicsyntax = badlicsyntax > 0
1866                 badprovsyntax = badprovsyntax > 0
1867
1868                 # uselist checks - global
1869                 myuse = []
1870                 default_use = []
1871                 for myflag in myaux["IUSE"].split():
1872                         flag_name = myflag.lstrip("+-")
1873                         used_useflags.add(flag_name)
1874                         if myflag != flag_name:
1875                                 default_use.append(myflag)
1876                         if flag_name not in uselist:
1877                                 myuse.append(flag_name)
1878
1879                 # uselist checks - metadata
1880                 for mypos in range(len(myuse)-1,-1,-1):
1881                         if myuse[mypos] and (myuse[mypos] in muselist):
1882                                 del myuse[mypos]
1883
1884                 if default_use and not eapi_has_iuse_defaults(eapi):
1885                         for myflag in default_use:
1886                                 stats['EAPI.incompatible'] += 1
1887                                 fails['EAPI.incompatible'].append(
1888                                         (relative_path + ": IUSE defaults" + \
1889                                         " not supported with EAPI='%s':" + \
1890                                         " '%s'") % (eapi, myflag))
1891
1892                 for mypos in range(len(myuse)):
1893                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1894                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1895
1896                 # license checks
1897                 if not badlicsyntax:
1898                         # Parse the LICENSE variable, remove USE conditions and
1899                         # flatten it.
1900                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
1901                         # Check each entry to ensure that it exists in PORTDIR's
1902                         # license directory.
1903                         for lic in licenses:
1904                                 # Need to check for "||" manually as no portage
1905                                 # function will remove it without removing values.
1906                                 if lic not in liclist and lic != "||":
1907                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1908                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
1909
1910                 #keyword checks
1911                 myuse = myaux["KEYWORDS"].split()
1912                 for mykey in myuse:
1913                         myskey=mykey[:]
1914                         if myskey[0]=="-":
1915                                 myskey=myskey[1:]
1916                         if myskey[0]=="~":
1917                                 myskey=myskey[1:]
1918                         if mykey!="-*":
1919                                 if myskey not in kwlist:
1920                                         stats["KEYWORDS.invalid"] += 1
1921                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1922                                 elif myskey not in profiles:
1923                                         stats["KEYWORDS.invalid"] += 1
1924                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1925
1926                 #restrict checks
1927                 myrestrict = None
1928                 try:
1929                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
1930                 except portage.exception.InvalidDependString as e:
1931                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1932                         fails["RESTRICT.syntax"].append(
1933                                 "%s: RESTRICT: %s" % (relative_path, e))
1934                         del e
1935                 if myrestrict:
1936                         myrestrict = set(myrestrict)
1937                         mybadrestrict = myrestrict.difference(valid_restrict)
1938                         if mybadrestrict:
1939                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
1940                                 for mybad in mybadrestrict:
1941                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1942                 #REQUIRED_USE check
1943                 required_use = myaux["REQUIRED_USE"]
1944                 if required_use:
1945                         if not eapi_has_required_use(eapi):
1946                                 stats['EAPI.incompatible'] += 1
1947                                 fails['EAPI.incompatible'].append(
1948                                         relative_path + ": REQUIRED_USE" + \
1949                                         " not supported with EAPI='%s'" % (eapi,))
1950                         try:
1951                                 portage.dep.check_required_use(required_use, (),
1952                                         pkg.iuse.is_valid_flag)
1953                         except portage.exception.InvalidDependString as e:
1954                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
1955                                 fails["REQUIRED_USE.syntax"].append(
1956                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
1957                                 del e
1958
1959                 # Syntax Checks
1960                 relative_path = os.path.join(x, y + ".ebuild")
1961                 full_path = os.path.join(repodir, relative_path)
1962                 if not vcs_preserves_mtime:
1963                         if ebuild_path not in new_ebuilds and \
1964                                 ebuild_path not in modified_ebuilds:
1965                                 pkg.mtime = None
1966                 try:
1967                         # All ebuilds should have utf_8 encoding.
1968                         f = io.open(_unicode_encode(full_path,
1969                                 encoding=_encodings['fs'], errors='strict'),
1970                                 mode='r', encoding=_encodings['repo.content'])
1971                         try:
1972                                 for check_name, e in run_checks(f, pkg):
1973                                         stats[check_name] += 1
1974                                         fails[check_name].append(relative_path + ': %s' % e)
1975                         finally:
1976                                 f.close()
1977                 except UnicodeDecodeError:
1978                         # A file.UTF8 failure will have already been recorded above.
1979                         pass
1980
1981                 if options.force:
1982                         # The dep_check() calls are the most expensive QA test. If --force
1983                         # is enabled, there's no point in wasting time on these since the
1984                         # user is intent on forcing the commit anyway.
1985                         continue
1986
1987                 for keyword,arch,groups in arches:
1988
1989                         if arch not in profiles:
1990                                 # A missing profile will create an error further down
1991                                 # during the KEYWORDS verification.
1992                                 continue
1993
1994                         for prof in profiles[arch]:
1995
1996                                 if prof.status not in ("stable", "dev") or \
1997                                         prof.status == "dev" and not options.include_dev:
1998                                         continue
1999
2000                                 dep_settings = arch_caches.get(prof.sub_path)
2001                                 if dep_settings is None:
2002                                         dep_settings = portage.config(
2003                                                 config_profile_path=prof.abs_path,
2004                                                 config_incrementals=repoman_incrementals,
2005                                                 local_config=False,
2006                                                 _unmatched_removal=options.unmatched_removal,
2007                                                 env=env, _eprefix=eprefix)
2008                                         dep_settings.categories = repoman_settings.categories
2009                                         if options.without_mask:
2010                                                 dep_settings._mask_manager = \
2011                                                         copy.deepcopy(dep_settings._mask_manager)
2012                                                 dep_settings._mask_manager._pmaskdict.clear()
2013                                         arch_caches[prof.sub_path] = dep_settings
2014
2015                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
2016                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
2017                                 if xcache is None:
2018                                         portdb.melt()
2019                                         portdb.freeze()
2020                                         xcache = portdb.xcache
2021                                         xcache.update(shared_xmatch_caches)
2022                                         arch_xmatch_caches[xmatch_cache_key] = xcache
2023
2024                                 trees[root]["porttree"].settings = dep_settings
2025                                 portdb.settings = dep_settings
2026                                 portdb.xcache = xcache
2027                                 # for package.use.mask support inside dep_check
2028                                 dep_settings.setcpv(pkg)
2029                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
2030                                 # just in case, prevent config.reset() from nuking these.
2031                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
2032
2033                                 if not baddepsyntax:
2034                                         ismasked = not ebuild_archs or \
2035                                                 pkg.cpv not in portdb.xmatch("match-visible", pkg.cp)
2036                                         if ismasked:
2037                                                 if not have_pmasked:
2038                                                         have_pmasked = bool(dep_settings._getMaskAtom(
2039                                                                 pkg.cpv, pkg.metadata))
2040                                                 if options.ignore_masked:
2041                                                         continue
2042                                                 #we are testing deps for a masked package; give it some lee-way
2043                                                 suffix="masked"
2044                                                 matchmode = "minimum-all"
2045                                         else:
2046                                                 suffix=""
2047                                                 matchmode = "minimum-visible"
2048
2049                                         if not have_dev_keywords:
2050                                                 have_dev_keywords = \
2051                                                         bool(dev_keywords.intersection(keywords))
2052
2053                                         if prof.status == "dev":
2054                                                 suffix=suffix+"indev"
2055
2056                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
2057
2058                                                 mykey=mytype+".bad"+suffix
2059                                                 myvalue = myaux[mytype]
2060                                                 if not myvalue:
2061                                                         continue
2062
2063                                                 success, atoms = portage.dep_check(myvalue, portdb,
2064                                                         dep_settings, use="all", mode=matchmode,
2065                                                         trees=trees)
2066
2067                                                 if success:
2068                                                         if atoms:
2069
2070                                                                 # Don't bother with dependency.unknown for
2071                                                                 # cases in which *DEPEND.bad is triggered.
2072                                                                 for atom in atoms:
2073                                                                         # dep_check returns all blockers and they
2074                                                                         # aren't counted for *DEPEND.bad, so we
2075                                                                         # ignore them here.
2076                                                                         if not atom.blocker:
2077                                                                                 unknown_pkgs.discard(
2078                                                                                         (mytype, atom.unevaluated_atom))
2079
2080                                                                 if not prof.sub_path:
2081                                                                         # old-style virtuals currently aren't
2082                                                                         # resolvable with empty profile, since
2083                                                                         # 'virtuals' mappings are unavailable
2084                                                                         # (it would be expensive to search
2085                                                                         # for PROVIDE in all ebuilds)
2086                                                                         atoms = [atom for atom in atoms if not \
2087                                                                                 (atom.cp.startswith('virtual/') and \
2088                                                                                 not portdb.cp_list(atom.cp))]
2089
2090                                                                 #we have some unsolvable deps
2091                                                                 #remove ! deps, which always show up as unsatisfiable
2092                                                                 atoms = [str(atom.unevaluated_atom) \
2093                                                                         for atom in atoms if not atom.blocker]
2094
2095                                                                 #if we emptied out our list, continue:
2096                                                                 if not atoms:
2097                                                                         continue
2098                                                                 stats[mykey]=stats[mykey]+1
2099                                                                 fails[mykey].append("%s: %s(%s) %s" % \
2100                                                                         (relative_path, keyword,
2101                                                                         prof, repr(atoms)))
2102                                                 else:
2103                                                         stats[mykey]=stats[mykey]+1
2104                                                         fails[mykey].append("%s: %s(%s) %s" % \
2105                                                                 (relative_path, keyword,
2106                                                                 prof, repr(atoms)))
2107
2108                 if not baddepsyntax and unknown_pkgs:
2109                         type_map = {}
2110                         for mytype, atom in unknown_pkgs:
2111                                 type_map.setdefault(mytype, set()).add(atom)
2112                         for mytype, atoms in type_map.items():
2113                                 stats["dependency.unknown"] += 1
2114                                 fails["dependency.unknown"].append("%s: %s: %s" %
2115                                         (relative_path, mytype, ", ".join(sorted(atoms))))
2116
2117         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
2118         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
2119         #if not portage.portdb.xmatch("bestmatch-visible",x):
2120         #    stats["ebuild.nostable"]+=1
2121         #    fails["ebuild.nostable"].append(x)
2122         if ebuildlist and allmasked and repolevel == 3:
2123                 stats["ebuild.allmasked"]+=1
2124                 fails["ebuild.allmasked"].append(x)
2125
2126         # check if there are unused local USE-descriptions in metadata.xml
2127         # (unless there are any invalids, to avoid noise)
2128         if allvalid:
2129                 for myflag in muselist.difference(used_useflags):
2130                         stats["metadata.warning"] += 1
2131                         fails["metadata.warning"].append(
2132                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
2133                                 (x, myflag))
2134
2135 if options.if_modified == "y" and len(effective_scanlist) < 1:
2136         logging.warn("--if-modified is enabled, but no modified packages were found!")
2137
2138 if options.mode == "manifest":
2139         sys.exit(dofail)
2140
2141 #dofail will be set to 1 if we have failed in at least one non-warning category
2142 dofail=0
2143 #dowarn will be set to 1 if we tripped any warnings
2144 dowarn=0
2145 #dofull will be set if we should print a "repoman full" informational message
2146 dofull = options.mode != 'full'
2147
2148 for x in qacats:
2149         if not stats[x]:
2150                 continue
2151         dowarn = 1
2152         if x not in qawarnings:
2153                 dofail = 1
2154
2155 if dofail or \
2156         (dowarn and not (options.quiet or options.mode == "scan")):
2157         dofull = 0
2158
2159 # Save QA output so that it can be conveniently displayed
2160 # in $EDITOR while the user creates a commit message.
2161 # Otherwise, the user would not be able to see this output
2162 # once the editor has taken over the screen.
2163 qa_output = io.StringIO()
2164 style_file = ConsoleStyleFile(sys.stdout)
2165 if options.mode == 'commit' and \
2166         (not commitmessage or not commitmessage.strip()):
2167         style_file.write_listener = qa_output
2168 console_writer = StyleWriter(file=style_file, maxcol=9999)
2169 console_writer.style_listener = style_file.new_styles
2170
2171 f = formatter.AbstractFormatter(console_writer)
2172
2173 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
2174
2175 style_file.flush()
2176 del console_writer, f, style_file
2177 qa_output = qa_output.getvalue()
2178 qa_output = qa_output.splitlines(True)
2179
2180 def grouplist(mylist,seperator="/"):
2181         """(list,seperator="/") -- Takes a list of elements; groups them into
2182         same initial element categories. Returns a dict of {base:[sublist]}
2183         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2184         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2185         mygroups={}
2186         for x in mylist:
2187                 xs=x.split(seperator)
2188                 if xs[0]==".":
2189                         xs=xs[1:]
2190                 if xs[0] not in mygroups:
2191                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2192                 else:
2193                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2194         return mygroups
2195
2196 suggest_ignore_masked = False
2197 suggest_include_dev = False
2198
2199 if have_pmasked and not (options.without_mask or options.ignore_masked):
2200         suggest_ignore_masked = True
2201 if have_dev_keywords and not options.include_dev:
2202         suggest_include_dev = True
2203
2204 if suggest_ignore_masked or suggest_include_dev:
2205         print()
2206         if suggest_ignore_masked:
2207                 print(bold("Note: use --without-mask to check " + \
2208                         "KEYWORDS on dependencies of masked packages"))
2209
2210         if suggest_include_dev:
2211                 print(bold("Note: use --include-dev (-d) to check " + \
2212                         "dependencies for 'dev' profiles"))
2213         print()
2214
2215 if options.mode != 'commit':
2216         if dofull:
2217                 print(bold("Note: type \"repoman full\" for a complete listing."))
2218         if dowarn and not dofail:
2219                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2220         elif not dofail:
2221                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2222         elif dofail:
2223                 print(bad("Please fix these important QA issues first."))
2224                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2225                 sys.exit(1)
2226 else:
2227         if dofail and can_force and options.force and not options.pretend:
2228                 print(green("RepoMan sez:") + \
2229                         " \"You want to commit even with these QA issues?\n" + \
2230                         "              I'll take it this time, but I'm not happy.\"\n")
2231         elif dofail:
2232                 if options.force and not can_force:
2233                         print(bad("The --force option has been disabled due to extraordinary issues."))
2234                 print(bad("Please fix these important QA issues first."))
2235                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2236                 sys.exit(1)
2237
2238         if options.pretend:
2239                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2240
2241         myunadded = []
2242         if vcs == "cvs":
2243                 try:
2244                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2245                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2246                 except SystemExit as e:
2247                         raise  # TODO propagate this
2248                 except:
2249                         err("Error retrieving CVS tree; exiting.")
2250         if vcs == "svn":
2251                 try:
2252                         with os.popen("svn status --no-ignore") as f:
2253                                 svnstatus = f.readlines()
2254                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2255                 except SystemExit as e:
2256                         raise  # TODO propagate this
2257                 except:
2258                         err("Error retrieving SVN info; exiting.")
2259         if vcs == "git":
2260                 # get list of files not under version control or missing
2261                 myf = os.popen("git ls-files --others")
2262                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2263                 myf.close()
2264         if vcs == "bzr":
2265                 try:
2266                         with os.popen("bzr status -S .") as f:
2267                                 bzrstatus = f.readlines()
2268                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2269                 except SystemExit as e:
2270                         raise  # TODO propagate this
2271                 except:
2272                         err("Error retrieving bzr info; exiting.")
2273         if vcs == "hg":
2274                 with os.popen("hg status --no-status --unknown .") as f:
2275                         myunadded = f.readlines()
2276                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2277
2278                 # Mercurial doesn't handle manually deleted files as removed from
2279                 # the repository, so the user need to remove them before commit,
2280                 # using "hg remove [FILES]"
2281                 with os.popen("hg status --no-status --deleted .") as f:
2282                         mydeleted = f.readlines()
2283                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2284
2285
2286         myautoadd=[]
2287         if myunadded:
2288                 for x in range(len(myunadded)-1,-1,-1):
2289                         xs=myunadded[x].split("/")
2290                         if xs[-1]=="files":
2291                                 print("!!! files dir is not added! Please correct this.")
2292                                 sys.exit(-1)
2293                         elif xs[-1]=="Manifest":
2294                                 # It's a manifest... auto add
2295                                 myautoadd+=[myunadded[x]]
2296                                 del myunadded[x]
2297
2298         if myunadded:
2299                 print(red("!!! The following files are in your local tree but are not added to the master"))
2300                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2301                 for x in myunadded:
2302                         print("   ",x)
2303                 print()
2304                 print()
2305                 sys.exit(1)
2306
2307         if vcs == "hg" and mydeleted:
2308                 print(red("!!! The following files are removed manually from your local tree but are not"))
2309                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2310                 for x in mydeleted:
2311                         print("   ",x)
2312                 print()
2313                 print()
2314                 sys.exit(1)
2315
2316         if vcs == "cvs":
2317                 mycvstree = cvstree.getentries("./", recursive=1)
2318                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2319                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2320                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2321                 bin_blob_pattern = re.compile("^-kb$")
2322                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2323                         recursive=1, basedir="./"))
2324
2325
2326         if vcs == "svn":
2327                 with os.popen("svn status") as f:
2328                         svnstatus = f.readlines()
2329                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2330                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2331                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2332
2333                 # Subversion expands keywords specified in svn:keywords properties.
2334                 with os.popen("svn propget -R svn:keywords") as f:
2335                         props = f.readlines()
2336                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2337                         for prop in props if " - " in prop)
2338
2339         elif vcs == "git":
2340                 with os.popen("git diff-index --name-only "
2341                         "--relative --diff-filter=M HEAD") as f:
2342                         mychanged = f.readlines()
2343                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2344
2345                 with os.popen("git diff-index --name-only "
2346                         "--relative --diff-filter=A HEAD") as f:
2347                         mynew = f.readlines()
2348                 mynew = ["./" + elem[:-1] for elem in mynew]
2349
2350                 with os.popen("git diff-index --name-only "
2351                         "--relative --diff-filter=D HEAD") as f:
2352                         myremoved = f.readlines()
2353                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2354
2355         if vcs == "bzr":
2356                 with os.popen("bzr status -S .") as f:
2357                         bzrstatus = f.readlines()
2358                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2359                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2360                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2361                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2362                 # Bazaar expands nothing.
2363
2364         if vcs == "hg":
2365                 with os.popen("hg status --no-status --modified .") as f:
2366                         mychanged = f.readlines()
2367                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2368
2369                 with os.popen("hg status --no-status --added .") as f:
2370                         mynew = f.readlines()
2371                 mynew = ["./" + elem.rstrip() for elem in mynew]
2372
2373                 with os.popen("hg status --no-status --removed .") as f:
2374                         myremoved = f.readlines()
2375                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2376
2377         if vcs:
2378                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2379                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2380                         print()
2381                         print("(Didn't find any changed files...)")
2382                         print()
2383                         sys.exit(1)
2384
2385         # Manifests need to be regenerated after all other commits, so don't commit
2386         # them now even if they have changed.
2387         mymanifests = set()
2388         myupdates = set()
2389         for f in mychanged + mynew:
2390                 if "Manifest" == os.path.basename(f):
2391                         mymanifests.add(f)
2392                 else:
2393                         myupdates.add(f)
2394         myupdates.difference_update(myremoved)
2395         myupdates = list(myupdates)
2396         mymanifests = list(mymanifests)
2397         myheaders = []
2398         mydirty = []
2399
2400         commitmessage = options.commitmsg
2401         if options.commitmsgfile:
2402                 try:
2403                         f = io.open(_unicode_encode(options.commitmsgfile,
2404                         encoding=_encodings['fs'], errors='strict'),
2405                         mode='r', encoding=_encodings['content'], errors='replace')
2406                         commitmessage = f.read()
2407                         f.close()
2408                         del f
2409                 except (IOError, OSError) as e:
2410                         if e.errno == errno.ENOENT:
2411                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2412                         else:
2413                                 raise
2414                 # We've read the content so the file is no longer needed.
2415                 commitmessagefile = None
2416         if not commitmessage or not commitmessage.strip():
2417                 try:
2418                         editor = os.environ.get("EDITOR")
2419                         if editor and utilities.editor_is_executable(editor):
2420                                 commitmessage = utilities.get_commit_message_with_editor(
2421                                         editor, message=qa_output)
2422                         else:
2423                                 commitmessage = utilities.get_commit_message_with_stdin()
2424                 except KeyboardInterrupt:
2425                         exithandler()
2426                 if not commitmessage or not commitmessage.strip():
2427                         print("* no commit message?  aborting commit.")
2428                         sys.exit(1)
2429         commitmessage = commitmessage.rstrip()
2430         changelog_msg = commitmessage
2431         portage_version = getattr(portage, "VERSION", None)
2432         if portage_version is None:
2433                 sys.stderr.write("Failed to insert portage version in message!\n")
2434                 sys.stderr.flush()
2435                 portage_version = "Unknown"
2436         unameout = platform.system() + " "
2437         if platform.system() in ["Darwin", "SunOS"]:
2438                 unameout += platform.processor()
2439         else:
2440                 unameout += platform.machine()
2441         commitmessage += "\n\n(Portage version: %s/%s/%s" % \
2442                 (portage_version, vcs, unameout)
2443         if options.force:
2444                 commitmessage += ", RepoMan options: --force"
2445         commitmessage += ")"
2446
2447         if options.ask and userquery('Commit changes?', True) != 'Yes':
2448                 print("* aborting commit.")
2449                 sys.exit(1)
2450
2451         if options.echangelog in ('y', 'force'):
2452                 logging.info("checking for unmodified ChangeLog files")
2453                 committer_name = utilities.get_committer_name(env=repoman_settings)
2454                 for x in sorted(vcs_files_to_cps(
2455                         chain(myupdates, mymanifests, myremoved))):
2456                         catdir, pkgdir = x.split("/")
2457                         checkdir = repodir + "/" + x
2458                         checkdir_relative = ""
2459                         if repolevel < 3:
2460                                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
2461                         if repolevel < 2:
2462                                 checkdir_relative = os.path.join(catdir, checkdir_relative)
2463                         checkdir_relative = os.path.join(".", checkdir_relative)
2464
2465                         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
2466                         changelog_modified = changelog_path in modified_changelogs
2467                         if changelog_modified and options.echangelog != 'force':
2468                                 continue
2469
2470                         # get changes for this package
2471                         cdrlen = len(checkdir_relative)
2472                         clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)]
2473                         clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)]
2474                         clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)]
2475                         new_changelog = utilities.UpdateChangeLog(checkdir_relative,
2476                                 committer_name, changelog_msg,
2477                                 os.path.join(repodir, 'skel.ChangeLog'),
2478                                 catdir, pkgdir,
2479                                 new=clnew, removed=clremoved, changed=clchanged,
2480                                 pretend=options.pretend)
2481                         if new_changelog is None:
2482                                 writemsg_level("!!! Updating the ChangeLog failed\n", \
2483                                         level=logging.ERROR, noiselevel=-1)
2484                                 sys.exit(1)
2485
2486                         # if the ChangeLog was just created, add it to vcs
2487                         if new_changelog:
2488                                 myautoadd.append(changelog_path)
2489                                 # myautoadd is appended to myupdates below
2490                         else:
2491                                 myupdates.append(changelog_path)
2492
2493         if myautoadd:
2494                 print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
2495                 add_cmd = [vcs, "add"]
2496                 add_cmd += myautoadd
2497                 if options.pretend:
2498                         portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
2499                                 noiselevel=-1)
2500                 else:
2501                         if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
2502                                 # Python 3.1 produces the following TypeError if raw bytes are
2503                                 # passed to subprocess.call():
2504                                 #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
2505                                 #     errread, errwrite)
2506                                 #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
2507                                 #     raise child_exception
2508                                 # TypeError: expected an object with the buffer interface
2509                                 add_cmd = [_unicode_encode(arg) for arg in add_cmd]
2510                         retcode = subprocess.call(add_cmd)
2511                         if retcode != os.EX_OK:
2512                                 logging.error(
2513                                         "Exiting on %s error code: %s\n" % (vcs, retcode))
2514                                 sys.exit(retcode)
2515
2516                 myupdates += myautoadd
2517
2518         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2519
2520         if vcs not in ('cvs', 'svn'):
2521                 # With git, bzr and hg, there's never any keyword expansion, so
2522                 # there's no need to regenerate manifests and all files will be
2523                 # committed in one big commit at the end.
2524                 print()
2525         elif not repo_config.thin_manifest:
2526                 if vcs == 'cvs':
2527                         headerstring = "'\$(Header|Id).*\$'"
2528                 elif vcs == "svn":
2529                         svn_keywords = dict((k.lower(), k) for k in [
2530                                         "Rev",
2531                                         "Revision",
2532                                         "LastChangedRevision",
2533                                         "Date",
2534                                         "LastChangedDate",
2535                                         "Author",
2536                                         "LastChangedBy",
2537                                         "URL",
2538                                         "HeadURL",
2539                                         "Id",
2540                                         "Header",
2541                         ])
2542
2543                 for myfile in myupdates:
2544
2545                         # for CVS, no_expansion contains files that are excluded from expansion
2546                         if vcs == "cvs":
2547                                 if myfile in no_expansion:
2548                                         continue
2549
2550                         # for SVN, expansion contains files that are included in expansion
2551                         elif vcs == "svn":
2552                                 if myfile not in expansion:
2553                                         continue
2554
2555                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2556                                 enabled_keywords = []
2557                                 for k in expansion[myfile]:
2558                                         keyword = svn_keywords.get(k.lower())
2559                                         if keyword is not None:
2560                                                 enabled_keywords.append(keyword)
2561
2562                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2563
2564                         myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile)
2565                         if myout[0] == 0:
2566                                 myheaders.append(myfile)
2567
2568                 print("%s have headers that will change." % green(str(len(myheaders))))
2569                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2570
2571         logging.info("myupdates: %s", myupdates)
2572         logging.info("myheaders: %s", myheaders)
2573
2574         # Handle the case where committed files have keywords which
2575         # will change and need a priming commit before the Manifest
2576         # can be committed.
2577         if (myupdates or myremoved) and myheaders:
2578                 myfiles = myupdates + myremoved
2579                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2580                 mymsg = os.fdopen(fd, "wb")
2581                 mymsg.write(_unicode_encode(commitmessage))
2582                 mymsg.close()
2583
2584                 print()
2585                 print(green("Using commit message:"))
2586                 print(green("------------------------------------------------------------------------------"))
2587                 print(commitmessage)
2588                 print(green("------------------------------------------------------------------------------"))
2589                 print()
2590
2591                 # Having a leading ./ prefix on file paths can trigger a bug in
2592                 # the cvs server when committing files to multiple directories,
2593                 # so strip the prefix.
2594                 myfiles = [f.lstrip("./") for f in myfiles]
2595
2596                 commit_cmd = [vcs]
2597                 commit_cmd.extend(vcs_global_opts)
2598                 commit_cmd.append("commit")
2599                 commit_cmd.extend(vcs_local_opts)
2600                 commit_cmd.extend(["-F", commitmessagefile])
2601                 commit_cmd.extend(myfiles)
2602
2603                 try:
2604                         if options.pretend:
2605                                 print("(%s)" % (" ".join(commit_cmd),))
2606                         else:
2607                                 retval = spawn(commit_cmd, env=os.environ)
2608                                 if retval != os.EX_OK:
2609                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2610                                                 "error code: %s\n") % (vcs, retval),
2611                                                 level=logging.ERROR, noiselevel=-1)
2612                                         sys.exit(retval)
2613                 finally:
2614                         try:
2615                                 os.unlink(commitmessagefile)
2616                         except OSError:
2617                                 pass
2618
2619         # Setup the GPG commands
2620         def gpgsign(filename):
2621                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2622                 if gpgcmd is None:
2623                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2624                                 " Is make.globals missing?")
2625                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2626                         "PORTAGE_GPG_KEY" not in repoman_settings:
2627                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2628                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2629                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2630                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2631                                         os.path.expanduser("~/.gnupg")
2632                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2633                                         % repoman_settings["PORTAGE_GPG_DIR"])
2634                         else:
2635                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2636                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2637                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2638                                 raise portage.exception.InvalidLocation(
2639                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2640                                         repoman_settings["PORTAGE_GPG_DIR"])
2641                 gpgvars = {"FILE": filename}
2642                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2643                         v = repoman_settings.get(k)
2644                         if v is not None:
2645                                 gpgvars[k] = v
2646                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2647                 if options.pretend:
2648                         print("("+gpgcmd+")")
2649                 else:
2650                         rValue = os.system(gpgcmd)
2651                         if rValue == os.EX_OK:
2652                                 os.rename(filename+".asc", filename)
2653                         else:
2654                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2655
2656         # When files are removed and re-added, the cvs server will put /Attic/
2657         # inside the $Header path. This code detects the problem and corrects it
2658         # so that the Manifest will generate correctly. See bug #169500.
2659         # Use binary mode in order to avoid potential character encoding issues.
2660         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2661         attic_str = b'/Attic/'
2662         attic_replace = b'/'
2663         for x in myheaders:
2664                 f = open(_unicode_encode(x,
2665                         encoding=_encodings['fs'], errors='strict'),
2666                         mode='rb')
2667                 mylines = f.readlines()
2668                 f.close()
2669                 modified = False
2670                 for i, line in enumerate(mylines):
2671                         if cvs_header_re.match(line) is not None and \
2672                                 attic_str in line:
2673                                 mylines[i] = line.replace(attic_str, attic_replace)
2674                                 modified = True
2675                 if modified:
2676                         portage.util.write_atomic(x, b''.join(mylines),
2677                                 mode='wb')
2678
2679         if repolevel == 1:
2680                 print(green("RepoMan sez:"), "\"You're rather crazy... "
2681                         "doing the entire repository.\"\n")
2682
2683         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2684
2685                 for x in sorted(vcs_files_to_cps(
2686                         chain(myupdates, myremoved, mymanifests))):
2687                         repoman_settings["O"] = os.path.join(repodir, x)
2688                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2689
2690         signed = False
2691         if sign_manifests:
2692                 signed = True
2693                 try:
2694                         for x in sorted(vcs_files_to_cps(
2695                                 chain(myupdates, myremoved, mymanifests))):
2696                                 repoman_settings["O"] = os.path.join(repodir, x)
2697                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2698                 except portage.exception.PortageException as e:
2699                         portage.writemsg("!!! %s\n" % str(e))
2700                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2701                         signed = False
2702
2703         if vcs == 'git':
2704                 # It's not safe to use the git commit -a option since there might
2705                 # be some modified files elsewhere in the working tree that the
2706                 # user doesn't want to commit. Therefore, call git update-index
2707                 # in order to ensure that the index is updated with the latest
2708                 # versions of all new and modified files in the relevant portion
2709                 # of the working tree.
2710                 myfiles = mymanifests + myupdates
2711                 myfiles.sort()
2712                 update_index_cmd = ["git", "update-index"]
2713                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2714                 if options.pretend:
2715                         print("(%s)" % (" ".join(update_index_cmd),))
2716                 else:
2717                         retval = spawn(update_index_cmd, env=os.environ)
2718                         if retval != os.EX_OK:
2719                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2720                                         "error code: %s\n") % (vcs, retval),
2721                                         level=logging.ERROR, noiselevel=-1)
2722                                 sys.exit(retval)
2723
2724         if True:
2725
2726                 myfiles = mymanifests[:]
2727                 # If there are no header (SVN/CVS keywords) changes in
2728                 # the files, this Manifest commit must include the
2729                 # other (yet uncommitted) files.
2730                 if not myheaders:
2731                         myfiles += myupdates
2732                         myfiles += myremoved
2733                 myfiles.sort()
2734
2735                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2736                 mymsg = os.fdopen(fd, "wb")
2737                 # strip the closing parenthesis
2738                 mymsg.write(_unicode_encode(commitmessage[:-1]))
2739                 if signed:
2740                         mymsg.write(_unicode_encode(
2741                                 ", signed Manifest commit with key %s)" % \
2742                                 repoman_settings["PORTAGE_GPG_KEY"]))
2743                 else:
2744                         mymsg.write(b", unsigned Manifest commit)")
2745                 mymsg.close()
2746
2747                 commit_cmd = []
2748                 if options.pretend and vcs is None:
2749                         # substitute a bogus value for pretend output
2750                         commit_cmd.append("cvs")
2751                 else:
2752                         commit_cmd.append(vcs)
2753                 commit_cmd.extend(vcs_global_opts)
2754                 commit_cmd.append("commit")
2755                 commit_cmd.extend(vcs_local_opts)
2756                 if vcs == "hg":
2757                         commit_cmd.extend(["--logfile", commitmessagefile])
2758                         commit_cmd.extend(myfiles)
2759                 else:
2760                         commit_cmd.extend(["-F", commitmessagefile])
2761                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2762
2763                 try:
2764                         if options.pretend:
2765                                 print("(%s)" % (" ".join(commit_cmd),))
2766                         else:
2767                                 retval = spawn(commit_cmd, env=os.environ)
2768                                 if retval != os.EX_OK:
2769                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2770                                                 "error code: %s\n") % (vcs, retval),
2771                                                 level=logging.ERROR, noiselevel=-1)
2772                                         sys.exit(retval)
2773                 finally:
2774                         try:
2775                                 os.unlink(commitmessagefile)
2776                         except OSError:
2777                                 pass
2778
2779         print()
2780         if vcs:
2781                 print("Commit complete.")
2782         else:
2783                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2784         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2785 sys.exit(0)
2786