bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2011 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # Next to do: dep syntax checking in mask files
   6 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   7 # that last one is tricky because multiple profiles need to be checked.
   8
   9 from __future__ import print_function
  10
  11 import calendar
  12 import codecs
  13 import copy
  14 import errno
  15 import formatter
  16 import logging
  17 import optparse
  18 import re
  19 import signal
  20 import stat
  21 import sys
  22 import tempfile
  23 import time
  24 import platform
  25
  26 try:
  27         from urllib.request import urlopen as urllib_request_urlopen
  28 except ImportError:
  29         from urllib import urlopen as urllib_request_urlopen
  30
  31 from itertools import chain
  32 from stat import S_ISDIR
  33
  34 try:
  35         import portage
  36 except ImportError:
  37         from os import path as osp
  38         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  39         import portage
  40 portage._disable_legacy_globals()
  41 portage.dep._internal_warnings = True
  42
  43 try:
  44         import xml.etree.ElementTree
  45         from xml.parsers.expat import ExpatError
  46 except ImportError:
  47         msg = ["Please enable python's \"xml\" USE flag in order to use repoman."]
  48         from portage.output import EOutput
  49         out = EOutput()
  50         for line in msg:
  51                 out.eerror(line)
  52         sys.exit(1)
  53
  54 from portage import os
  55 from portage import subprocess_getstatusoutput
  56 from portage import _encodings
  57 from portage import _unicode_encode
  58 from portage import StringIO
  59 from repoman.checks import run_checks
  60 from repoman import utilities
  61 from repoman.herdbase import make_herd_base
  62 from _emerge.Package import Package
  63 from _emerge.RootConfig import RootConfig
  64 from _emerge.userquery import userquery
  65 import portage.checksum
  66 import portage.const
  67 from portage import cvstree, normalize_path
  68 from portage import util
  69 from portage.exception import (FileNotFound, MissingParameter,
  70         ParseError, PermissionDenied)
  71 from portage.manifest import Manifest
  72 from portage.process import find_binary, spawn
  73 from portage.output import bold, create_color_func, \
  74         green, nocolor, red
  75 from portage.output import ConsoleStyleFile, StyleWriter
  76 from portage.util import cmp_sort_key, writemsg_level
  77 from portage.package.ebuild.digestgen import digestgen
  78 from portage.eapi import eapi_has_slot_deps, \
  79         eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_iuse_defaults, \
  80         eapi_has_required_use, eapi_has_use_dep_defaults
  81
  82 if sys.hexversion >= 0x3000000:
  83         basestring = str
  84
  85 util.initialize_logger()
  86
  87 # 14 is the length of DESCRIPTION=""
  88 max_desc_len = 100
  89 allowed_filename_chars="a-zA-Z0-9._-+:"
  90 disallowed_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]')
  91 pv_toolong_re = re.compile(r'[0-9]{19,}')
  92 bad = create_color_func("BAD")
  93
  94 # A sane umask is needed for files that portage creates.
  95 os.umask(0o22)
  96 # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
  97 # behave incrementally.
  98 repoman_incrementals = tuple(x for x in \
  99         portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
 100 repoman_settings = portage.config(local_config=False)
 101 repoman_settings.lock()
 102
 103 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 104         repoman_settings.get('TERM') == 'dumb' or \
 105         not sys.stdout.isatty():
 106         nocolor()
 107
 108 def warn(txt):
 109         print("repoman: " + txt)
 110
 111 def err(txt):
 112         warn(txt)
 113         sys.exit(1)
 114
 115 def exithandler(signum=None, frame=None):
 116         logging.fatal("Interrupted; exiting...")
 117         if signum is None:
 118                 sys.exit(1)
 119         else:
 120                 sys.exit(128 + signum)
 121
 122 signal.signal(signal.SIGINT,exithandler)
 123
 124 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 125         """Repoman needs it's own HelpFormatter for now, because the default ones
 126         murder the help text."""
 127
 128         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 129                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 130
 131         def format_description(self, description):
 132                 return description
 133
 134 class RepomanOptionParser(optparse.OptionParser):
 135         """Add the on_tail function, ruby has it, optionParser should too
 136         """
 137
 138         def __init__(self, *args, **kwargs):
 139                 optparse.OptionParser.__init__(self, *args, **kwargs)
 140                 self.tail = ""
 141
 142         def on_tail(self, description):
 143                 self.tail += description
 144
 145         def format_help(self, formatter=None):
 146                 result = optparse.OptionParser.format_help(self, formatter)
 147                 result += self.tail
 148                 return result
 149
 150
 151 def ParseArgs(argv, qahelp):
 152         """This function uses a customized optionParser to parse command line arguments for repoman
 153         Args:
 154           argv - a sequence of command line arguments
 155                 qahelp - a dict of qa warning to help message
 156         Returns:
 157           (opts, args), just like a call to parser.parse_args()
 158         """
 159
 160         if argv and sys.hexversion < 0x3000000 and not isinstance(argv[0], unicode):
 161                 argv = [portage._unicode_decode(x) for x in argv]
 162
 163         modes = {
 164                 'commit' : 'Run a scan then commit changes',
 165                 'ci' : 'Run a scan then commit changes',
 166                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 167                 'full' : 'Scan directory tree and print all issues (not a summary)',
 168                 'help' : 'Show this screen',
 169                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 170                 'manifest-check' : 'Check Manifests for missing or incorrect digests',
 171                 'scan' : 'Scan directory tree for QA issues'
 172         }
 173
 174         mode_keys = list(modes)
 175         mode_keys.sort()
 176
 177         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 178         parser.description = green(" ".join((os.path.basename(argv[0]), "1.2")))
 179         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 180         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 181         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 182
 183         parser.add_option('-a', '--ask', dest='ask', action='store_true', default=False,
 184                 help='Request a confirmation before commiting')
 185
 186         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 187                 help='specify a commit message on the command line')
 188
 189         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 190                 help='specify a path to a file that contains a commit message')
 191
 192         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 193                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 194
 195         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 196                 help='do not print unnecessary messages')
 197
 198         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 199                 help='Commit with QA violations')
 200
 201         parser.add_option('--vcs', dest='vcs',
 202                 help='Force using specific VCS instead of autodetection')
 203
 204         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 205                 help='be very verbose in output', default=0)
 206
 207         parser.add_option('-V', '--version', dest='version', action='store_true',
 208                 help='show version info')
 209
 210         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 211                 default=False, help='forces the metadata.xml parse check to be carried out')
 212
 213         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 214                 default=False, help='ignore arch-specific failures (where arch != host)')
 215
 216         parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
 217                 default=False, help='ignore masked packages (not allowed with commit mode)')
 218
 219         parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
 220                 default=False, help='include dev profiles in dependency checks')
 221
 222         parser.add_option('--unmatched-removal', dest='unmatched_removal', action='store_true',
 223                 default=False, help='enable strict checking of package.mask and package.unmask files for unmatched removal atoms')
 224
 225         parser.add_option('--without-mask', dest='without_mask', action='store_true',
 226                 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
 227
 228         parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
 229                 help='specify which mode repoman will run in (default=full)')
 230
 231         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 232
 233         for k in mode_keys:
 234                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 235
 236         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 237
 238         sorted_qa = list(qahelp)
 239         sorted_qa.sort()
 240         for k in sorted_qa:
 241                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 242
 243         opts, args = parser.parse_args(argv[1:])
 244
 245         if opts.mode == 'help':
 246                 parser.print_help(short=False)
 247
 248         for arg in args:
 249                 if arg in modes:
 250                         if not opts.mode:
 251                                 opts.mode = arg
 252                                 break
 253                 else:
 254                         parser.error("invalid mode: %s" % arg)
 255
 256         if not opts.mode:
 257                 opts.mode = 'full'
 258
 259         if opts.mode == 'ci':
 260                 opts.mode = 'commit'  # backwards compat shortcut
 261
 262         if opts.mode == 'commit' and not (opts.force or opts.pretend):
 263                 if opts.ignore_masked:
 264                         parser.error('Commit mode and --ignore-masked are not compatible')
 265                 if opts.without_mask:
 266                         parser.error('Commit mode and --without-mask are not compatible')
 267
 268         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 269         for val in range(opts.verbosity):
 270                 logger = logging.getLogger()
 271                 logger.setLevel(logger.getEffectiveLevel() - 10)
 272
 273         for val in range(opts.quiet):
 274                 logger = logging.getLogger()
 275                 logger.setLevel(logger.getEffectiveLevel() + 10)
 276
 277         return (opts, args)
 278
 279 qahelp={
 280         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 281         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 282         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 283         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 284         "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
 285         "changelog.missing":"Missing ChangeLog files",
 286         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 287         "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
 288         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 289         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
 290         "file.size":"Files in the files directory must be under 20 KiB",
 291         "file.size.fatal":"Files in the files directory must be under 60 KiB",
 292         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 293         "file.UTF8":"File is not UTF8 compliant",
 294         "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf",
 295         "inherit.deprecated":"Ebuild inherits a deprecated eclass",
 296         "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
 297         "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 298         "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
 299         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 300         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 301         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 302         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 303         "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable",
 304         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 305         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 306         "EAPI.definition":"EAPI is defined after an inherit call (must be defined before)",
 307         "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI",
 308         "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
 309         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 310         "SLOT.invalid":"Ebuilds that have a missing or invalid SLOT variable value",
 311         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 312         "HOMEPAGE.virtual":"Virtuals that have a non-empty HOMEPAGE variable",
 313         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 314         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 315         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 316         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 317         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 318         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 319         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 320         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 321         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 322         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 323         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 324         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 325         "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
 326         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 327         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 328         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 329         "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 330         "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 331         "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
 332         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 333         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 334         "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
 335         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 336         "REQUIRED_USE.syntax":"Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)",
 337         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 338         "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
 339         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 340         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 341         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 342         "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 343         "variable.readonly":"Assigning a readonly variable",
 344         "variable.usedwithhelpers":"Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers",
 345         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 346         "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 347         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
 348         "IUSE.missing":"This ebuild has a USE conditional which references a flag that is not listed in IUSE",
 349         "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
 350         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 351         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 352         "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)",
 353         "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
 354         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 355         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 356         "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
 357         "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
 358         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 359         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 360         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 361         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 362         "ebuild.badheader":"This ebuild has a malformed header",
 363         "eprefixify.defined":"The ebuild uses eprefixify, but does not inherit the prefix eclass",
 364         "manifest.bad":"Manifest has missing or incorrect digests",
 365         "metadata.missing":"Missing metadata.xml files",
 366         "metadata.bad":"Bad metadata.xml files",
 367         "metadata.warning":"Warnings in metadata.xml files",
 368         "portage.internal":"The ebuild uses an internal Portage function",
 369         "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)",
 370         "usage.obsolete":"The ebuild makes use of an obsolete construct",
 371         "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
 372 }
 373
 374 qacats = list(qahelp)
 375 qacats.sort()
 376
 377 qawarnings = set((
 378 "changelog.missing",
 379 "changelog.notadded",
 380 "digest.assumed",
 381 "digest.unused",
 382 "ebuild.notadded",
 383 "ebuild.nostable",
 384 "ebuild.allmasked",
 385 "ebuild.nesteddie",
 386 "desktop.invalid",
 387 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 388 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 389 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 390 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
 391 "DESCRIPTION.toolong",
 392 "EAPI.deprecated",
 393 "HOMEPAGE.virtual",
 394 "LICENSE.virtual",
 395 "KEYWORDS.dropped",
 396 "KEYWORDS.stupid",
 397 "KEYWORDS.missing",
 398 "IUSE.undefined",
 399 "PDEPEND.suspect",
 400 "RDEPEND.implicit",
 401 "RDEPEND.suspect",
 402 "RESTRICT.invalid",
 403 "SRC_URI.mirror",
 404 "ebuild.minorsyn",
 405 "ebuild.badheader",
 406 "ebuild.patches",
 407 "file.size",
 408 "inherit.autotools",
 409 "inherit.deprecated",
 410 "java.eclassesnotused",
 411 "wxwidgets.eclassnotused",
 412 "metadata.warning",
 413 "portage.internal",
 414 "usage.obsolete",
 415 "upstream.workaround",
 416 "virtual.oldstyle",
 417 "LIVEVCS.stable",
 418 "LIVEVCS.unmasked",
 419 ))
 420
 421 non_ascii_re = re.compile(r'[^\x00-\x7f]')
 422
 423 missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"]
 424 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
 425 allvars.update(Package.metadata_keys)
 426 allvars = sorted(allvars)
 427 commitmessage=None
 428 for x in missingvars:
 429         x += ".missing"
 430         if x not in qacats:
 431                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 432                 qacats.append(x)
 433                 qawarnings.add(x)
 434
 435 valid_restrict = frozenset(["binchecks", "bindist",
 436         "fetch", "installsources", "mirror",
 437         "primaryuri", "strip", "test", "userpriv"])
 438
 439 live_eclasses = frozenset([
 440         "bzr",
 441         "cvs",
 442         "darcs",
 443         "git",
 444         "git-2",
 445         "mercurial",
 446         "subversion",
 447         "tla",
 448 ])
 449
 450 suspect_rdepend = frozenset([
 451         "app-arch/cabextract",
 452         "app-arch/rpm2targz",
 453         "app-doc/doxygen",
 454         "dev-lang/nasm",
 455         "dev-lang/swig",
 456         "dev-lang/yasm",
 457         "dev-perl/extutils-pkgconfig",
 458         "dev-util/byacc",
 459         "dev-util/cmake",
 460         "dev-util/ftjam",
 461         "dev-util/gperf",
 462         "dev-util/gtk-doc",
 463         "dev-util/gtk-doc-am",
 464         "dev-util/intltool",
 465         "dev-util/jam",
 466         "dev-util/pkgconfig",
 467         "dev-util/scons",
 468         "dev-util/unifdef",
 469         "dev-util/yacc",
 470         "media-gfx/ebdftopcf",
 471         "sys-apps/help2man",
 472         "sys-devel/autoconf",
 473         "sys-devel/automake",
 474         "sys-devel/bin86",
 475         "sys-devel/bison",
 476         "sys-devel/dev86",
 477         "sys-devel/flex",
 478         "sys-devel/m4",
 479         "sys-devel/pmake",
 480         "virtual/linux-sources",
 481         "x11-misc/bdftopcf",
 482         "x11-misc/imake",
 483 ])
 484
 485 metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd'
 486 # force refetch if the local copy creation time is older than this
 487 metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days
 488
 489 # file.executable
 490 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 491
 492 options, arguments = ParseArgs(sys.argv, qahelp)
 493
 494 if options.version:
 495         print("Portage", portage.VERSION)
 496         sys.exit(0)
 497
 498 # Set this to False when an extraordinary issue (generally
 499 # something other than a QA issue) makes it impossible to
 500 # commit (like if Manifest generation fails).
 501 can_force = True
 502
 503 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 504 if portdir is None:
 505         sys.exit(1)
 506
 507 myreporoot = os.path.basename(portdir_overlay)
 508 myreporoot += mydir[len(portdir_overlay):]
 509
 510 if options.vcs:
 511         if options.vcs in ('cvs', 'svn', 'git', 'bzr', 'hg'):
 512                 vcs = options.vcs
 513         else:
 514                 vcs = None
 515 else:
 516         vcses = utilities.FindVCS()
 517         if len(vcses) > 1:
 518                 print(red('*** Ambiguous workdir -- more than one VCS found at the same depth: %s.' % ', '.join(vcses)))
 519                 print(red('*** Please either clean up your workdir or specify --vcs option.'))
 520                 sys.exit(1)
 521         elif vcses:
 522                 vcs = vcses[0]
 523         else:
 524                 vcs = None
 525
 526 # Note: We don't use ChangeLogs in distributed SCMs.
 527 # It will be generated on server side from scm log,
 528 # before package moves to the rsync server.
 529 # This is needed because we try to avoid merge collisions.
 530 check_changelog = vcs in ('cvs', 'svn')
 531
 532 # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075).
 533 vcs_preserves_mtime = vcs not in ('git',)
 534
 535 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
 536 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
 537 if vcs_global_opts is None:
 538         if vcs in ('cvs', 'svn'):
 539                 vcs_global_opts = "-q"
 540         else:
 541                 vcs_global_opts = ""
 542 vcs_global_opts = vcs_global_opts.split()
 543
 544 if vcs == "cvs" and \
 545         "commit" == options.mode and \
 546         "RMD160" not in portage.checksum.hashorigin_map:
 547         from portage.util import grablines
 548         repo_lines = grablines("./CVS/Repository")
 549         if repo_lines and \
 550                 "gentoo-x86" == repo_lines[0].strip().split(os.path.sep)[0]:
 551                 msg = "Please install " \
 552                 "pycrypto or enable python's ssl USE flag in order " \
 553                 "to enable RMD160 hash support. See bug #198398 for " \
 554                 "more information."
 555                 prefix = bad(" * ")
 556                 from textwrap import wrap
 557                 for line in wrap(msg, 70):
 558                         print(prefix + line)
 559                 sys.exit(1)
 560         del repo_lines
 561
 562 if options.mode == 'commit' and not options.pretend and not vcs:
 563         logging.info("Not in a version controlled repository; enabling pretend mode.")
 564         options.pretend = True
 565
 566 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
 567 repoman_settings = portage.config(local_config=False)
 568 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
 569         (repoman_settings.get('PORTDIR_OVERLAY', ''), portdir_overlay)
 570 # We have to call the config constructor again so
 571 # that config.repositories is initialized correctly.
 572 repoman_settings = portage.config(local_config=False, env=dict(os.environ,
 573         PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY']))
 574
 575 root = '/'
 576 trees = {
 577         root : {'porttree' : portage.portagetree(root, settings=repoman_settings)}
 578 }
 579 portdb = trees[root]['porttree'].dbapi
 580
 581 # Constrain dependency resolution to the master(s)
 582 # that are specified in layout.conf.
 583 portdir_overlay = os.path.realpath(portdir_overlay)
 584 repo_info = portdb._repo_info[portdir_overlay]
 585 portdb.porttrees = list(repo_info.eclass_db.porttrees)
 586 portdir = portdb.porttrees[0]
 587
 588 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
 589 # profile-specific config constructor calls.
 590 env = os.environ.copy()
 591 env['PORTDIR'] = portdir
 592 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
 593
 594 logging.info('Setting paths:')
 595 logging.info('PORTDIR = "' + portdir + '"')
 596 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
 597
 598 # It's confusing if these warnings are displayed without the user
 599 # being told which profile they come from, so disable them.
 600 env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
 601
 602 categories = []
 603 for path in set([portdir, portdir_overlay]):
 604         categories.extend(portage.util.grabfile(
 605                 os.path.join(path, 'profiles', 'categories')))
 606 repoman_settings.categories = tuple(sorted(
 607         portage.util.stack_lists([categories], incremental=1)))
 608 del categories
 609
 610 portdb.settings = repoman_settings
 611 root_config = RootConfig(repoman_settings, trees[root], None)
 612 # We really only need to cache the metadata that's necessary for visibility
 613 # filtering. Anything else can be discarded to reduce memory consumption.
 614 portdb._aux_cache_keys.clear()
 615 portdb._aux_cache_keys.update(["EAPI", "KEYWORDS", "SLOT"])
 616
 617 reposplit = myreporoot.split(os.path.sep)
 618 repolevel = len(reposplit)
 619
 620 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 621 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 622 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 623 if options.mode == 'commit' and repolevel not in [1,2,3]:
 624         print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
 625         print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
 626         print(red("***")+" This is intended behaviour, to ensure the manifest is recommitted for a package.")
 627         print(red("***"))
 628         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 629
 630 startdir = normalize_path(mydir)
 631 repodir = startdir
 632 for x in range(0, repolevel - 1):
 633         repodir = os.path.dirname(repodir)
 634 repodir = os.path.realpath(repodir)
 635
 636 def caterror(mycat):
 637         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 638
 639 class ProfileDesc(object):
 640         __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
 641         def __init__(self, arch, status, sub_path, tree_path):
 642                 self.arch = arch
 643                 self.status = status
 644                 if sub_path:
 645                         sub_path = normalize_path(sub_path.lstrip(os.sep))
 646                 self.sub_path = sub_path
 647                 self.tree_path = tree_path
 648                 if tree_path:
 649                         self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
 650                 else:
 651                         self.abs_path = tree_path
 652
 653         def __str__(self):
 654                 if self.sub_path:
 655                         return self.sub_path
 656                 return 'empty profile'
 657
 658 profile_list = []
 659 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
 660
 661 # get lists of valid keywords, licenses, and use
 662 kwlist = set()
 663 liclist = set()
 664 uselist = set()
 665 global_pmasklines = []
 666
 667 for path in portdb.porttrees:
 668         try:
 669                 liclist.update(os.listdir(os.path.join(path, "licenses")))
 670         except OSError:
 671                 pass
 672         kwlist.update(portage.grabfile(os.path.join(path,
 673                 "profiles", "arch.list")))
 674
 675         use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
 676         for x in use_desc:
 677                 x = x.split()
 678                 if x:
 679                         uselist.add(x[0])
 680
 681         expand_desc_dir = os.path.join(path, 'profiles', 'desc')
 682         try:
 683                 expand_list = os.listdir(expand_desc_dir)
 684         except OSError:
 685                 pass
 686         else:
 687                 for fn in expand_list:
 688                         if not fn[-5:] == '.desc':
 689                                 continue
 690                         use_prefix = fn[:-5].lower() + '_'
 691                         for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
 692                                 x = x.split()
 693                                 if x:
 694                                         uselist.add(use_prefix + x[0])
 695
 696         global_pmasklines.append(portage.util.grabfile_package(
 697                 os.path.join(path, 'profiles', 'package.mask'), recursive=1, verify_eapi=True))
 698
 699         desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 700         try:
 701                 desc_file = codecs.open(_unicode_encode(desc_path,
 702                         encoding=_encodings['fs'], errors='strict'),
 703                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 704         except EnvironmentError:
 705                 pass
 706         else:
 707                 for i, x in enumerate(desc_file):
 708                         if x[0] == "#":
 709                                 continue
 710                         arch = x.split()
 711                         if len(arch) == 0:
 712                                 continue
 713                         if len(arch) != 3:
 714                                 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
 715                                         desc_path + " line %d" % (i+1, ))
 716                         elif arch[0] not in kwlist:
 717                                 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
 718                                         desc_path + " line %d" % (i+1, ))
 719                         elif arch[2] not in valid_profile_types:
 720                                 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
 721                                         desc_path + " line %d" % (i+1, ))
 722                         profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path)
 723                         if not os.path.isdir(profile_desc.abs_path):
 724                                 logging.error(
 725                                         "Invalid %s profile (%s) for arch %s in %s line %d",
 726                                         arch[2], arch[1], arch[0], desc_path, i+1)
 727                                 continue
 728                         if os.path.exists(
 729                                 os.path.join(profile_desc.abs_path, 'deprecated')):
 730                                 continue
 731                         profile_list.append(profile_desc)
 732                 desc_file.close()
 733
 734 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 735 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 736
 737 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
 738 global_pmaskdict = {}
 739 for x in global_pmasklines:
 740         global_pmaskdict.setdefault(x.cp, []).append(x)
 741 del global_pmasklines
 742
 743 def has_global_mask(pkg):
 744         mask_atoms = global_pmaskdict.get(pkg.cp)
 745         if mask_atoms:
 746                 pkg_list = [pkg]
 747                 for x in mask_atoms:
 748                         if portage.dep.match_from_list(x, pkg_list):
 749                                 return x
 750         return None
 751
 752 # Ensure that profile sub_path attributes are unique. Process in reverse order
 753 # so that profiles with duplicate sub_path from overlays will override
 754 # profiles with the same sub_path from parent repos.
 755 profiles = {}
 756 profile_list.reverse()
 757 profile_sub_paths = set()
 758 for prof in profile_list:
 759         if prof.sub_path in profile_sub_paths:
 760                 continue
 761         profile_sub_paths.add(prof.sub_path)
 762         profiles.setdefault(prof.arch, []).append(prof)
 763
 764 # Use an empty profile for checking dependencies of
 765 # packages that have empty KEYWORDS.
 766 prof = ProfileDesc('**', 'stable', '', '')
 767 profiles.setdefault(prof.arch, []).append(prof)
 768
 769 for x in repoman_settings.archlist():
 770         if x[0] == "~":
 771                 continue
 772         if x not in profiles:
 773                 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
 774                 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
 775                 print(red("up with the "+x+" team."))
 776                 print()
 777
 778 if not liclist:
 779         logging.fatal("Couldn't find licenses?")
 780         sys.exit(1)
 781
 782 if not kwlist:
 783         logging.fatal("Couldn't read KEYWORDS from arch.list")
 784         sys.exit(1)
 785
 786 if not uselist:
 787         logging.fatal("Couldn't find use.desc?")
 788         sys.exit(1)
 789
 790 scanlist=[]
 791 if repolevel==2:
 792         #we are inside a category directory
 793         catdir=reposplit[-1]
 794         if catdir not in repoman_settings.categories:
 795                 caterror(catdir)
 796         mydirlist=os.listdir(startdir)
 797         for x in mydirlist:
 798                 if x == "CVS" or x.startswith("."):
 799                         continue
 800                 if os.path.isdir(startdir+"/"+x):
 801                         scanlist.append(catdir+"/"+x)
 802         repo_subdir = catdir + os.sep
 803 elif repolevel==1:
 804         for x in repoman_settings.categories:
 805                 if not os.path.isdir(startdir+"/"+x):
 806                         continue
 807                 for y in os.listdir(startdir+"/"+x):
 808                         if y == "CVS" or y.startswith("."):
 809                                 continue
 810                         if os.path.isdir(startdir+"/"+x+"/"+y):
 811                                 scanlist.append(x+"/"+y)
 812         repo_subdir = ""
 813 elif repolevel==3:
 814         catdir = reposplit[-2]
 815         if catdir not in repoman_settings.categories:
 816                 caterror(catdir)
 817         scanlist.append(catdir+"/"+reposplit[-1])
 818         repo_subdir = scanlist[-1] + os.sep
 819 else:
 820         msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
 821                 ' from the current working directory'
 822         logging.critical(msg)
 823         sys.exit(1)
 824
 825 repo_subdir_len = len(repo_subdir)
 826 scanlist.sort()
 827
 828 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 829
 830 def dev_keywords(profiles):
 831         """
 832         Create a set of KEYWORDS values that exist in 'dev'
 833         profiles. These are used
 834         to trigger a message notifying the user when they might
 835         want to add the --include-dev option.
 836         """
 837         type_arch_map = {}
 838         for arch, arch_profiles in profiles.items():
 839                 for prof in arch_profiles:
 840                         arch_set = type_arch_map.get(prof.status)
 841                         if arch_set is None:
 842                                 arch_set = set()
 843                                 type_arch_map[prof.status] = arch_set
 844                         arch_set.add(arch)
 845
 846         dev_keywords = type_arch_map.get('dev', set())
 847         dev_keywords.update(['~' + arch for arch in dev_keywords])
 848         return frozenset(dev_keywords)
 849
 850 dev_keywords = dev_keywords(profiles)
 851
 852 stats={}
 853 fails={}
 854
 855 # provided by the desktop-file-utils package
 856 desktop_file_validate = find_binary("desktop-file-validate")
 857 desktop_pattern = re.compile(r'.*\.desktop$')
 858
 859 for x in qacats:
 860         stats[x]=0
 861         fails[x]=[]
 862
 863 xmllint_capable = False
 864 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
 865
 866 def parsedate(s):
 867         """Parse a RFC 822 date and time string.
 868         This is required for python3 compatibility, since the
 869         rfc822.parsedate() function is not available."""
 870
 871         s_split = []
 872         for x in s.upper().split():
 873                 for y in x.split(','):
 874                         if y:
 875                                 s_split.append(y)
 876
 877         if len(s_split) != 6:
 878                 return None
 879
 880         # %a, %d %b %Y %H:%M:%S %Z
 881         a, d, b, Y, H_M_S, Z = s_split
 882
 883         # Convert month to integer, since strptime %w is locale-dependent.
 884         month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
 885                 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
 886         m = month_map.get(b)
 887         if m is None:
 888                 return None
 889         m = str(m).rjust(2, '0')
 890
 891         return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
 892
 893 def fetch_metadata_dtd():
 894         """
 895         Fetch metadata.dtd if it doesn't exist or the ctime is older than
 896         metadata_dtd_ctime_interval.
 897         @rtype: bool
 898         @returns: True if successful, otherwise False
 899         """
 900
 901         must_fetch = True
 902         metadata_dtd_st = None
 903         current_time = int(time.time())
 904         try:
 905                 metadata_dtd_st = os.stat(metadata_dtd)
 906         except EnvironmentError as e:
 907                 if e.errno not in (errno.ENOENT, errno.ESTALE):
 908                         raise
 909                 del e
 910         else:
 911                 # Trigger fetch if metadata.dtd mtime is old or clock is wrong.
 912                 if abs(current_time - metadata_dtd_st.st_ctime) \
 913                         < metadata_dtd_ctime_interval:
 914                         must_fetch = False
 915
 916         if must_fetch:
 917                 print()
 918                 print(green("***") + " the local copy of metadata.dtd " + \
 919                         "needs to be refetched, doing that now")
 920                 print()
 921                 try:
 922                         url_f = urllib_request_urlopen(metadata_dtd_uri)
 923                         msg_info = url_f.info()
 924                         last_modified = msg_info.get('last-modified')
 925                         if last_modified is not None:
 926                                 last_modified = parsedate(last_modified)
 927                                 if last_modified is not None:
 928                                         last_modified = calendar.timegm(last_modified)
 929
 930                         metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
 931                         try:
 932                                 local_f = open(metadata_dtd_tmp, mode='wb')
 933                                 local_f.write(url_f.read())
 934                                 local_f.close()
 935                                 if last_modified is not None:
 936                                         try:
 937                                                 os.utime(metadata_dtd_tmp,
 938                                                         (int(last_modified), int(last_modified)))
 939                                         except OSError:
 940                                                 # This fails on some odd non-unix-like filesystems.
 941                                                 # We don't really need the mtime to be preserved
 942                                                 # anyway here (currently we use ctime to trigger
 943                                                 # fetch), so just ignore it.
 944                                                 pass
 945                                 os.rename(metadata_dtd_tmp, metadata_dtd)
 946                         finally:
 947                                 try:
 948                                         os.unlink(metadata_dtd_tmp)
 949                                 except OSError:
 950                                         pass
 951
 952                         url_f.close()
 953
 954                 except EnvironmentError as e:
 955                         print()
 956                         print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_uri)
 957                         print(red("!!!")+" exception '%s' though." % (e,))
 958                         print(red("!!!")+" fetching new metadata.dtd failed, aborting")
 959                         return False
 960
 961         return True
 962
 963 if options.mode == "manifest":
 964         pass
 965 elif not find_binary('xmllint'):
 966         print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
 967         if options.xml_parse or repolevel==3:
 968                 print(red("!!!")+" sorry, xmllint is needed.  failing\n")
 969                 sys.exit(1)
 970 else:
 971         if not fetch_metadata_dtd():
 972                 sys.exit(1)
 973         #this can be problematic if xmllint changes their output
 974         xmllint_capable=True
 975
 976 if options.mode == 'commit' and vcs:
 977         utilities.detect_vcs_conflicts(options, vcs)
 978
 979 if options.mode == "manifest":
 980         pass
 981 elif options.pretend:
 982         print(green("\nRepoMan does a once-over of the neighborhood..."))
 983 else:
 984         print(green("\nRepoMan scours the neighborhood..."))
 985
 986 new_ebuilds = set()
 987 modified_ebuilds = set()
 988 modified_changelogs = set()
 989 mychanged = []
 990 mynew = []
 991 myremoved = []
 992
 993 if vcs == "cvs":
 994         mycvstree = cvstree.getentries("./", recursive=1)
 995         mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
 996         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
 997 if vcs == "svn":
 998         svnstatus = os.popen("svn status").readlines()
 999         mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1000         mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1001 elif vcs == "git":
1002         mychanged = os.popen("git diff-index --name-only --relative --diff-filter=M HEAD").readlines()
1003         mychanged = ["./" + elem[:-1] for elem in mychanged]
1004
1005         mynew = os.popen("git diff-index --name-only --relative --diff-filter=A HEAD").readlines()
1006         mynew = ["./" + elem[:-1] for elem in mynew]
1007 elif vcs == "bzr":
1008         bzrstatus = os.popen("bzr status -S .").readlines()
1009         mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
1010         mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ]
1011 elif vcs == "hg":
1012         mychanged = os.popen("hg status --no-status --modified .").readlines()
1013         mychanged = ["./" + elem.rstrip() for elem in mychanged]
1014         mynew = os.popen("hg status --no-status --added .").readlines()
1015         mynew = ["./" + elem.rstrip() for elem in mynew]
1016
1017 if vcs:
1018         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
1019         modified_ebuilds.update(x for x in mychanged if x.endswith(".ebuild"))
1020         modified_changelogs.update(x for x in chain(mychanged, mynew) \
1021                 if os.path.basename(x) == "ChangeLog")
1022
1023 have_pmasked = False
1024 have_dev_keywords = False
1025 dofail = 0
1026 arch_caches={}
1027 arch_xmatch_caches = {}
1028 shared_xmatch_caches = {"cp-list":{}}
1029
1030 # Disable the "ebuild.notadded" check when not in commit mode and
1031 # running `svn status` in every package dir will be too expensive.
1032
1033 check_ebuild_notadded = not \
1034         (vcs == "svn" and repolevel < 3 and options.mode != "commit")
1035
1036 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
1037 thirdpartymirrors = []
1038 for v in repoman_settings.thirdpartymirrors().values():
1039         thirdpartymirrors.extend(v)
1040
1041 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
1042         """
1043         Implements doctype() as required to avoid deprecation warnings with
1044         >=python-2.7.
1045         """
1046         def doctype(self, name, pubid, system):
1047                 pass
1048
1049 try:
1050         herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
1051 except (EnvironmentError, ParseError, PermissionDenied) as e:
1052         err(str(e))
1053 except FileNotFound:
1054         # TODO: Download as we do for metadata.dtd, but add a way to
1055         # disable for non-gentoo repoman users who may not have herds.
1056         herd_base = None
1057
1058 for x in scanlist:
1059         #ebuilds and digests added to cvs respectively.
1060         logging.info("checking package %s" % x)
1061         eadded=[]
1062         catdir,pkgdir=x.split("/")
1063         checkdir=repodir+"/"+x
1064         checkdir_relative = ""
1065         if repolevel < 3:
1066                 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
1067         if repolevel < 2:
1068                 checkdir_relative = os.path.join(catdir, checkdir_relative)
1069         checkdir_relative = os.path.join(".", checkdir_relative)
1070         generated_manifest = False
1071
1072         if options.mode == "manifest" or \
1073           (options.mode != 'manifest-check' and \
1074           'digest' in repoman_settings.features) or \
1075           options.mode in ('commit', 'fix') and not options.pretend:
1076                 auto_assumed = set()
1077                 fetchlist_dict = portage.FetchlistDict(checkdir,
1078                         repoman_settings, portdb)
1079                 if options.mode == 'manifest' and options.force:
1080                         portage._doebuild_manifest_exempt_depend += 1
1081                         try:
1082                                 distdir = repoman_settings['DISTDIR']
1083                                 mf = portage.manifest.Manifest(checkdir, distdir,
1084                                         fetchlist_dict=fetchlist_dict)
1085                                 mf.create(requiredDistfiles=None,
1086                                         assumeDistHashesAlways=True)
1087                                 for distfiles in fetchlist_dict.values():
1088                                         for distfile in distfiles:
1089                                                 if os.path.isfile(os.path.join(distdir, distfile)):
1090                                                         mf.fhashdict['DIST'].pop(distfile, None)
1091                                                 else:
1092                                                         auto_assumed.add(distfile)
1093                                 mf.write()
1094                         finally:
1095                                 portage._doebuild_manifest_exempt_depend -= 1
1096
1097                 repoman_settings["O"] = checkdir
1098                 generated_manifest = digestgen(
1099                         mysettings=repoman_settings, myportdb=portdb)
1100
1101                 if not generated_manifest:
1102                         print("Unable to generate manifest.")
1103                         dofail = 1
1104
1105                 if options.mode == "manifest":
1106                         if not dofail and options.force and auto_assumed and \
1107                                 'assume-digests' in repoman_settings.features:
1108                                 # Show which digests were assumed despite the --force option
1109                                 # being given. This output will already have been shown by
1110                                 # digestgen() if assume-digests is not enabled, so only show
1111                                 # it here if assume-digests is enabled.
1112                                 pkgs = list(fetchlist_dict)
1113                                 pkgs.sort()
1114                                 portage.writemsg_stdout("  digest.assumed" + \
1115                                         portage.output.colorize("WARN",
1116                                         str(len(auto_assumed)).rjust(18)) + "\n")
1117                                 for cpv in pkgs:
1118                                         fetchmap = fetchlist_dict[cpv]
1119                                         pf = portage.catsplit(cpv)[1]
1120                                         for distfile in sorted(fetchmap):
1121                                                 if distfile in auto_assumed:
1122                                                         portage.writemsg_stdout(
1123                                                                 "   %s::%s\n" % (pf, distfile))
1124                         continue
1125                 elif dofail:
1126                         sys.exit(1)
1127
1128         if not generated_manifest:
1129                 repoman_settings['O'] = checkdir
1130                 repoman_settings['PORTAGE_QUIET'] = '1'
1131                 if not portage.digestcheck([], repoman_settings, strict=1):
1132                         stats["manifest.bad"] += 1
1133                         fails["manifest.bad"].append(os.path.join(x, 'Manifest'))
1134                 repoman_settings.pop('PORTAGE_QUIET', None)
1135
1136         if options.mode == 'manifest-check':
1137                 continue
1138
1139         checkdirlist=os.listdir(checkdir)
1140         ebuildlist=[]
1141         pkgs = {}
1142         allvalid = True
1143         for y in checkdirlist:
1144                 if (y in no_exec or y.endswith(".ebuild")) and \
1145                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
1146                                 stats["file.executable"] += 1
1147                                 fails["file.executable"].append(os.path.join(checkdir, y))
1148                 if y.endswith(".ebuild"):
1149                         pf = y[:-7]
1150                         ebuildlist.append(pf)
1151                         cpv = "%s/%s" % (catdir, pf)
1152                         try:
1153                                 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
1154                         except KeyError:
1155                                 allvalid = False
1156                                 stats["ebuild.syntax"] += 1
1157                                 fails["ebuild.syntax"].append(os.path.join(x, y))
1158                                 continue
1159                         except IOError:
1160                                 allvalid = False
1161                                 stats["ebuild.output"] += 1
1162                                 fails["ebuild.output"].append(os.path.join(x, y))
1163                                 continue
1164                         if not portage.eapi_is_supported(myaux["EAPI"]):
1165                                 allvalid = False
1166                                 stats["EAPI.unsupported"] += 1
1167                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
1168                                 continue
1169                         pkgs[pf] = Package(cpv=cpv, metadata=myaux,
1170                                 root_config=root_config, type_name="ebuild")
1171
1172         # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
1173         pkgsplits = {}
1174         for i in range(len(ebuildlist)):
1175                 ebuild_split = portage.pkgsplit(ebuildlist[i])
1176                 pkgsplits[ebuild_split] = ebuildlist[i]
1177                 ebuildlist[i] = ebuild_split
1178         ebuildlist.sort(key=cmp_sort_key(portage.pkgcmp))
1179         for i in range(len(ebuildlist)):
1180                 ebuildlist[i] = pkgsplits[ebuildlist[i]]
1181         del pkgsplits
1182
1183         slot_keywords = {}
1184
1185         if len(pkgs) != len(ebuildlist):
1186                 # If we can't access all the metadata then it's totally unsafe to
1187                 # commit since there's no way to generate a correct Manifest.
1188                 # Do not try to do any more QA checks on this package since missing
1189                 # metadata leads to false positives for several checks, and false
1190                 # positives confuse users.
1191                 can_force = False
1192                 continue
1193
1194         for y in checkdirlist:
1195                 m = disallowed_filename_chars_re.search(y.strip(os.sep))
1196                 if m is not None:
1197                         stats["file.name"] += 1
1198                         fails["file.name"].append("%s/%s: char '%s'" % \
1199                                 (checkdir, y, m.group(0)))
1200
1201                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1202                         continue
1203                 try:
1204                         line = 1
1205                         for l in codecs.open(_unicode_encode(os.path.join(checkdir, y),
1206                                 encoding=_encodings['fs'], errors='strict'),
1207                                 mode='r', encoding=_encodings['repo.content']):
1208                                 line +=1
1209                 except UnicodeDecodeError as ue:
1210                         stats["file.UTF8"] += 1
1211                         s = ue.object[:ue.start]
1212                         l2 = s.count("\n")
1213                         line += l2
1214                         if l2 != 0:
1215                                 s = s[s.rfind("\n") + 1:]
1216                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1217
1218         if vcs in ("git", "hg") and check_ebuild_notadded:
1219                 if vcs == "git":
1220                         myf = os.popen("git ls-files --others %s" % \
1221                                 (portage._shell_quote(checkdir_relative),))
1222                 if vcs == "hg":
1223                         myf = os.popen("hg status --no-status --unknown %s" % \
1224                                 (portage._shell_quote(checkdir_relative),))
1225                 for l in myf:
1226                         if l[:-1][-7:] == ".ebuild":
1227                                 stats["ebuild.notadded"] += 1
1228                                 fails["ebuild.notadded"].append(
1229                                         os.path.join(x, os.path.basename(l[:-1])))
1230                 myf.close()
1231
1232         if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded:
1233                 try:
1234                         if vcs == "cvs":
1235                                 myf=open(checkdir+"/CVS/Entries","r")
1236                         if vcs == "svn":
1237                                 myf = os.popen("svn status --depth=files --verbose " + checkdir)
1238                         if vcs == "bzr":
1239                                 myf = os.popen("bzr ls -v --kind=file " + checkdir)
1240                         myl = myf.readlines()
1241                         myf.close()
1242                         for l in myl:
1243                                 if vcs == "cvs":
1244                                         if l[0]!="/":
1245                                                 continue
1246                                         splitl=l[1:].split("/")
1247                                         if not len(splitl):
1248                                                 continue
1249                                         if splitl[0][-7:]==".ebuild":
1250                                                 eadded.append(splitl[0][:-7])
1251                                 if vcs == "svn":
1252                                         if l[:1] == "?":
1253                                                 continue
1254                                         if l[:7] == '      >':
1255                                                 # tree conflict, new in subversion 1.6
1256                                                 continue
1257                                         l = l.split()[-1]
1258                                         if l[-7:] == ".ebuild":
1259                                                 eadded.append(os.path.basename(l[:-7]))
1260                                 if vcs == "bzr":
1261                                         if l[1:2] == "?":
1262                                                 continue
1263                                         l = l.split()[-1]
1264                                         if l[-7:] == ".ebuild":
1265                                                 eadded.append(os.path.basename(l[:-7]))
1266                         if vcs == "svn":
1267                                 myf = os.popen("svn status " + checkdir)
1268                                 myl=myf.readlines()
1269                                 myf.close()
1270                                 for l in myl:
1271                                         if l[0] == "A":
1272                                                 l = l.rstrip().split(' ')[-1]
1273                                                 if l[-7:] == ".ebuild":
1274                                                         eadded.append(os.path.basename(l[:-7]))
1275                 except IOError:
1276                         if vcs == "cvs":
1277                                 stats["CVS/Entries.IO_error"] += 1
1278                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1279                         else:
1280                                 raise
1281                         continue
1282
1283         mf = Manifest(checkdir, repoman_settings["DISTDIR"])
1284         mydigests=mf.getTypeDigests("DIST")
1285
1286         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1287         myfiles_all = []
1288         src_uri_error = False
1289         for mykey in fetchlist_dict:
1290                 try:
1291                         myfiles_all.extend(fetchlist_dict[mykey])
1292                 except portage.exception.InvalidDependString as e:
1293                         src_uri_error = True
1294                         try:
1295                                 portdb.aux_get(mykey, ["SRC_URI"])
1296                         except KeyError:
1297                                 # This will be reported as an "ebuild.syntax" error.
1298                                 pass
1299                         else:
1300                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1301                                 fails["SRC_URI.syntax"].append(
1302                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
1303         del fetchlist_dict
1304         if not src_uri_error:
1305                 # This test can produce false positives if SRC_URI could not
1306                 # be parsed for one or more ebuilds. There's no point in
1307                 # producing a false error here since the root cause will
1308                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1309                 # or "ebuild.sytax".
1310                 myfiles_all = set(myfiles_all)
1311                 for entry in mydigests:
1312                         if entry not in myfiles_all:
1313                                 stats["digest.unused"] += 1
1314                                 fails["digest.unused"].append(checkdir+"::"+entry)
1315                 for entry in myfiles_all:
1316                         if entry not in mydigests:
1317                                 stats["digest.missing"] += 1
1318                                 fails["digest.missing"].append(checkdir+"::"+entry)
1319         del myfiles_all
1320
1321         if os.path.exists(checkdir+"/files"):
1322                 filesdirlist=os.listdir(checkdir+"/files")
1323
1324                 # recurse through files directory
1325                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1326                 while filesdirlist:
1327                         y = filesdirlist.pop(0)
1328                         relative_path = os.path.join(x, "files", y)
1329                         full_path = os.path.join(repodir, relative_path)
1330                         try:
1331                                 mystat = os.stat(full_path)
1332                         except OSError as oe:
1333                                 if oe.errno == 2:
1334                                         # don't worry about it.  it likely was removed via fix above.
1335                                         continue
1336                                 else:
1337                                         raise oe
1338                         if S_ISDIR(mystat.st_mode):
1339                                 # !!! VCS "portability" alert!  Need some function isVcsDir() or alike !!!
1340                                 if y == "CVS" or y == ".svn":
1341                                         continue
1342                                 for z in os.listdir(checkdir+"/files/"+y):
1343                                         if z == "CVS" or z == ".svn":
1344                                                 continue
1345                                         filesdirlist.append(y+"/"+z)
1346                         # Current policy is no files over 20 KiB, these are the checks. File size between
1347                         # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1348                         elif mystat.st_size > 61440:
1349                                 stats["file.size.fatal"] += 1
1350                                 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1351                         elif mystat.st_size > 20480:
1352                                 stats["file.size"] += 1
1353                                 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1354
1355                         m = disallowed_filename_chars_re.search(
1356                                 os.path.basename(y.rstrip(os.sep)))
1357                         if m is not None:
1358                                 stats["file.name"] += 1
1359                                 fails["file.name"].append("%s/files/%s: char '%s'" % \
1360                                         (checkdir, y, m.group(0)))
1361
1362                         if desktop_file_validate and desktop_pattern.match(y):
1363                                 status, cmd_output = subprocess_getstatusoutput(
1364                                         "'%s' '%s'" % (desktop_file_validate, full_path))
1365                                 if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK:
1366                                         # Note: in the future we may want to grab the
1367                                         # warnings in addition to the errors. We're
1368                                         # just doing errors now since we don't want
1369                                         # to generate too much noise at first.
1370                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
1371                                         for line in cmd_output.splitlines():
1372                                                 error_match = error_re.match(line)
1373                                                 if error_match is None:
1374                                                         continue
1375                                                 stats["desktop.invalid"] += 1
1376                                                 fails["desktop.invalid"].append(
1377                                                         relative_path + ': %s' % error_match.group(1))
1378
1379         del mydigests
1380
1381         if check_changelog and "ChangeLog" not in checkdirlist:
1382                 stats["changelog.missing"]+=1
1383                 fails["changelog.missing"].append(x+"/ChangeLog")
1384
1385         musedict = {}
1386         #metadata.xml file check
1387         if "metadata.xml" not in checkdirlist:
1388                 stats["metadata.missing"]+=1
1389                 fails["metadata.missing"].append(x+"/metadata.xml")
1390         #metadata.xml parse check
1391         else:
1392                 metadata_bad = False
1393
1394                 # read metadata.xml into memory
1395                 try:
1396                         _metadata_xml = xml.etree.ElementTree.parse(
1397                                 os.path.join(checkdir, "metadata.xml"),
1398                                 parser=xml.etree.ElementTree.XMLParser(
1399                                         target=_MetadataTreeBuilder()))
1400                 except (xml.etree.ElementTree.ParseError, ExpatError, EnvironmentError) as e:
1401                         metadata_bad = True
1402                         stats["metadata.bad"] += 1
1403                         fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1404                         del e
1405                 else:
1406                         # load USE flags from metadata.xml
1407                         try:
1408                                 musedict = utilities.parse_metadata_use(_metadata_xml)
1409                         except portage.exception.ParseError as e:
1410                                 metadata_bad = True
1411                                 stats["metadata.bad"] += 1
1412                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1413
1414                         # Run other metadata.xml checkers
1415                         try:
1416                                 utilities.check_metadata(_metadata_xml, herd_base)
1417                         except (utilities.UnknownHerdsError, ) as e:
1418                                 metadata_bad = True
1419                                 stats["metadata.bad"] += 1
1420                                 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1421                                 del e
1422
1423                 #Only carry out if in package directory or check forced
1424                 if xmllint_capable and not metadata_bad:
1425                         # xmlint can produce garbage output even on success, so only dump
1426                         # the ouput when it fails.
1427                         st, out = subprocess_getstatusoutput(
1428                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1429                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1430                         if st != os.EX_OK:
1431                                 print(red("!!!") + " metadata.xml is invalid:")
1432                                 for z in out.splitlines():
1433                                         print(red("!!! ")+z)
1434                                 stats["metadata.bad"]+=1
1435                                 fails["metadata.bad"].append(x+"/metadata.xml")
1436
1437                 del metadata_bad
1438         muselist = frozenset(musedict)
1439
1440         changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1441         changelog_modified = changelog_path in modified_changelogs
1442
1443         allmasked = True
1444         # detect unused local USE-descriptions
1445         used_useflags = set()
1446
1447         for y in ebuildlist:
1448                 relative_path = os.path.join(x, y + ".ebuild")
1449                 full_path = os.path.join(repodir, relative_path)
1450                 ebuild_path = y + ".ebuild"
1451                 if repolevel < 3:
1452                         ebuild_path = os.path.join(pkgdir, ebuild_path)
1453                 if repolevel < 2:
1454                         ebuild_path = os.path.join(catdir, ebuild_path)
1455                 ebuild_path = os.path.join(".", ebuild_path)
1456                 if check_changelog and not changelog_modified \
1457                         and ebuild_path in new_ebuilds:
1458                         stats['changelog.ebuildadded'] += 1
1459                         fails['changelog.ebuildadded'].append(relative_path)
1460
1461                 if vcs in ("cvs", "svn", "bzr") and check_ebuild_notadded and y not in eadded:
1462                         #ebuild not added to vcs
1463                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1464                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1465                 myesplit=portage.pkgsplit(y)
1466                 if myesplit is None or myesplit[0] != x.split("/")[-1] \
1467                             or pv_toolong_re.search(myesplit[1]) \
1468                             or pv_toolong_re.search(myesplit[2]):
1469                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1470                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1471                         continue
1472                 elif myesplit[0]!=pkgdir:
1473                         print(pkgdir,myesplit[0])
1474                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1475                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1476                         continue
1477
1478                 pkg = pkgs[y]
1479
1480                 if pkg.invalid:
1481                         allvalid = False
1482                         for k, msgs in pkg.invalid.items():
1483                                 for msg in msgs:
1484                                         stats[k] = stats[k] + 1
1485                                         fails[k].append("%s %s" % (relative_path, msg))
1486                         continue
1487
1488                 myaux = pkg.metadata
1489                 eapi = myaux["EAPI"]
1490                 inherited = pkg.inherited
1491                 live_ebuild = live_eclasses.intersection(inherited)
1492
1493                 for k, v in myaux.items():
1494                         if not isinstance(v, basestring):
1495                                 continue
1496                         m = non_ascii_re.search(v)
1497                         if m is not None:
1498                                 stats["variable.invalidchar"] += 1
1499                                 fails["variable.invalidchar"].append(
1500                                         ("%s: %s variable contains non-ASCII " + \
1501                                         "character at position %s") % \
1502                                         (relative_path, k, m.start() + 1))
1503
1504                 if not src_uri_error:
1505                         # Check that URIs don't reference a server from thirdpartymirrors.
1506                         for uri in portage.dep.use_reduce( \
1507                                 myaux["SRC_URI"], matchall=True, is_src_uri=True, eapi=eapi, flat=True):
1508                                 contains_mirror = False
1509                                 for mirror in thirdpartymirrors:
1510                                         if uri.startswith(mirror):
1511                                                 contains_mirror = True
1512                                                 break
1513                                 if not contains_mirror:
1514                                         continue
1515
1516                                 stats["SRC_URI.mirror"] += 1
1517                                 fails["SRC_URI.mirror"].append(
1518                                         "%s: '%s' found in thirdpartymirrors" % \
1519                                         (relative_path, mirror))
1520
1521                 if myaux.get("PROVIDE"):
1522                         stats["virtual.oldstyle"]+=1
1523                         fails["virtual.oldstyle"].append(relative_path)
1524
1525                 for pos, missing_var in enumerate(missingvars):
1526                         if not myaux.get(missing_var):
1527                                 if catdir == "virtual" and \
1528                                         missing_var in ("HOMEPAGE", "LICENSE"):
1529                                         continue
1530                                 if live_ebuild and missing_var == "KEYWORDS":
1531                                         continue
1532                                 myqakey=missingvars[pos]+".missing"
1533                                 stats[myqakey]=stats[myqakey]+1
1534                                 fails[myqakey].append(x+"/"+y+".ebuild")
1535
1536                 if catdir == "virtual":
1537                         for var in ("HOMEPAGE", "LICENSE"):
1538                                 if myaux.get(var):
1539                                         myqakey = var + ".virtual"
1540                                         stats[myqakey] = stats[myqakey] + 1
1541                                         fails[myqakey].append(relative_path)
1542
1543                 # 14 is the length of DESCRIPTION=""
1544                 if len(myaux['DESCRIPTION']) > max_desc_len:
1545                         stats['DESCRIPTION.toolong'] += 1
1546                         fails['DESCRIPTION.toolong'].append(
1547                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1548                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1549
1550                 keywords = myaux["KEYWORDS"].split()
1551                 stable_keywords = []
1552                 for keyword in keywords:
1553                         if not keyword.startswith("~") and \
1554                                 not keyword.startswith("-"):
1555                                 stable_keywords.append(keyword)
1556                 if stable_keywords:
1557                         if ebuild_path in new_ebuilds:
1558                                 stable_keywords.sort()
1559                                 stats["KEYWORDS.stable"] += 1
1560                                 fails["KEYWORDS.stable"].append(
1561                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1562                                                 " ".join(stable_keywords))
1563
1564                 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1565                         if not kw.startswith("-"))
1566
1567                 previous_keywords = slot_keywords.get(myaux["SLOT"])
1568                 if previous_keywords is None:
1569                         slot_keywords[myaux["SLOT"]] = set()
1570                 elif ebuild_archs and not live_ebuild:
1571                         dropped_keywords = previous_keywords.difference(ebuild_archs)
1572                         if dropped_keywords:
1573                                 stats["KEYWORDS.dropped"] += 1
1574                                 fails["KEYWORDS.dropped"].append(
1575                                         relative_path + ": %s" % \
1576                                         " ".join(sorted(dropped_keywords)))
1577
1578                 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1579
1580                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1581                 if "-*" in keywords:
1582                         haskeyword = False
1583                         for kw in keywords:
1584                                 if kw[0] == "~":
1585                                         kw = kw[1:]
1586                                 if kw in kwlist:
1587                                         haskeyword = True
1588                         if not haskeyword:
1589                                 stats["KEYWORDS.stupid"] += 1
1590                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1591
1592                 """
1593                 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1594                 not be allowed to be marked stable
1595                 """
1596                 if live_ebuild:
1597                         bad_stable_keywords = []
1598                         for keyword in keywords:
1599                                 if not keyword.startswith("~") and \
1600                                         not keyword.startswith("-"):
1601                                         bad_stable_keywords.append(keyword)
1602                                 del keyword
1603                         if bad_stable_keywords:
1604                                 stats["LIVEVCS.stable"] += 1
1605                                 fails["LIVEVCS.stable"].append(
1606                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1607                                                 bad_stable_keywords)
1608                         del bad_stable_keywords
1609
1610                         if keywords and not has_global_mask(pkg):
1611                                 stats["LIVEVCS.unmasked"] += 1
1612                                 fails["LIVEVCS.unmasked"].append(relative_path)
1613
1614                 if options.ignore_arches:
1615                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1616                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1617                 else:
1618                         arches=[]
1619                         for keyword in myaux["KEYWORDS"].split():
1620                                 if (keyword[0]=="-"):
1621                                         continue
1622                                 elif (keyword[0]=="~"):
1623                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1624                                 else:
1625                                         arches.append([keyword, keyword, [keyword]])
1626                                         allmasked = False
1627                         if not arches:
1628                                 # Use an empty profile for checking dependencies of
1629                                 # packages that have empty KEYWORDS.
1630                                 arches.append(['**', '**', ['**']])
1631
1632                 baddepsyntax = False
1633                 badlicsyntax = False
1634                 badprovsyntax = False
1635                 catpkg = catdir+"/"+y
1636
1637                 inherited_java_eclass = "java-pkg-2" in inherited or \
1638                         "java-pkg-opt-2" in inherited
1639                 inherited_wxwidgets_eclass = "wxwidgets" in inherited
1640                 operator_tokens = set(["||", "(", ")"])
1641                 type_list, badsyntax = [], []
1642                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1643                         "LICENSE", "PROPERTIES", "PROVIDE"):
1644                         mydepstr = myaux[mytype]
1645
1646                         token_class = None
1647                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1648                                 token_class=portage.dep.Atom
1649
1650                         try:
1651                                 atoms = portage.dep.use_reduce(mydepstr, matchall=1, flat=True, \
1652                                         is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
1653                         except portage.exception.InvalidDependString as e:
1654                                 atoms = None
1655                                 badsyntax.append(str(e))
1656
1657                         if atoms and mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1658                                 if mytype in ("RDEPEND", "PDEPEND") and \
1659                                         "test?" in mydepstr.split():
1660                                         stats[mytype + '.suspect'] += 1
1661                                         fails[mytype + '.suspect'].append(relative_path + \
1662                                                 ": 'test?' USE conditional in %s" % mytype)
1663
1664                                 for atom in atoms:
1665                                         if atom == "||":
1666                                                 continue
1667
1668                                         is_blocker = atom.blocker
1669
1670                                         if mytype == "DEPEND" and \
1671                                                 not is_blocker and \
1672                                                 not inherited_java_eclass and \
1673                                                 atom.cp == "virtual/jdk":
1674                                                 stats['java.eclassesnotused'] += 1
1675                                                 fails['java.eclassesnotused'].append(relative_path)
1676                                         elif mytype == "DEPEND" and \
1677                                                 not is_blocker and \
1678                                                 not inherited_wxwidgets_eclass and \
1679                                                 atom.cp == "x11-libs/wxGTK":
1680                                                 stats['wxwidgets.eclassnotused'] += 1
1681                                                 fails['wxwidgets.eclassnotused'].append(
1682                                                         relative_path + ": DEPENDs on x11-libs/wxGTK"
1683                                                         " without inheriting wxwidgets.eclass")
1684                                         elif mytype in ("PDEPEND", "RDEPEND"):
1685                                                 if not is_blocker and \
1686                                                         atom.cp in suspect_rdepend:
1687                                                         stats[mytype + '.suspect'] += 1
1688                                                         fails[mytype + '.suspect'].append(
1689                                                                 relative_path + ": '%s'" % atom)
1690
1691                                         if atom.operator == "~" and \
1692                                                 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1693                                                 stats[mytype + '.badtilde'] += 1
1694                                                 fails[mytype + '.badtilde'].append(
1695                                                         (relative_path + ": %s uses the ~ operator"
1696                                                          " with a non-zero revision:" + \
1697                                                          " '%s'") % (mytype, atom))
1698
1699                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1700
1701                 for m,b in zip(type_list, badsyntax):
1702                         stats[m+".syntax"] += 1
1703                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1704
1705                 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1706                 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1707                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1708                 badlicsyntax = badlicsyntax > 0
1709                 badprovsyntax = badprovsyntax > 0
1710
1711                 # uselist checks - global
1712                 myuse = []
1713                 default_use = []
1714                 for myflag in myaux["IUSE"].split():
1715                         flag_name = myflag.lstrip("+-")
1716                         used_useflags.add(flag_name)
1717                         if myflag != flag_name:
1718                                 default_use.append(myflag)
1719                         if flag_name not in uselist:
1720                                 myuse.append(flag_name)
1721
1722                 # uselist checks - metadata
1723                 for mypos in range(len(myuse)-1,-1,-1):
1724                         if myuse[mypos] and (myuse[mypos] in muselist):
1725                                 del myuse[mypos]
1726
1727                 if default_use and not eapi_has_iuse_defaults(eapi):
1728                         for myflag in default_use:
1729                                 stats['EAPI.incompatible'] += 1
1730                                 fails['EAPI.incompatible'].append(
1731                                         (relative_path + ": IUSE defaults" + \
1732                                         " not supported with EAPI='%s':" + \
1733                                         " '%s'") % (eapi, myflag))
1734
1735                 for mypos in range(len(myuse)):
1736                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1737                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1738
1739                 # license checks
1740                 if not badlicsyntax:
1741                         # Parse the LICENSE variable, remove USE conditions and
1742                         # flatten it.
1743                         licenses = portage.dep.use_reduce(myaux["LICENSE"], matchall=1, flat=True)
1744                         # Check each entry to ensure that it exists in PORTDIR's
1745                         # license directory.
1746                         for lic in licenses:
1747                                 # Need to check for "||" manually as no portage
1748                                 # function will remove it without removing values.
1749                                 if lic not in liclist and lic != "||":
1750                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1751                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % lic)
1752
1753                 #keyword checks
1754                 myuse = myaux["KEYWORDS"].split()
1755                 for mykey in myuse:
1756                         myskey=mykey[:]
1757                         if myskey[0]=="-":
1758                                 myskey=myskey[1:]
1759                         if myskey[0]=="~":
1760                                 myskey=myskey[1:]
1761                         if mykey!="-*":
1762                                 if myskey not in kwlist:
1763                                         stats["KEYWORDS.invalid"] += 1
1764                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1765                                 elif myskey not in profiles:
1766                                         stats["KEYWORDS.invalid"] += 1
1767                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1768
1769                 #restrict checks
1770                 myrestrict = None
1771                 try:
1772                         myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
1773                 except portage.exception.InvalidDependString as e:
1774                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1775                         fails["RESTRICT.syntax"].append(
1776                                 "%s: RESTRICT: %s" % (relative_path, e))
1777                         del e
1778                 if myrestrict:
1779                         myrestrict = set(myrestrict)
1780                         mybadrestrict = myrestrict.difference(valid_restrict)
1781                         if mybadrestrict:
1782                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
1783                                 for mybad in mybadrestrict:
1784                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1785                 #REQUIRED_USE check
1786                 required_use = myaux["REQUIRED_USE"]
1787                 if required_use:
1788                         if not eapi_has_required_use(eapi):
1789                                 stats['EAPI.incompatible'] += 1
1790                                 fails['EAPI.incompatible'].append(
1791                                         relative_path + ": REQUIRED_USE" + \
1792                                         " not supported with EAPI='%s'" % (eapi,))
1793                         try:
1794                                 portage.dep.check_required_use(required_use, (),
1795                                         pkg.iuse.is_valid_flag)
1796                         except portage.exception.InvalidDependString as e:
1797                                 stats["REQUIRED_USE.syntax"] = stats["REQUIRED_USE.syntax"] + 1
1798                                 fails["REQUIRED_USE.syntax"].append(
1799                                         "%s: REQUIRED_USE: %s" % (relative_path, e))
1800                                 del e
1801
1802                 # Syntax Checks
1803                 relative_path = os.path.join(x, y + ".ebuild")
1804                 full_path = os.path.join(repodir, relative_path)
1805                 if not vcs_preserves_mtime:
1806                         if ebuild_path not in new_ebuilds and \
1807                                 ebuild_path not in modified_ebuilds:
1808                                 pkg.mtime = None
1809                 try:
1810                         # All ebuilds should have utf_8 encoding.
1811                         f = codecs.open(_unicode_encode(full_path,
1812                                 encoding=_encodings['fs'], errors='strict'),
1813                                 mode='r', encoding=_encodings['repo.content'])
1814                         try:
1815                                 for check_name, e in run_checks(f, pkg):
1816                                         stats[check_name] += 1
1817                                         fails[check_name].append(relative_path + ': %s' % e)
1818                         finally:
1819                                 f.close()
1820                 except UnicodeDecodeError:
1821                         # A file.UTF8 failure will have already been recorded above.
1822                         pass
1823
1824                 if options.force:
1825                         # The dep_check() calls are the most expensive QA test. If --force
1826                         # is enabled, there's no point in wasting time on these since the
1827                         # user is intent on forcing the commit anyway.
1828                         continue
1829
1830                 for keyword,arch,groups in arches:
1831
1832                         if arch not in profiles:
1833                                 # A missing profile will create an error further down
1834                                 # during the KEYWORDS verification.
1835                                 continue
1836
1837                         for prof in profiles[arch]:
1838
1839                                 if prof.status not in ("stable", "dev") or \
1840                                         prof.status == "dev" and not options.include_dev:
1841                                         continue
1842
1843                                 dep_settings = arch_caches.get(prof.sub_path)
1844                                 if dep_settings is None:
1845                                         dep_settings = portage.config(
1846                                                 config_profile_path=prof.abs_path,
1847                                                 config_incrementals=repoman_incrementals,
1848                                                 local_config=False,
1849                                                 _unmatched_removal=options.unmatched_removal,
1850                                                 env=env)
1851                                         if options.without_mask:
1852                                                 dep_settings._mask_manager = \
1853                                                         copy.deepcopy(dep_settings._mask_manager)
1854                                                 dep_settings._mask_manager._pmaskdict.clear()
1855                                         arch_caches[prof.sub_path] = dep_settings
1856
1857                                 xmatch_cache_key = (prof.sub_path, tuple(groups))
1858                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
1859                                 if xcache is None:
1860                                         portdb.melt()
1861                                         portdb.freeze()
1862                                         xcache = portdb.xcache
1863                                         xcache.update(shared_xmatch_caches)
1864                                         arch_xmatch_caches[xmatch_cache_key] = xcache
1865
1866                                 trees["/"]["porttree"].settings = dep_settings
1867                                 portdb.settings = dep_settings
1868                                 portdb.xcache = xcache
1869                                 # for package.use.mask support inside dep_check
1870                                 dep_settings.setcpv(pkg)
1871                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1872                                 # just in case, prevent config.reset() from nuking these.
1873                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1874
1875                                 if not baddepsyntax:
1876                                         ismasked = not ebuild_archs or \
1877                                                 pkg.cpv not in portdb.xmatch("list-visible", pkg.cp)
1878                                         if ismasked:
1879                                                 if not have_pmasked:
1880                                                         have_pmasked = bool(dep_settings._getMaskAtom(
1881                                                                 pkg.cpv, pkg.metadata))
1882                                                 if options.ignore_masked:
1883                                                         continue
1884                                                 #we are testing deps for a masked package; give it some lee-way
1885                                                 suffix="masked"
1886                                                 matchmode = "minimum-all"
1887                                         else:
1888                                                 suffix=""
1889                                                 matchmode = "minimum-visible"
1890
1891                                         if not have_dev_keywords:
1892                                                 have_dev_keywords = \
1893                                                         bool(dev_keywords.intersection(keywords))
1894
1895                                         if prof.status == "dev":
1896                                                 suffix=suffix+"indev"
1897
1898                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1899
1900                                                 mykey=mytype+".bad"+suffix
1901                                                 myvalue = myaux[mytype]
1902                                                 if not myvalue:
1903                                                         continue
1904
1905                                                 success, atoms = portage.dep_check(myvalue, portdb,
1906                                                         dep_settings, use="all", mode=matchmode,
1907                                                         trees=trees)
1908
1909                                                 if success:
1910                                                         if atoms:
1911
1912                                                                 if not prof.sub_path:
1913                                                                         # old-style virtuals currently aren't
1914                                                                         # resolvable with empty profile, since
1915                                                                         # 'virtuals' mappings are unavailable
1916                                                                         # (it would be expensive to search
1917                                                                         # for PROVIDE in all ebuilds)
1918                                                                         atoms = [atom for atom in atoms if not \
1919                                                                                 (atom.cp.startswith('virtual/') and \
1920                                                                                 not portdb.cp_list(atom.cp))]
1921
1922                                                                 #we have some unsolvable deps
1923                                                                 #remove ! deps, which always show up as unsatisfiable
1924                                                                 atoms = [str(atom.unevaluated_atom) \
1925                                                                         for atom in atoms if not atom.blocker]
1926
1927                                                                 #if we emptied out our list, continue:
1928                                                                 if not atoms:
1929                                                                         continue
1930                                                                 stats[mykey]=stats[mykey]+1
1931                                                                 fails[mykey].append("%s: %s(%s) %s" % \
1932                                                                         (relative_path, keyword,
1933                                                                         prof, repr(atoms)))
1934                                                 else:
1935                                                         stats[mykey]=stats[mykey]+1
1936                                                         fails[mykey].append("%s: %s(%s) %s" % \
1937                                                                 (relative_path, keyword,
1938                                                                 prof, repr(atoms)))
1939
1940         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1941         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1942         #if not portage.portdb.xmatch("bestmatch-visible",x):
1943         #    stats["ebuild.nostable"]+=1
1944         #    fails["ebuild.nostable"].append(x)
1945         if allmasked and repolevel == 3:
1946                 stats["ebuild.allmasked"]+=1
1947                 fails["ebuild.allmasked"].append(x)
1948
1949         # check if there are unused local USE-descriptions in metadata.xml
1950         # (unless there are any invalids, to avoid noise)
1951         if allvalid:
1952                 for myflag in muselist.difference(used_useflags):
1953                         stats["metadata.warning"] += 1
1954                         fails["metadata.warning"].append(
1955                                 "%s/metadata.xml: unused local USE-description: '%s'" % \
1956                                 (x, myflag))
1957
1958 if options.mode == "manifest":
1959         sys.exit(dofail)
1960
1961 #dofail will be set to 1 if we have failed in at least one non-warning category
1962 dofail=0
1963 #dowarn will be set to 1 if we tripped any warnings
1964 dowarn=0
1965 #dofull will be set if we should print a "repoman full" informational message
1966 dofull = options.mode != 'full'
1967
1968 for x in qacats:
1969         if not stats[x]:
1970                 continue
1971         dowarn = 1
1972         if x not in qawarnings:
1973                 dofail = 1
1974
1975 if dofail or \
1976         (dowarn and not (options.quiet or options.mode == "scan")):
1977         dofull = 0
1978
1979 # Save QA output so that it can be conveniently displayed
1980 # in $EDITOR while the user creates a commit message.
1981 # Otherwise, the user would not be able to see this output
1982 # once the editor has taken over the screen.
1983 qa_output = StringIO()
1984 style_file = ConsoleStyleFile(sys.stdout)
1985 if options.mode == 'commit' and \
1986         (not commitmessage or not commitmessage.strip()):
1987         style_file.write_listener = qa_output
1988 console_writer = StyleWriter(file=style_file, maxcol=9999)
1989 console_writer.style_listener = style_file.new_styles
1990
1991 f = formatter.AbstractFormatter(console_writer)
1992
1993 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
1994
1995 style_file.flush()
1996 del console_writer, f, style_file
1997 qa_output = qa_output.getvalue()
1998 qa_output = qa_output.splitlines(True)
1999
2000 def grouplist(mylist,seperator="/"):
2001         """(list,seperator="/") -- Takes a list of elements; groups them into
2002         same initial element categories. Returns a dict of {base:[sublist]}
2003         From: ["blah/foo","spork/spatula","blah/weee/splat"]
2004         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
2005         mygroups={}
2006         for x in mylist:
2007                 xs=x.split(seperator)
2008                 if xs[0]==".":
2009                         xs=xs[1:]
2010                 if xs[0] not in mygroups:
2011                         mygroups[xs[0]]=[seperator.join(xs[1:])]
2012                 else:
2013                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
2014         return mygroups
2015
2016 suggest_ignore_masked = False
2017 suggest_include_dev = False
2018
2019 if have_pmasked and not (options.without_mask or options.ignore_masked):
2020         suggest_ignore_masked = True
2021 if have_dev_keywords and not options.include_dev:
2022         suggest_include_dev = True
2023
2024 if suggest_ignore_masked or suggest_include_dev:
2025         print()
2026         if suggest_ignore_masked:
2027                 print(bold("Note: use --without-mask to check " + \
2028                         "KEYWORDS on dependencies of masked packages"))
2029
2030         if suggest_include_dev:
2031                 print(bold("Note: use --include-dev (-d) to check " + \
2032                         "dependencies for 'dev' profiles"))
2033         print()
2034
2035 if options.mode != 'commit':
2036         if dofull:
2037                 print(bold("Note: type \"repoman full\" for a complete listing."))
2038         if dowarn and not dofail:
2039                 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\"")
2040         elif not dofail:
2041                 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
2042         elif dofail:
2043                 print(bad("Please fix these important QA issues first."))
2044                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2045                 sys.exit(1)
2046 else:
2047         if dofail and can_force and options.force and not options.pretend:
2048                 print(green("RepoMan sez:") + \
2049                         " \"You want to commit even with these QA issues?\n" + \
2050                         "              I'll take it this time, but I'm not happy.\"\n")
2051         elif dofail:
2052                 if options.force and not can_force:
2053                         print(bad("The --force option has been disabled due to extraordinary issues."))
2054                 print(bad("Please fix these important QA issues first."))
2055                 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
2056                 sys.exit(1)
2057
2058         if options.pretend:
2059                 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
2060
2061         myunadded = []
2062         if vcs == "cvs":
2063                 try:
2064                         myvcstree=portage.cvstree.getentries("./",recursive=1)
2065                         myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
2066                 except SystemExit as e:
2067                         raise  # TODO propagate this
2068                 except:
2069                         err("Error retrieving CVS tree; exiting.")
2070         if vcs == "svn":
2071                 try:
2072                         svnstatus=os.popen("svn status --no-ignore").readlines()
2073                         myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
2074                 except SystemExit as e:
2075                         raise  # TODO propagate this
2076                 except:
2077                         err("Error retrieving SVN info; exiting.")
2078         if vcs == "git":
2079                 # get list of files not under version control or missing
2080                 myf = os.popen("git ls-files --others")
2081                 myunadded = [ "./" + elem[:-1] for elem in myf ]
2082                 myf.close()
2083         if vcs == "bzr":
2084                 try:
2085                         bzrstatus=os.popen("bzr status -S .").readlines()
2086                         myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ]
2087                 except SystemExit as e:
2088                         raise  # TODO propagate this
2089                 except:
2090                         err("Error retrieving bzr info; exiting.")
2091         if vcs == "hg":
2092                 myunadded = os.popen("hg status --no-status --unknown .").readlines()
2093                 myunadded = ["./" + elem.rstrip() for elem in myunadded]
2094
2095                 # Mercurial doesn't handle manually deleted files as removed from
2096                 # the repository, so the user need to remove them before commit,
2097                 # using "hg remove [FILES]"
2098                 mydeleted = os.popen("hg status --no-status --deleted .").readlines()
2099                 mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
2100
2101
2102         myautoadd=[]
2103         if myunadded:
2104                 for x in range(len(myunadded)-1,-1,-1):
2105                         xs=myunadded[x].split("/")
2106                         if xs[-1]=="files":
2107                                 print("!!! files dir is not added! Please correct this.")
2108                                 sys.exit(-1)
2109                         elif xs[-1]=="Manifest":
2110                                 # It's a manifest... auto add
2111                                 myautoadd+=[myunadded[x]]
2112                                 del myunadded[x]
2113
2114         if myautoadd:
2115                 print(">>> Auto-Adding missing Manifest(s)...")
2116                 if options.pretend:
2117                         if vcs == "cvs":
2118                                 print("(cvs add "+" ".join(myautoadd)+")")
2119                         elif vcs == "svn":
2120                                 print("(svn add "+" ".join(myautoadd)+")")
2121                         elif vcs == "git":
2122                                 print("(git add "+" ".join(myautoadd)+")")
2123                         elif vcs == "bzr":
2124                                 print("(bzr add "+" ".join(myautoadd)+")")
2125                         elif vcs == "hg":
2126                                 print("(hg add "+" ".join(myautoadd)+")")
2127                         retval=0
2128                 else:
2129                         if vcs == "cvs":
2130                                 retval=os.system("cvs add "+" ".join(myautoadd))
2131                         elif vcs == "svn":
2132                                 retval=os.system("svn add "+" ".join(myautoadd))
2133                         elif vcs == "git":
2134                                 retval=os.system("git add "+" ".join(myautoadd))
2135                         elif vcs == "bzr":
2136                                 retval=os.system("bzr add "+" ".join(myautoadd))
2137                         elif vcs == "hg":
2138                                 retval=os.system("hg add "+" ".join(myautoadd))
2139                 if retval:
2140                         writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \
2141                                 (vcs, retval), level=logging.ERROR, noiselevel=-1)
2142                         sys.exit(retval)
2143
2144         if myunadded:
2145                 print(red("!!! The following files are in your local tree but are not added to the master"))
2146                 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
2147                 for x in myunadded:
2148                         print("   ",x)
2149                 print()
2150                 print()
2151                 sys.exit(1)
2152
2153         if vcs == "hg" and mydeleted:
2154                 print(red("!!! The following files are removed manually from your local tree but are not"))
2155                 print(red("!!! removed from the repository. Please remove them, using \"hg remove [FILES]\"."))
2156                 for x in mydeleted:
2157                         print("   ",x)
2158                 print()
2159                 print()
2160                 sys.exit(1)
2161
2162         if vcs == "cvs":
2163                 mycvstree = cvstree.getentries("./", recursive=1)
2164                 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
2165                 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
2166                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
2167                 bin_blob_pattern = re.compile("^-kb$")
2168                 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
2169                         recursive=1, basedir="./"))
2170
2171
2172         if vcs == "svn":
2173                 svnstatus = os.popen("svn status").readlines()
2174                 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")]
2175                 mynew     = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")]
2176                 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")]
2177
2178                 # Subversion expands keywords specified in svn:keywords properties.
2179                 props = os.popen("svn propget -R svn:keywords").readlines()
2180                 expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \
2181                         for prop in props if " - " in prop)
2182
2183         elif vcs == "git":
2184                 mychanged = os.popen("git diff-index --name-only --relative --diff-filter=M HEAD").readlines()
2185                 mychanged = ["./" + elem[:-1] for elem in mychanged]
2186
2187                 mynew = os.popen("git diff-index --name-only --relative --diff-filter=A HEAD").readlines()
2188                 mynew = ["./" + elem[:-1] for elem in mynew]
2189
2190                 myremoved = os.popen("git diff-index --name-only --relative --diff-filter=D HEAD").readlines()
2191                 myremoved = ["./" + elem[:-1] for elem in myremoved]
2192
2193         if vcs == "bzr":
2194                 bzrstatus = os.popen("bzr status -S .").readlines()
2195                 mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ]
2196                 mynew     = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ]
2197                 myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ]
2198                 myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ]
2199                 # Bazaar expands nothing.
2200
2201         if vcs == "hg":
2202                 mychanged = os.popen("hg status --no-status --modified .").readlines()
2203                 mychanged = ["./" + elem.rstrip() for elem in mychanged]
2204                 mynew = os.popen("hg status --no-status --added .").readlines()
2205                 mynew = ["./" + elem.rstrip() for elem in mynew]
2206                 myremoved = os.popen("hg status --no-status --removed .").readlines()
2207                 myremoved = ["./" + elem.rstrip() for elem in myremoved]
2208
2209         if vcs:
2210                 if not (mychanged or mynew or myremoved or (vcs == "hg" and mydeleted)):
2211                         print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
2212                         print()
2213                         print("(Didn't find any changed files...)")
2214                         print()
2215                         sys.exit(1)
2216
2217         # Manifests need to be regenerated after all other commits, so don't commit
2218         # them now even if they have changed.
2219         mymanifests = set()
2220         myupdates = set()
2221         for f in mychanged + mynew:
2222                 if "Manifest" == os.path.basename(f):
2223                         mymanifests.add(f)
2224                 else:
2225                         myupdates.add(f)
2226         if vcs in ('git', 'hg'):
2227                 myupdates.difference_update(myremoved)
2228         myupdates = list(myupdates)
2229         mymanifests = list(mymanifests)
2230         myheaders = []
2231         mydirty = []
2232
2233         print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2234         if vcs not in ('cvs', 'svn'):
2235                 # With git, bzr and hg, there's never any keyword expansion, so
2236                 # there's no need to regenerate manifests and all files will be
2237                 # committed in one big commit at the end.
2238                 print()
2239         else:
2240                 if vcs == 'cvs':
2241                         headerstring = "'\$(Header|Id).*\$'"
2242                 elif vcs == "svn":
2243                         svn_keywords = dict((k.lower(), k) for k in [
2244                                         "Rev",
2245                                         "Revision",
2246                                         "LastChangedRevision",
2247                                         "Date",
2248                                         "LastChangedDate",
2249                                         "Author",
2250                                         "LastChangedBy",
2251                                         "URL",
2252                                         "HeadURL",
2253                                         "Id",
2254                                         "Header",
2255                         ])
2256
2257                 for myfile in myupdates:
2258
2259                         # for CVS, no_expansion contains files that are excluded from expansion
2260                         if vcs == "cvs":
2261                                 if myfile in no_expansion:
2262                                         continue
2263
2264                         # for SVN, expansion contains files that are included in expansion
2265                         elif vcs == "svn":
2266                                 if myfile not in expansion:
2267                                         continue
2268
2269                                 # Subversion keywords are case-insensitive in svn:keywords properties, but case-sensitive in contents of files.
2270                                 enabled_keywords = []
2271                                 for k in expansion[myfile]:
2272                                         keyword = svn_keywords.get(k.lower())
2273                                         if keyword is not None:
2274                                                 enabled_keywords.append(keyword)
2275
2276                                 headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
2277
2278                         myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile)
2279                         if myout[0] == 0:
2280                                 myheaders.append(myfile)
2281
2282                 print("%s have headers that will change." % green(str(len(myheaders))))
2283                 print("* Files with headers will cause the manifests to be changed and committed separately.")
2284
2285         logging.info("myupdates: %s", myupdates)
2286         logging.info("myheaders: %s", myheaders)
2287
2288         commitmessage = options.commitmsg
2289         if options.commitmsgfile:
2290                 try:
2291                         f = codecs.open(_unicode_encode(options.commitmsgfile,
2292                         encoding=_encodings['fs'], errors='strict'),
2293                         mode='r', encoding=_encodings['content'], errors='replace')
2294                         commitmessage = f.read()
2295                         f.close()
2296                         del f
2297                 except (IOError, OSError) as e:
2298                         if e.errno == errno.ENOENT:
2299                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2300                         else:
2301                                 raise
2302                 # We've read the content so the file is no longer needed.
2303                 commitmessagefile = None
2304         if not commitmessage or not commitmessage.strip():
2305                 try:
2306                         editor = os.environ.get("EDITOR")
2307                         if editor and utilities.editor_is_executable(editor):
2308                                 commitmessage = utilities.get_commit_message_with_editor(
2309                                         editor, message=qa_output)
2310                         else:
2311                                 commitmessage = utilities.get_commit_message_with_stdin()
2312                 except KeyboardInterrupt:
2313                         exithandler()
2314                 if not commitmessage or not commitmessage.strip():
2315                         print("* no commit message?  aborting commit.")
2316                         sys.exit(1)
2317         commitmessage = commitmessage.rstrip()
2318         portage_version = getattr(portage, "VERSION", None)
2319         if portage_version is None:
2320                 sys.stderr.write("Failed to insert portage version in message!\n")
2321                 sys.stderr.flush()
2322                 portage_version = "Unknown"
2323         unameout = platform.system() + " "
2324         if platform.system() in ["Darwin", "SunOS"]:
2325                 unameout += platform.processor()
2326         else:
2327                 unameout += platform.machine()
2328         commitmessage += "\n\n(Portage version: %s/%s/%s" % \
2329                 (portage_version, vcs, unameout)
2330         if options.force:
2331                 commitmessage += ", RepoMan options: --force"
2332         commitmessage += ")"
2333
2334         if options.ask and userquery('Commit changes?', True) != 'Yes':
2335                 print("* aborting commit.")
2336                 sys.exit(1)
2337
2338         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2339                 myfiles = myupdates + myremoved
2340                 if not myheaders and "sign" not in repoman_settings.features:
2341                         myfiles += mymanifests
2342                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2343                 mymsg = os.fdopen(fd, "wb")
2344                 mymsg.write(_unicode_encode(commitmessage))
2345                 mymsg.close()
2346
2347                 print()
2348                 print(green("Using commit message:"))
2349                 print(green("------------------------------------------------------------------------------"))
2350                 print(commitmessage)
2351                 print(green("------------------------------------------------------------------------------"))
2352                 print()
2353
2354                 # Having a leading ./ prefix on file paths can trigger a bug in
2355                 # the cvs server when committing files to multiple directories,
2356                 # so strip the prefix.
2357                 myfiles = [f.lstrip("./") for f in myfiles]
2358
2359                 commit_cmd = [vcs]
2360                 commit_cmd.extend(vcs_global_opts)
2361                 commit_cmd.append("commit")
2362                 commit_cmd.extend(vcs_local_opts)
2363                 commit_cmd.extend(["-F", commitmessagefile])
2364                 commit_cmd.extend(myfiles)
2365
2366                 try:
2367                         if options.pretend:
2368                                 print("(%s)" % (" ".join(commit_cmd),))
2369                         else:
2370                                 retval = spawn(commit_cmd, env=os.environ)
2371                                 if retval != os.EX_OK:
2372                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2373                                                 "error code: %s\n") % (vcs, retval),
2374                                                 level=logging.ERROR, noiselevel=-1)
2375                                         sys.exit(retval)
2376                 finally:
2377                         try:
2378                                 os.unlink(commitmessagefile)
2379                         except OSError:
2380                                 pass
2381
2382         # Setup the GPG commands
2383         def gpgsign(filename):
2384                 gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
2385                 if gpgcmd is None:
2386                         raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" + \
2387                                 " Is make.globals missing?")
2388                 if "${PORTAGE_GPG_KEY}" in gpgcmd and \
2389                         "PORTAGE_GPG_KEY" not in repoman_settings:
2390                         raise MissingParameter("PORTAGE_GPG_KEY is unset!")
2391                 if "${PORTAGE_GPG_DIR}" in gpgcmd:
2392                         if "PORTAGE_GPG_DIR" not in repoman_settings:
2393                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2394                                         os.path.expanduser("~/.gnupg")
2395                                 logging.info("Automatically setting PORTAGE_GPG_DIR to '%s'" \
2396                                         % repoman_settings["PORTAGE_GPG_DIR"])
2397                         else:
2398                                 repoman_settings["PORTAGE_GPG_DIR"] = \
2399                                         os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
2400                         if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2401                                 raise portage.exception.InvalidLocation(
2402                                         "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2403                                         repoman_settings["PORTAGE_GPG_DIR"])
2404                 gpgvars = {"FILE": filename}
2405                 for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
2406                         v = repoman_settings.get(k)
2407                         if v is not None:
2408                                 gpgvars[k] = v
2409                 gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
2410                 if options.pretend:
2411                         print("("+gpgcmd+")")
2412                 else:
2413                         rValue = os.system(gpgcmd)
2414                         if rValue == os.EX_OK:
2415                                 os.rename(filename+".asc", filename)
2416                         else:
2417                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2418
2419         # When files are removed and re-added, the cvs server will put /Attic/
2420         # inside the $Header path. This code detects the problem and corrects it
2421         # so that the Manifest will generate correctly. See bug #169500.
2422         # Use binary mode in order to avoid potential character encoding issues.
2423         cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
2424         attic_str = b'/Attic/'
2425         attic_replace = b'/'
2426         for x in myheaders:
2427                 f = open(_unicode_encode(x,
2428                         encoding=_encodings['fs'], errors='strict'),
2429                         mode='rb')
2430                 mylines = f.readlines()
2431                 f.close()
2432                 modified = False
2433                 for i, line in enumerate(mylines):
2434                         if cvs_header_re.match(line) is not None and \
2435                                 attic_str in line:
2436                                 mylines[i] = line.replace(attic_str, attic_replace)
2437                                 modified = True
2438                 if modified:
2439                         portage.util.write_atomic(x, b''.join(mylines),
2440                                 mode='wb')
2441
2442         manifest_commit_required = True
2443         if vcs in ('cvs', 'svn') and (myupdates or myremoved):
2444                 myfiles = myupdates + myremoved
2445                 for x in range(len(myfiles)-1, -1, -1):
2446                         if myfiles[x].count("/") < 4-repolevel:
2447                                 del myfiles[x]
2448                 mydone=[]
2449                 if repolevel==3:   # In a package dir
2450                         repoman_settings["O"] = startdir
2451                         digestgen(mysettings=repoman_settings, myportdb=portdb)
2452                 elif repolevel==2: # In a category dir
2453                         for x in myfiles:
2454                                 xs=x.split("/")
2455                                 if len(xs) < 4-repolevel:
2456                                         continue
2457                                 if xs[0]==".":
2458                                         xs=xs[1:]
2459                                 if xs[0] in mydone:
2460                                         continue
2461                                 mydone.append(xs[0])
2462                                 repoman_settings["O"] = os.path.join(startdir, xs[0])
2463                                 if not os.path.isdir(repoman_settings["O"]):
2464                                         continue
2465                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2466                 elif repolevel==1: # repo-cvsroot
2467                         print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
2468                         for x in myfiles:
2469                                 xs=x.split("/")
2470                                 if len(xs) < 4-repolevel:
2471                                         continue
2472                                 if xs[0]==".":
2473                                         xs=xs[1:]
2474                                 if "/".join(xs[:2]) in mydone:
2475                                         continue
2476                                 mydone.append("/".join(xs[:2]))
2477                                 repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1])
2478                                 if not os.path.isdir(repoman_settings["O"]):
2479                                         continue
2480                                 digestgen(mysettings=repoman_settings, myportdb=portdb)
2481                 else:
2482                         print(red("I'm confused... I don't know where I am!"))
2483                         sys.exit(1)
2484
2485                 # Force an unsigned commit when more than one Manifest needs to be signed.
2486                 if repolevel < 3 and "sign" in repoman_settings.features:
2487
2488                         fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2489                         mymsg = os.fdopen(fd, "wb")
2490                         mymsg.write(_unicode_encode(commitmessage))
2491                         mymsg.write(b"\n (Unsigned Manifest commit)")
2492                         mymsg.close()
2493
2494                         commit_cmd = [vcs]
2495                         commit_cmd.extend(vcs_global_opts)
2496                         commit_cmd.append("commit")
2497                         commit_cmd.extend(vcs_local_opts)
2498                         commit_cmd.extend(["-F", commitmessagefile])
2499                         commit_cmd.extend(f.lstrip("./") for f in mymanifests)
2500
2501                         try:
2502                                 if options.pretend:
2503                                         print("(%s)" % (" ".join(commit_cmd),))
2504                                 else:
2505                                         retval = spawn(commit_cmd, env=os.environ)
2506                                         if retval:
2507                                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2508                                                         "error code: %s\n") % (vcs, retval),
2509                                                         level=logging.ERROR, noiselevel=-1)
2510                                                 sys.exit(retval)
2511                         finally:
2512                                 try:
2513                                         os.unlink(commitmessagefile)
2514                                 except OSError:
2515                                         pass
2516                         manifest_commit_required = False
2517
2518         signed = False
2519         if "sign" in repoman_settings.features:
2520                 signed = True
2521                 myfiles = myupdates + myremoved + mymanifests
2522                 try:
2523                         if repolevel==3:   # In a package dir
2524                                 repoman_settings["O"] = "."
2525                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2526                         elif repolevel==2: # In a category dir
2527                                 mydone=[]
2528                                 for x in myfiles:
2529                                         xs=x.split("/")
2530                                         if len(xs) < 4-repolevel:
2531                                                 continue
2532                                         if xs[0]==".":
2533                                                 xs=xs[1:]
2534                                         if xs[0] in mydone:
2535                                                 continue
2536                                         mydone.append(xs[0])
2537                                         repoman_settings["O"] = os.path.join(".", xs[0])
2538                                         if not os.path.isdir(repoman_settings["O"]):
2539                                                 continue
2540                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2541                         elif repolevel==1: # repo-cvsroot
2542                                 print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
2543                                 mydone=[]
2544                                 for x in myfiles:
2545                                         xs=x.split("/")
2546                                         if len(xs) < 4-repolevel:
2547                                                 continue
2548                                         if xs[0]==".":
2549                                                 xs=xs[1:]
2550                                         if "/".join(xs[:2]) in mydone:
2551                                                 continue
2552                                         mydone.append("/".join(xs[:2]))
2553                                         repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
2554                                         if not os.path.isdir(repoman_settings["O"]):
2555                                                 continue
2556                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2557                 except portage.exception.PortageException as e:
2558                         portage.writemsg("!!! %s\n" % str(e))
2559                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
2560                         signed = False
2561
2562         if vcs == 'git':
2563                 # It's not safe to use the git commit -a option since there might
2564                 # be some modified files elsewhere in the working tree that the
2565                 # user doesn't want to commit. Therefore, call git update-index
2566                 # in order to ensure that the index is updated with the latest
2567                 # versions of all new and modified files in the relevant portion
2568                 # of the working tree.
2569                 myfiles = mymanifests + myupdates
2570                 myfiles.sort()
2571                 update_index_cmd = ["git", "update-index"]
2572                 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2573                 if options.pretend:
2574                         print("(%s)" % (" ".join(update_index_cmd),))
2575                 else:
2576                         retval = spawn(update_index_cmd, env=os.environ)
2577                         if retval != os.EX_OK:
2578                                 writemsg_level(("!!! Exiting on %s (shell) " + \
2579                                         "error code: %s\n") % (vcs, retval),
2580                                         level=logging.ERROR, noiselevel=-1)
2581                                 sys.exit(retval)
2582
2583         if vcs in ['git', 'bzr', 'hg'] or manifest_commit_required or signed:
2584
2585                 myfiles = mymanifests[:]
2586                 if vcs in ['git', 'bzr', 'hg']:
2587                         myfiles += myupdates
2588                         myfiles += myremoved
2589                 myfiles.sort()
2590
2591                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2592                 mymsg = os.fdopen(fd, "wb")
2593                 # strip the closing parenthesis
2594                 mymsg.write(_unicode_encode(commitmessage[:-1]))
2595                 if signed:
2596                         mymsg.write(_unicode_encode(
2597                                 ", signed Manifest commit with key %s)" % \
2598                                 repoman_settings["PORTAGE_GPG_KEY"]))
2599                 else:
2600                         mymsg.write(b", unsigned Manifest commit)")
2601                 mymsg.close()
2602
2603                 commit_cmd = []
2604                 if options.pretend and vcs is None:
2605                         # substitute a bogus value for pretend output
2606                         commit_cmd.append("cvs")
2607                 else:
2608                         commit_cmd.append(vcs)
2609                 commit_cmd.extend(vcs_global_opts)
2610                 commit_cmd.append("commit")
2611                 commit_cmd.extend(vcs_local_opts)
2612                 if vcs == "hg":
2613                         commit_cmd.extend(["--logfile", commitmessagefile])
2614                         commit_cmd.extend(myfiles)
2615                 else:
2616                         commit_cmd.extend(["-F", commitmessagefile])
2617                         commit_cmd.extend(f.lstrip("./") for f in myfiles)
2618
2619                 try:
2620                         if options.pretend:
2621                                 print("(%s)" % (" ".join(commit_cmd),))
2622                         else:
2623                                 retval = spawn(commit_cmd, env=os.environ)
2624                                 if retval != os.EX_OK:
2625                                         writemsg_level(("!!! Exiting on %s (shell) " + \
2626                                                 "error code: %s\n") % (vcs, retval),
2627                                                 level=logging.ERROR, noiselevel=-1)
2628                                         sys.exit(retval)
2629                 finally:
2630                         try:
2631                                 os.unlink(commitmessagefile)
2632                         except OSError:
2633                                 pass
2634
2635         print()
2636         if vcs:
2637                 print("Commit complete.")
2638         else:
2639                 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2640         print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")
2641 sys.exit(0)
2642