bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2006 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 # Next to do: dep syntax checking in mask files
   7 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   8 # that last one is tricky because multiple profiles need to be checked.
   9
  10 import codecs
  11 import errno
  12 import formatter
  13 import logging
  14 import optparse
  15 import os
  16 import re
  17 import signal
  18 import stat
  19 import sys
  20 import tempfile
  21 import time
  22
  23 from commands import getstatusoutput
  24 from itertools import izip
  25 from stat import S_ISDIR, ST_CTIME
  26
  27 try:
  28         import cPickle as pickle
  29 except ImportError:
  30         import pickle
  31
  32 try:
  33         import cStringIO as StringIO
  34 except ImportError:
  35         import StringIO
  36
  37 if not hasattr(__builtins__, "set"):
  38         from sets import Set as set
  39
  40 os.environ["PORTAGE_LEGACY_GLOBALS"] = "false"
  41 try:
  42         import portage
  43 except ImportError:
  44         from os import path as osp
  45         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  46         import portage
  47 del os.environ["PORTAGE_LEGACY_GLOBALS"]
  48
  49 try:
  50         from repoman.checks import run_checks
  51         from repoman import utilities
  52 except ImportError:
  53         from os import path as osp
  54         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym'))
  55         from repoman.checks import run_checks
  56         from repoman import utilities
  57
  58 import portage.checksum
  59 import portage.const
  60 import portage.dep
  61 portage.dep._dep_check_strict = True
  62 import portage.exception
  63 from portage import cvstree, normalize_path
  64 from portage import util
  65 from portage.exception import ParseError
  66 from portage.manifest import Manifest
  67 from portage.process import find_binary, spawn
  68 from portage.output import bold, create_color_func, darkgreen, \
  69         green, nocolor, red, turquoise, yellow
  70 from portage.output import ConsoleStyleFile, StyleWriter
  71
  72 util.initialize_logger()
  73
  74 # 14 is the length of DESCRIPTION=""
  75 max_desc_len = 100
  76 allowed_filename_chars="a-zA-Z0-9._-+:"
  77 allowed_filename_chars_set = {}
  78 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('a'), ord('z')+1)))
  79 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('A'), ord('Z')+1)))
  80 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('0'), ord('9')+1)))
  81 map(allowed_filename_chars_set.setdefault, map(chr, map(ord, [".", "-", "_", "+", ":"])))
  82 bad = create_color_func("BAD")
  83
  84 # A sane umask is needed for files that portage creates.
  85 os.umask(022)
  86 repoman_settings = portage.config(local_config=False,
  87         config_incrementals=portage.const.INCREMENTALS)
  88 repoman_settings.lock()
  89
  90 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
  91         not sys.stdout.isatty():
  92         nocolor()
  93
  94 def warn(txt):
  95         print "repoman: " + txt
  96
  97 def err(txt):
  98         warn(txt)
  99         sys.exit(1)
 100
 101 def exithandler(signum=None, frame=None):
 102         logging.fatal("Interrupted; exiting...")
 103         sys.exit(1)
 104         os.kill(0, signal.SIGKILL)
 105
 106 signal.signal(signal.SIGINT,exithandler)
 107
 108 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
 109         """Repoman needs it's own HelpFormatter for now, because the default ones
 110         murder the help text."""
 111
 112         def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
 113                 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
 114
 115         def format_description(self, description):
 116                 return description
 117
 118 class RepomanOptionParser(optparse.OptionParser):
 119         """Add the on_tail function, ruby has it, optionParser should too
 120         """
 121
 122         def __init__(self, *args, **kwargs):
 123                 optparse.OptionParser.__init__(self, *args, **kwargs)
 124                 self.tail = ""
 125
 126         def on_tail(self, description):
 127                 self.tail += description
 128
 129         def format_help(self, formatter=None):
 130                 result = optparse.OptionParser.format_help(self, formatter)
 131                 result += self.tail
 132                 return result
 133
 134
 135 def ParseArgs(args, qahelp):
 136         """This function uses a customized optionParser to parse command line arguments for repoman
 137         Args:
 138           args - a sequence of command line arguments
 139                 qahelp - a dict of qa warning to help message
 140         Returns:
 141           (opts, args), just like a call to parser.parse_args()
 142         """
 143
 144         modes = {
 145                 'commit' : 'Run a scan then commit changes',
 146                 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
 147                 'full' : 'Scan directory tree and print all issues (not a summary)',
 148                 'help' : 'Show this screen',
 149                 'last' : 'Remember report from last run',
 150                 'lfull' : 'Remember report from last run (full listing)',
 151                 'manifest' : 'Generate a Manifest (fetches files if necessary)',
 152                 'scan' : 'Scan directory tree for QA issues'
 153         }
 154
 155         mode_keys = modes.keys()
 156         mode_keys.sort()
 157
 158         parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
 159         parser.description = green(" ".join((os.path.basename(args[0]), "1.2")))
 160         parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
 161         parser.description += "\nDistributed under the terms of the GNU General Public License v2"
 162         parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
 163
 164         parser.add_option('-m', '--commitmsg', dest='commitmsg',
 165                 help='specify a commit message on the command line')
 166
 167         parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
 168                 help='specify a path to a file that contains a commit message')
 169
 170         parser.add_option('-p', '--pretend', dest='pretend', default=False,
 171                 action='store_true', help='don\'t commit or fix anything; just show what would be done')
 172
 173         parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
 174                 help='do not print unnecessary messages')
 175
 176         parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
 177                 help='Commit with QA violations')
 178
 179         parser.add_option('-v', '--verbose', dest="verbosity", action='count',
 180                 help='be very verbose in output', default=0)
 181
 182         parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
 183                 default=False, help='forces the metadata.xml parse check to be carried out')
 184
 185         parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
 186                 default=False, help='ignore arch-specific failures (where arch != host)')
 187
 188         parser.add_option('-I', '--ignored-masked', dest='ignore_masked', action='store_true',
 189                 default=False, help='ignore masked packages (not allowed with commit mode')
 190
 191         parser.add_option('--mode', type='choice', dest='mode', choices=modes.keys(),
 192                 help='specify which mode repoman will run in (default=full)')
 193
 194         parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
 195
 196         for k in mode_keys:
 197                 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
 198
 199         parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
 200
 201         sorted_qa = qahelp.keys()
 202         sorted_qa.sort()
 203         for k in sorted_qa:
 204                 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
 205
 206         if not args:
 207                 args = sys.argv
 208         opts, args = parser.parse_args(args)
 209
 210         if opts.mode == 'help':
 211                 parser.print_help(short=False)
 212
 213         for arg in args:
 214                 if arg in modes:
 215                         if not opts.mode:
 216                                 opts.mode = arg
 217                                 break
 218
 219         if not opts.mode:
 220                 opts.mode = 'full'      #default to full
 221
 222         if opts.mode == 'commit' and opts.ignore_masked:
 223                 parser.error('Commit mode and --ignore_masked are not compatable')
 224
 225         # Use the verbosity and quiet options to fiddle with the loglevel appropriately
 226         for val in range(opts.verbosity):
 227                 logger = logging.getLogger()
 228                 logger.setLevel(logger.getEffectiveLevel() - 10)
 229
 230         for val in range(opts.quiet):
 231                 logger = logging.getLogger()
 232                 logger.setLevel(logger.getEffectiveLevel() + 10)
 233
 234         return (opts, args)
 235
 236 qahelp={
 237         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 238         "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
 239         "digest.partial":"Digest files do not contain all corresponding URI elements",
 240         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 241         "digestentry.unused":"Digest/Manifest entry has no matching SRC_URI entry",
 242         "digest.fail":"Digest does not match the specified local file",
 243         "digest.stray":"Digest files that do not have a corresponding ebuild",
 244         "digest.missing":"Digest files that are missing (ebuild exists, digest doesn't)",
 245         "digest.unmatch":"Digests which are incomplete (please check if your USE/ARCH includes all files)",
 246         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 247         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 248         "changelog.missing":"Missing ChangeLog files",
 249         "ebuild.disjointed":"Ebuilds not added to cvs when the matching digest has been added",
 250         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 251         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 252         "filedir.missing":"Package lacks a files directory",
 253         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
 254         "file.size":"Files in the files directory must be under 20k",
 255         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 256         "file.UTF8":"File is not UTF8 compliant",
 257         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 258         "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
 259         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 260         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 261         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 262         "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
 263         "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
 264         "SLOT.missing":"Ebuilds that have a missing or empty SLOT variable",
 265         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 266         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 267         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 268         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 269         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 270         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 271         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 272         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 273         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 274         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 275         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 276         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 277         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 278         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 279         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 280         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 281         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 282         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 283         "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
 284         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 285         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
 286         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 287         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 288         "variable.readonly":"Assigning a readonly variable",
 289         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 290         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file",
 291         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 292         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 293         "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
 294         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 295         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 296         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 297         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 298         "ebuild.badheader":"This ebuild has a malformed header",
 299         "metadata.missing":"Missing metadata.xml files",
 300         "metadata.bad":"Bad metadata.xml files",
 301         "virtual.versioned":"PROVIDE contains virtuals with versions",
 302         "virtual.exists":"PROVIDE contains existing package names",
 303         "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default",
 304         "usage.obsolete":"The ebuild makes use of an obsolete construct"
 305 }
 306
 307 qacats = qahelp.keys()
 308 qacats.sort()
 309
 310 qawarnings=[
 311 "changelog.missing",
 312 "changelog.notadded",
 313 "ebuild.notadded",
 314 "ebuild.nostable",
 315 "ebuild.allmasked",
 316 "ebuild.nesteddie",
 317 "desktop.invalid",
 318 "digest.assumed",
 319 "digest.missing",
 320 "digestentry.unused",
 321 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 322 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 323 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 324 "DESCRIPTION.toolong",
 325 "IUSE.invalid",
 326 "KEYWORDS.stupid",
 327 "KEYWORDS.missing",
 328 "RESTRICT.invalid",
 329 "ebuild.minorsyn",
 330 "ebuild.badheader",
 331 "file.size",
 332 "metadata.missing",
 333 "metadata.bad",
 334 "virtual.versioned",
 335 "virtual.exists",
 336 "virtual.unavailable",
 337 "usage.obsolete",
 338 "LIVEVCS.stable"
 339 ]
 340
 341 missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"]
 342 allvars=portage.auxdbkeys
 343 commitmessage=None
 344 for x in missingvars:
 345         x += ".missing"
 346         if x not in qacats:
 347                 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
 348                 qacats.append(x)
 349                 qawarnings.append(x)
 350
 351 valid_restrict = frozenset(["binchecks", "bindist",
 352         "fetch", "installsources", "mirror",
 353         "primaryuri", "strip", "test", "userpriv"])
 354
 355 # file.executable
 356 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
 357
 358
 359 def last(full=False):
 360         """Print the results of the last repoman run
 361         Args:
 362                 full - Print the complete results, if false, print a summary
 363         Returns:
 364                 Doesn't return (invokes sys.exit()
 365         """
 366         #Retrieve and unpickle stats and fails from saved files
 367         savedf=open('/var/cache/edb/repo.stats','r')
 368         stats = pickle.load(savedf)
 369         savedf.close()
 370         savedf=open('/var/cache/edb/repo.fails','r')
 371         fails = pickle.load(savedf)
 372         savedf.close()
 373
 374         #dofail will be set to 1 if we have failed in at least one non-warning category
 375         dofail=0
 376         #dowarn will be set to 1 if we tripped any warnings
 377         dowarn=0
 378         #dofull will be set if we should print a "repoman full" informational message
 379         dofull=0
 380
 381         dofull = options.mode not in ("full", "lfull")
 382
 383         for x in qacats:
 384                 if not stats[x]:
 385                         continue
 386                 dowarn = 1
 387                 if x not in qawarnings:
 388                         dofail = 1
 389
 390         print
 391         print green("RepoMan remembers...")
 392         print
 393         style_file = ConsoleStyleFile(sys.stdout)
 394         console_writer = StyleWriter(file=style_file, maxcol=9999)
 395         console_writer.style_listener = style_file.new_styles
 396         f = formatter.AbstractFormatter(console_writer)
 397         utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
 398         print
 399         if dofull:
 400                 print bold("Note: type \"repoman lfull\" for a complete listing of repomans last run.")
 401                 print
 402         if dowarn and not dofail:
 403                 print green("RepoMan sez:"),"\"You only gave me a partial QA payment last time?\n              I took it, but I wasn't happy.\""
 404         elif not dofail:
 405                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
 406         print
 407         sys.exit(0)
 408
 409 options, arguments = ParseArgs(sys.argv, qahelp)
 410
 411 if options.mode in ('last', 'lfull'):
 412         last('lfull' in options.mode)
 413
 414 # Set this to False when an extraordinary issue (generally
 415 # something other than a QA issue) makes it impossible to
 416 # commit (like if Manifest generation fails).
 417 can_force = True
 418
 419
 420 isCvs=False
 421 myreporoot=None
 422 if os.path.isdir("CVS"):
 423         isCvs = True
 424
 425 if isCvs and \
 426         "commit" == options.mode and \
 427         "RMD160" not in portage.checksum.hashorigin_map:
 428         from portage.util import grablines
 429         repo_lines = grablines("./CVS/Repository")
 430         if repo_lines and \
 431                 "gentoo-x86" == repo_lines[0].strip().split(os.path.sep)[0]:
 432                 msg = "Please install " \
 433                 "pycrypto or enable python's ssl USE flag in order " \
 434                 "to enable RMD160 hash support. See bug #198398 for " \
 435                 "more information."
 436                 prefix = bad(" * ")
 437                 from textwrap import wrap
 438                 for line in wrap(msg, 70):
 439                         print prefix + line
 440                 sys.exit(1)
 441         del repo_lines
 442
 443 if options.mode == 'commit' and not options.pretend and not isCvs:
 444         logging.info("Not in a CVS repository; enabling pretend mode.")
 445         options.pretend = True
 446
 447 try:
 448         portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
 449 except ValueError:
 450         sys.exit(1)
 451
 452 os.environ["PORTDIR"] = portdir
 453 if portdir_overlay != portdir:
 454         os.environ["PORTDIR_OVERLAY"] = portdir_overlay
 455 else:
 456         os.environ["PORTDIR_OVERLAY"] = ""
 457
 458 logging.info('Setting paths:')
 459 logging.info('PORTDIR = "' + os.environ['PORTDIR'] + '"')
 460 logging.info('PORTDIR_OVERLAY = "' + os.environ['PORTDIR_OVERLAY']+'"')
 461
 462 # Now that PORTDIR_OVERLAY is properly overridden, create the portdb.
 463 repoman_settings = portage.config(local_config=False,
 464         config_incrementals=portage.const.INCREMENTALS)
 465 trees = portage.create_trees()
 466 trees["/"]["porttree"].settings = repoman_settings
 467 portdb = trees["/"]["porttree"].dbapi
 468 portdb.mysettings = repoman_settings
 469 # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman.
 470 del trees["/"]["vartree"]
 471
 472 myreporoot = os.path.basename(portdir_overlay)
 473 myreporoot += mydir[len(portdir_overlay):]
 474
 475 reposplit = myreporoot.split(os.path.sep)
 476 repolevel = len(reposplit)
 477
 478 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 479 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 480 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 481 if options.mode == 'commit' and repolevel not in [1,2,3]:
 482         print red("***")+" Commit attempts *must* be from within a cvs co, category, or package directory."
 483         print red("***")+" Attempting to commit from a packages files directory will be blocked for instance."
 484         print red("***")+" This is intended behaviour, to ensure the manifest is recommited for a package."
 485         print red("***")
 486         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 487
 488 startdir = normalize_path(mydir)
 489 repodir = startdir
 490 for x in range(0, repolevel - 1):
 491         repodir = os.path.dirname(repodir)
 492
 493 # retreive local USE list
 494 luselist={}
 495 try:
 496         f = open(os.path.join(portdir, "profiles", "use.local.desc"))
 497         utilities.parse_use_local_desc(f, luselist)
 498         f.close()
 499 except (IOError, OSError, ParseError), e:
 500         logging.exception("Couldn't read from use.local.desc", e)
 501         sys.exit(1)
 502
 503 if portdir_overlay != portdir:
 504         filename = os.path.join(portdir_overlay, "profiles", "use.local.desc")
 505         if os.path.exists(filename):
 506                 try:
 507                         f = open(filename)
 508                         parse_use_local_desc(f, luselist)
 509                         f.close()
 510                 except (IOError, OSError, ParseError), e:
 511                         logging.exception("Couldn't read from '%s'" % filename, e)
 512                         sys.exit(1)
 513         del filename
 514
 515 # setup a uselist from portage
 516 uselist=[]
 517 try:
 518         uselist=portage.grabfile(portdir+"/profiles/use.desc")
 519         for l in range(0,len(uselist)):
 520                 uselist[l]=uselist[l].split()[0]
 521         for var in repoman_settings["USE_EXPAND"].split():
 522                 vardescs = portage.grabfile(portdir+"/profiles/desc/"+var.lower()+".desc")
 523                 for l in range(0, len(vardescs)):
 524                         uselist.append(var.lower() + "_" + vardescs[l].split()[0])
 525 except (IOError, OSError, ParseError), e:
 526         logging.exception("Couldn't read USE flags from use.desc", e)
 527         sys.exit(1)
 528
 529 # retrieve a list of current licenses in portage
 530 liclist = set(portage.listdir(os.path.join(portdir, "licenses")))
 531 if not liclist:
 532         logging.fatal("Couldn't find licenses?")
 533         sys.exit(1)
 534 if portdir_overlay != portdir:
 535         liclist.update(portage.listdir(os.path.join(portdir_overlay, "licenses")))
 536
 537 # retrieve list of offical keywords
 538 kwlist = set(portage.grabfile(os.path.join(portdir, "profiles", "arch.list")))
 539 if not kwlist:
 540         logging.fatal("Couldn't read KEYWORDS from arch.list")
 541         sys.exit(1)
 542
 543 manifest1_compat = False
 544 if portdir_overlay != portdir:
 545         kwlist.update(portage.grabfile(
 546                 os.path.join(portdir_overlay, "profiles", "arch.list")))
 547
 548 scanlist = utilities.FindPackagesToScan(repoman_settings, startdir, reposplit)
 549 scanlist.sort()
 550 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 551
 552 profiles={}
 553 descfile=portdir+"/profiles/profiles.desc"
 554 if os.path.exists(descfile):
 555         for x in portage.grabfile(descfile):
 556                 if x[0]=="#":
 557                         continue
 558                 arch=x.split()
 559                 if len(arch)!=3:
 560                         print "wrong format: \""+red(x)+"\" in "+descfile
 561                         continue
 562                 if not os.path.isdir(portdir+"/profiles/"+arch[1]):
 563                         print "Invalid "+arch[2]+" profile ("+arch[1]+") for arch "+arch[0]
 564                         continue
 565                 if profiles.has_key(arch[0]):
 566                         profiles[arch[0]]+= [[arch[1], arch[2]]]
 567                 else:
 568                         profiles[arch[0]] = [[arch[1], arch[2]]]
 569
 570         for x in repoman_settings.archlist():
 571                 if x[0] == "~":
 572                         continue
 573                 if not profiles.has_key(x):
 574                         print red("\""+x+"\" doesn't have a valid profile listed in profiles.desc.")
 575                         print red("You need to either \"cvs update\" your profiles dir or follow this")
 576                         print red("up with the "+x+" team.")
 577                         print
 578 else:
 579         print red("profiles.desc does not exist: "+descfile)
 580         print red("You need to do \"cvs update\" in profiles dir.")
 581         print
 582         sys.exit(1)
 583
 584
 585 stats={}
 586 fails={}
 587
 588 # provided by the desktop-file-utils package
 589 desktop_file_validate = find_binary("desktop-file-validate")
 590 desktop_pattern = re.compile(r'.*\.desktop$')
 591
 592 for x in qacats:
 593         stats[x]=0
 594         fails[x]=[]
 595 xmllint_capable = False
 596 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
 597 if options.mode == "manifest":
 598         pass
 599 elif not find_binary('xmllint'):
 600         print red("!!! xmllint not found. Can't check metadata.xml.\n")
 601         if options.xml_parse or repolevel==3:
 602                 print red("!!!")+" sorry, xmllint is needed.  failing\n"
 603                 sys.exit(1)
 604 else:
 605         #hardcoded paths/urls suck. :-/
 606         must_fetch=1
 607         backup_exists=0
 608         try:
 609                 # if it's been over a week since fetching (or the system clock is fscked), grab an updated copy of metadata.dtd
 610                 # clock is fscked or it's been a week. time to grab a new one.
 611                 ct = os.stat(metadata_dtd)[ST_CTIME]
 612                 if abs(time.time() - ct) > (60*60*24*7):
 613                         # don't trap the exception, we're watching for errno 2 (file not found), anything else is a bug.
 614                         backup_exists=1
 615                 else:
 616                         must_fetch=0
 617
 618         except (OSError,IOError), e:
 619                 if e.errno != 2:
 620                         print red("!!!")+" caught exception '%s' for %s/metadata.dtd, bailing" % (str(e), portage.CACHE_PATH)
 621                         sys.exit(1)
 622
 623         if must_fetch:
 624                 print
 625                 print green("***")+" the local copy of metadata.dtd needs to be refetched, doing that now"
 626                 print
 627                 val = 0
 628                 try:
 629                         try:
 630                                 os.unlink(metadata_dtd)
 631                         except OSError, e:
 632                                 if e.errno != errno.ENOENT:
 633                                         raise
 634                                 del e
 635                         val=portage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoman_settings,fetchonly=0, \
 636                                 try_mirrors=0)
 637
 638                 except SystemExit, e:
 639                         raise  # Need to propogate this
 640                 except Exception,e:
 641                         print
 642                         print red("!!!")+" attempting to fetch 'http://www.gentoo.org/dtd/metadata.dtd', caught"
 643                         print red("!!!")+" exception '%s' though." % str(e)
 644                         val=0
 645                 if not val:
 646                         print red("!!!")+" fetching new metadata.dtd failed, aborting"
 647                         sys.exit(1)
 648         #this can be problematic if xmllint changes their output
 649         xmllint_capable=True
 650
 651 if options.mode == 'commit' and isCvs:
 652         utilties.detect_vcs_conflicts(options, vcs="cvs")
 653
 654 if options.mode == "manifest":
 655         pass
 656 elif options.pretend:
 657         print green("\nRepoMan does a once-over of the neighborhood...")
 658 else:
 659         print green("\nRepoMan scours the neighborhood...")
 660
 661 new_ebuilds = set()
 662 if isCvs:
 663         mycvstree = cvstree.getentries("./", recursive=1)
 664         mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
 665         new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
 666         del mycvstree, mynew
 667
 668 dofail = 0
 669 arch_caches={}
 670 arch_xmatch_caches = {}
 671 shared_xmatch_caches = {"cp-list":{}}
 672
 673 for x in scanlist:
 674         #ebuilds and digests added to cvs respectively.
 675         logging.info("checking package %s" % x)
 676         eadded=[]
 677         dadded=[]
 678         catdir,pkgdir=x.split("/")
 679         checkdir=repodir+"/"+x
 680
 681         if options.mode == "manifest" or \
 682           options.mode in ('commit', 'fix') and not options.pretend:
 683                 repoman_settings["O"] = checkdir
 684                 if not portage.digestgen([], repoman_settings, myportdb=portdb):
 685                         print "Unable to generate manifest."
 686                         dofail = 1
 687                 if options.mode == "manifest":
 688                         continue
 689                 elif dofail:
 690                         sys.exit(1)
 691
 692         checkdirlist=os.listdir(checkdir)
 693         ebuildlist=[]
 694         ebuild_metadata = {}
 695         for y in checkdirlist:
 696                 if y in no_exec and \
 697                         stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0111:
 698                                 stats["file.executable"] += 1
 699                                 fails["file.executable"].append(os.path.join(checkdir, y))
 700                 if y.endswith(".ebuild"):
 701                         pf = y[:-7]
 702                         ebuildlist.append(pf)
 703                         cpv = "%s/%s" % (catdir, pf)
 704                         try:
 705                                 myaux = dict(izip(allvars, portdb.aux_get(cpv, allvars)))
 706                         except KeyError:
 707                                 stats["ebuild.syntax"] += 1
 708                                 fails["ebuild.syntax"].append(os.path.join(x, y))
 709                                 continue
 710                         except IOError:
 711                                 stats["ebuild.output"] += 1
 712                                 fails["ebuild.output"].append(os.path.join(x, y))
 713                                 continue
 714                         if not portage.eapi_is_supported(myaux["EAPI"]):
 715                                 stats["EAPI.unsupported"] += 1
 716                                 fails["EAPI.unsupported"].append(os.path.join(x, y))
 717                                 continue
 718                         ebuild_metadata[pf] = myaux
 719         ebuildlist.sort()
 720
 721         if len(ebuild_metadata) != len(ebuildlist):
 722                 # If we can't access all the metadata then it's totally unsafe to
 723                 # commit since there's no way to generate a correct Manifest.
 724                 # Do not try to do any more QA checks on this package since missing
 725                 # metadata leads to false positives for several checks, and false
 726                 # positives confuse users.
 727                 can_force = False
 728                 continue
 729
 730         for y in checkdirlist:
 731                 for c in y.strip(os.path.sep):
 732                         if c not in allowed_filename_chars_set:
 733                                 stats["file.name"] += 1
 734                                 fails["file.name"].append("%s/%s: char '%s'" % (checkdir, y, c))
 735                                 break
 736
 737                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
 738                         continue
 739                 try:
 740                         line = 1
 741                         for l in codecs.open(checkdir+"/"+y, "r", "utf8"):
 742                                 line +=1
 743                 except UnicodeDecodeError, ue:
 744                         stats["file.UTF8"] += 1
 745                         s = ue.object[:ue.start]
 746                         l2 = s.count("\n")
 747                         line += l2
 748                         if l2 != 0:
 749                                 s = s[s.rfind("\n") + 1:]
 750                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
 751
 752         has_filesdir = True
 753         if not os.path.isdir(os.path.join(checkdir, "files")):
 754                 has_filesdir = False
 755                 if manifest1_compat:
 756                         stats["filedir.missing"] += 1
 757                         fails["filedir.missing"].append(checkdir)
 758
 759         if isCvs:
 760                 try:
 761                         myf=open(checkdir+"/CVS/Entries","r")
 762                         myl=myf.readlines()
 763                         for l in myl:
 764                                 if l[0]!="/":
 765                                         continue
 766                                 splitl=l[1:].split("/")
 767                                 if not len(splitl):
 768                                         continue
 769                                 if splitl[0][-7:]==".ebuild":
 770                                         eadded.append(splitl[0][:-7])
 771                 except IOError:
 772                         if options.mode == 'commit':
 773                                 stats["CVS/Entries.IO_error"] += 1
 774                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
 775                         continue
 776
 777         if isCvs and has_filesdir:
 778                 try:
 779                         myf=open(checkdir+"/files/CVS/Entries","r")
 780                         myl=myf.readlines()
 781                         for l in myl:
 782                                 if l[0]!="/":
 783                                         continue
 784                                 splitl=l[1:].split("/")
 785                                 if not len(splitl):
 786                                         continue
 787                                 if splitl[0][:7]=="digest-":
 788                                         dadded.append(splitl[0][7:])
 789                 except IOError:
 790                         if options.mode == 'commit':
 791                                 stats["CVS/Entries.IO_error"] += 1
 792                                 fails["CVS/Entries.IO_error"].append(checkdir+"/files/CVS/Entries")
 793                         continue
 794
 795         mf = Manifest(checkdir, repoman_settings["DISTDIR"])
 796         mydigests=mf.getTypeDigests("DIST")
 797
 798         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
 799         myfiles_all = []
 800         src_uri_error = False
 801         for mykey in fetchlist_dict:
 802                 try:
 803                         myfiles_all.extend(fetchlist_dict[mykey])
 804                 except portage.exception.InvalidDependString, e:
 805                         src_uri_error = True
 806                         try:
 807                                 portdb.aux_get(mykey, ["SRC_URI"])
 808                         except KeyError:
 809                                 # This will be reported as an "ebuild.syntax" error.
 810                                 pass
 811                         else:
 812                                 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
 813                                 fails["SRC_URI.syntax"].append(
 814                                         "%s.ebuild SRC_URI: %s" % (mykey, e))
 815         del fetchlist_dict
 816         if not src_uri_error:
 817                 # This test can produce false positives if SRC_URI could not
 818                 # be parsed for one or more ebuilds. There's no point in
 819                 # producing a false error here since the root cause will
 820                 # produce a valid error elsewhere, such as "SRC_URI.syntax"
 821                 # or "ebuild.sytax".
 822                 myfiles_all = set(myfiles_all)
 823                 for entry in mydigests:
 824                         if entry not in myfiles_all:
 825                                 stats["digestentry.unused"] += 1
 826                                 fails["digestentry.unused"].append(checkdir+"::"+entry)
 827         del myfiles_all
 828
 829         if os.path.exists(checkdir+"/files"):
 830                 filesdirlist=os.listdir(checkdir+"/files")
 831                 if manifest1_compat:
 832                         for y in filesdirlist:
 833                                 if not y.startswith("digest-"):
 834                                         continue
 835                                 relative_path = os.path.join(x, "files", y)
 836                                 full_path = os.path.join(repodir, relative_path)
 837                                 if stat.S_IMODE(os.stat(full_path).st_mode) & 0111:
 838                                         stats["file.executable"] += 1
 839                                         fails["file.executable"].append(x+"/files/"+y)
 840
 841                                 mykey = catdir + "/" + y[7:]
 842                                 if y[7:] not in ebuildlist:
 843                                         #stray digest
 844                                         if options.mode == "fix":
 845                                                 if options.pretend:
 846                                                         print "(cd "+repodir+"/"+x+"/files; cvs rm -f "+y+")"
 847                                                 else:
 848                                                         os.system("(cd "+repodir+"/"+x+"/files; cvs rm -f "+y+")")
 849                                         else:
 850                                                 stats["digest.stray"]=stats["digest.stray"]+1
 851                                                 fails["digest.stray"].append(x+"/files/"+y)
 852                                 else:
 853                                         # We have an ebuild
 854                                         try:
 855                                                 myuris, myfiles = portdb.getfetchlist(mykey, all=True)
 856                                         except portage.exception.InvalidDependString, e:
 857                                                 # Already handled above.
 858                                                 continue
 859
 860                                         uri_dict = {}
 861                                         for myu in myuris:
 862                                                 myubn = os.path.basename(myu)
 863                                                 if myubn not in uri_dict:
 864                                                         uri_dict[myubn]  = [myu]
 865                                                 else:
 866                                                         uri_dict[myubn] += [myu]
 867
 868                                         for myf in uri_dict:
 869                                                 myff = repoman_settings["DISTDIR"] + "/" + myf
 870                                                 if not mydigests.has_key(myf):
 871                                                         uri_settings = portage.config(clone=repoman_settings)
 872                                                         if options.mode == "fix":
 873                                                                 if not portage.fetch(uri_dict[myf], uri_settings):
 874                                                                         stats["digest.unmatch"] += 1
 875                                                                         fails["digest.unmatch"].append(y+"::"+myf)
 876                                                                 else:
 877                                                                         eb_name = portdb.findname2(mykey)[0]
 878                                                                         portage.doebuild(eb_name, "digest", "/",
 879                                                                                 uri_settings, tree="porttree",
 880                                                                                 mydbapi=portdb)
 881                                                         else:
 882                                                                 stats["digest.partial"] += 1
 883                                                                 fails["digest.partial"].append(y+"::"+myf)
 884                                                 elif "assume-digests" not in repoman_settings.features:
 885                                                         if os.path.exists(myff):
 886                                                                 if not portage.checksum.verify_all(myff, mydigests[myf])[0]:
 887                                                                         stats["digest.fail"] += 1
 888                                                                         fails["digest.fail"].append(y+"::"+myf)
 889                                                         elif repolevel == 3:
 890                                                                 stats["digest.assumed"] += 1
 891                                                                 fails["digest.assumed"].append(y+"::"+myf)
 892
 893                 # recurse through files directory
 894                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
 895                 while filesdirlist:
 896                         y = filesdirlist.pop(0)
 897                         relative_path = os.path.join(x, "files", y)
 898                         full_path = os.path.join(repodir, relative_path)
 899                         try:
 900                                 mystat = os.stat(full_path)
 901                         except OSError, oe:
 902                                 if oe.errno == 2:
 903                                         # don't worry about it.  it likely was removed via fix above.
 904                                         continue
 905                                 else:
 906                                         raise oe
 907                         if S_ISDIR(mystat.st_mode):
 908                                 if y == "CVS":
 909                                         continue
 910                                 for z in os.listdir(checkdir+"/files/"+y):
 911                                         if z == "CVS":
 912                                                 continue
 913                                         filesdirlist.append(y+"/"+z)
 914                         # current policy is no files over 20k, this is the check.
 915                         elif mystat.st_size > 20480:
 916                                 stats["file.size"] += 1
 917                                 fails["file.size"].append("("+ str(mystat.st_size/1024) + "K) "+x+"/files/"+y)
 918
 919                         for c in os.path.basename(y.rstrip(os.path.sep)):
 920                                 if c not in allowed_filename_chars_set:
 921                                         stats["file.name"] += 1
 922                                         fails["file.name"].append("%s/files/%s: char '%s'" % (checkdir, y, c))
 923                                         break
 924
 925                         if desktop_file_validate and desktop_pattern.match(y):
 926                                 status, cmd_output = commands.getstatusoutput(
 927                                         "'%s' '%s'" % (desktop_file_validate, full_path))
 928                                 if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK:
 929                                         # Note: in the future we may want to grab the
 930                                         # warnings in addition to the errors. We're
 931                                         # just doing errors now since we don't want
 932                                         # to generate too much noise at first.
 933                                         error_re = re.compile(r'.*\s*error:\s*(.*)')
 934                                         for line in cmd_output.splitlines():
 935                                                 error_match = error_re.match(line)
 936                                                 if error_match is None:
 937                                                         continue
 938                                                 stats["desktop.invalid"] += 1
 939                                                 fails["desktop.invalid"].append(
 940                                                         relative_path + ': %s' % error_match.group(1))
 941
 942         del mydigests
 943
 944         if "ChangeLog" not in checkdirlist:
 945                 stats["changelog.missing"]+=1
 946                 fails["changelog.missing"].append(x+"/ChangeLog")
 947
 948         #metadata.xml file check
 949         if "metadata.xml" not in checkdirlist:
 950                 stats["metadata.missing"]+=1
 951                 fails["metadata.missing"].append(x+"/metadata.xml")
 952         #metadata.xml parse check
 953         else:
 954                 #Only carry out if in package directory or check forced
 955                 if xmllint_capable:
 956                         # xmlint can produce garbage output even on success, so only dump
 957                         # the ouput when it fails.
 958                         st, out = getstatusoutput(
 959                                 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
 960                                  (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
 961                         if st != os.EX_OK:
 962                                 print red("!!!") + " metadata.xml is invalid:"
 963                                 for z in out.splitlines():
 964                                         print red("!!! ")+z
 965                                 stats["metadata.bad"]+=1
 966                                 fails["metadata.bad"].append(x+"/metadata.xml")
 967
 968         allmasked = True
 969
 970         for y in ebuildlist:
 971                 relative_path = os.path.join(x, y + ".ebuild")
 972                 full_path = os.path.join(repodir, relative_path)
 973                 if stat.S_IMODE(os.stat(full_path).st_mode) & 0111:
 974                         stats["file.executable"] += 1
 975                         fails["file.executable"].append(x+"/"+y+".ebuild")
 976                 if isCvs and y not in eadded:
 977                         #ebuild not added to cvs
 978                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
 979                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
 980                         if y in dadded:
 981                                 stats["ebuild.disjointed"]=stats["ebuild.disjointed"]+1
 982                                 fails["ebuild.disjointed"].append(x+"/"+y+".ebuild")
 983                 if manifest1_compat and \
 984                         not os.path.exists(os.path.join(checkdir, "files", "digest-"+y)):
 985                         if options.mode == "fix":
 986                                 if options.pretend:
 987                                         print "You will need to run:"
 988                                         print "  /usr/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest"
 989                                 else:
 990                                         retval=os.system("/usr/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest")
 991                                         if retval:
 992                                                 print "!!! Exiting on ebuild digest (shell) error code:",retval
 993                                                 sys.exit(retval)
 994                         else:
 995                                 stats["digest.missing"]=stats["digest.missing"]+1
 996                                 fails["digest.missing"].append(x+"/files/digest-"+y)
 997                 myesplit=portage.pkgsplit(y)
 998                 if myesplit is None or myesplit[0] != x.split("/")[-1]:
 999                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1000                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1001                         continue
1002                 elif myesplit[0]!=pkgdir:
1003                         print pkgdir,myesplit[0]
1004                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1005                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1006                         continue
1007
1008                 myaux = ebuild_metadata[y]
1009
1010                 # Test for negative logic and bad words in the RESTRICT var.
1011                 #for x in myaux[allvars.index("RESTRICT")].split():
1012                 #       if x.startswith("no"):
1013                 #               print "Bad RESTRICT value: %s" % x
1014                 try:
1015                         myaux["PROVIDE"] = portage.dep.use_reduce(
1016                                 portage.dep.paren_reduce(myaux["PROVIDE"]), matchall=1)
1017                 except portage.exception.InvalidDependString, e:
1018                         stats["PROVIDE.syntax"] = stats["PROVIDE.syntax"] + 1
1019                         fails["PROVIDE.syntax"].append(mykey+".ebuild PROVIDE: "+str(e))
1020                         del e
1021                         continue
1022                 myaux["PROVIDE"] = " ".join(portage.flatten(myaux["PROVIDE"]))
1023                 for myprovide in myaux["PROVIDE"].split():
1024                         prov_cp = portage.dep_getkey(myprovide)
1025                         if prov_cp != myprovide:
1026                                 stats["virtual.versioned"]+=1
1027                                 fails["virtual.versioned"].append(x+"/"+y+".ebuild: "+myprovide)
1028                         prov_pkg = portage.dep_getkey(
1029                                 portage.best(portdb.xmatch("match-all", prov_cp)))
1030                         if prov_cp == prov_pkg:
1031                                 stats["virtual.exists"]+=1
1032                                 fails["virtual.exists"].append(x+"/"+y+".ebuild: "+prov_cp)
1033
1034                 for pos, missing_var in enumerate(missingvars):
1035                         if not myaux.get(missing_var):
1036                                 if catdir == "virtual" and \
1037                                         missing_var in ("HOMEPAGE", "LICENSE"):
1038                                         continue
1039                                 myqakey=missingvars[pos]+".missing"
1040                                 stats[myqakey]=stats[myqakey]+1
1041                                 fails[myqakey].append(x+"/"+y+".ebuild")
1042
1043                 # 14 is the length of DESCRIPTION=""
1044                 if len(myaux['DESCRIPTION']) > max_desc_len:
1045                         stats['DESCRIPTION.toolong'] += 1
1046                         fails['DESCRIPTION.toolong'].append(
1047                                 "%s: DESCRIPTION is %d characters (max %d)" % \
1048                                 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1049
1050                 keywords = myaux["KEYWORDS"].split()
1051                 stable_keywords = []
1052                 for keyword in keywords:
1053                         if not keyword.startswith("~") and \
1054                                 not keyword.startswith("-"):
1055                                 stable_keywords.append(keyword)
1056                 if stable_keywords:
1057                         ebuild_path = y + ".ebuild"
1058                         if repolevel < 3:
1059                                 ebuild_path = os.path.join(pkgdir, ebuild_path)
1060                         if repolevel < 2:
1061                                 ebuild_path = os.path.join(catdir, ebuild_path)
1062                         ebuild_path = os.path.join(".", ebuild_path)
1063                         if ebuild_path in new_ebuilds:
1064                                 stable_keywords.sort()
1065                                 stats["KEYWORDS.stable"] += 1
1066                                 fails["KEYWORDS.stable"].append(
1067                                         x + "/" + y + ".ebuild added with stable keywords: %s" % \
1068                                                 " ".join(stable_keywords))
1069
1070                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1071                 if "-*" in keywords:
1072                         haskeyword = False
1073                         for kw in keywords:
1074                                 if kw[0] == "~":
1075                                         kw = kw[1:]
1076                                 if kw in kwlist:
1077                                         haskeyword = True
1078                         if not haskeyword:
1079                                 stats["KEYWORDS.stupid"] += 1
1080                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1081
1082                 """
1083                 Ebuilds that inherit a "Live" eclasss (darcs,subversion,git,cvs,etc..) should
1084                 not be allowed to be marked stable
1085                 """
1086                 if set(["darcs","cvs","subversion","git"]).intersection(
1087                         myaux["INHERITED"].split()):
1088                         bad_stable_keywords = []
1089                         for keyword in myaux["KEYWORDS"].split():
1090                                 if not keyword.startswith("~") and \
1091                                         not keyword.startswith("-"):
1092                                         bad_stable_keywords.append(keyword)
1093                                 del keyword
1094                         if bad_stable_keywords:
1095                                 stats["LIVEVCS.stable"] += 1
1096                                 fails["LIVEVCS.stable"].append(
1097                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1098                                                 bad_stable_keywords)
1099                         del bad_stable_keywords
1100
1101                 if options.ignore_arches:
1102                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1103                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1104                 else:
1105                         arches=[]
1106                         for keyword in myaux["KEYWORDS"].split():
1107                                 if (keyword[0]=="-"):
1108                                         continue
1109                                 elif (keyword[0]=="~"):
1110                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1111                                 else:
1112                                         arches.append([keyword, keyword, [keyword]])
1113                                         allmasked = False
1114
1115                 baddepsyntax = False
1116                 badlicsyntax = False
1117                 badprovsyntax = False
1118                 catpkg = catdir+"/"+y
1119                 myiuse = set(repoman_settings.archlist())
1120                 for myflag in myaux["IUSE"].split():
1121                         if myflag.startswith("+"):
1122                                 myflag = myflag[1:]
1123                         myiuse.add(myflag)
1124
1125                 operator_tokens = set(["||", "(", ")"])
1126                 type_list, badsyntax = [], []
1127                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND", "LICENSE", "PROVIDE"):
1128                         mydepstr = myaux[mytype]
1129
1130                         if mydepstr.find(" ?") != -1:
1131                                 badsyntax.append("'?' preceded by space")
1132
1133                         try:
1134                                 # Missing closing parenthesis will result in a ValueError
1135                                 mydeplist = portage.dep.paren_reduce(mydepstr)
1136                                 # Missing opening parenthesis will result in a final "" element
1137                                 if "" in mydeplist or "(" in mydeplist:
1138                                         raise ValueError
1139                         except ValueError:
1140                                 badsyntax.append("parenthesis mismatch")
1141                                 mydeplist = []
1142                         except portage.exception.InvalidDependString, e:
1143                                 badsyntax.append(str(e))
1144                                 del e
1145                                 mydeplist = []
1146
1147                         try:
1148                                 portage.dep.use_reduce(mydeplist, excludeall=myiuse)
1149                         except portage.exception.InvalidDependString, e:
1150                                 badsyntax.append(str(e))
1151
1152                         for token in operator_tokens:
1153                                 if mydepstr.startswith(token+" "):
1154                                         myteststr = mydepstr[len(token):]
1155                                 else:
1156                                         myteststr = mydepstr
1157                                 if myteststr.endswith(" "+token):
1158                                         myteststr = myteststr[:-len(token)]
1159                                 while myteststr.find(" "+token+" ") != -1:
1160                                         myteststr = " ".join(myteststr.split(" "+token+" ", 1))
1161                                 if myteststr.find(token) != -1:
1162                                         badsyntax.append("'%s' not separated by space" % (token))
1163
1164
1165                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1166                                 for token in mydepstr.split():
1167                                         if token in operator_tokens or \
1168                                                 token.endswith("?"):
1169                                                 continue
1170                                         if not portage.isvalidatom(token, allow_blockers=True) or \
1171                                                 ":" in token and myaux["EAPI"] == "0":
1172                                                 badsyntax.append("'%s' not a valid atom" % token)
1173
1174                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1175
1176                 for m,b in zip(type_list, badsyntax):
1177                         stats[m+".syntax"] += 1
1178                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1179
1180                 badlicsyntax = len(filter(lambda x:x=="LICENSE", type_list))
1181                 badprovsyntax = len(filter(lambda x:x=="PROVIDE", type_list))
1182                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1183                 badlicsyntax = badlicsyntax > 0
1184                 badprovsyntax = badprovsyntax > 0
1185
1186                 # uselist checks - global
1187                 myuse = []
1188                 default_use = []
1189                 for myflag in myaux["IUSE"].split():
1190                         flag_name = myflag.lstrip("+-")
1191                         if myflag != flag_name:
1192                                 default_use.append(myflag)
1193                         if flag_name not in uselist:
1194                                 myuse.append(flag_name)
1195
1196                 # uselist checks - local
1197                 mykey = portage.dep_getkey(catpkg)
1198                 if luselist.has_key(mykey):
1199                         for mypos in range(len(myuse)-1,-1,-1):
1200                                 if myuse[mypos] and (myuse[mypos] in luselist[mykey]):
1201                                         del myuse[mypos]
1202                 if default_use and myaux["EAPI"] == "0":
1203                         myuse += default_use
1204                 for mypos in range(len(myuse)):
1205                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1206                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1207
1208                 # license checks
1209                 if not badlicsyntax:
1210                         myuse = myaux["LICENSE"]
1211                         # Parse the LICENSE variable, remove USE conditions and
1212                         # flatten it.
1213                         myuse=portage.dep.use_reduce(portage.dep.paren_reduce(myuse), matchall=1)
1214                         myuse=portage.flatten(myuse)
1215                         # Check each entry to ensure that it exists in PORTDIR's
1216                         # license directory.
1217                         for mypos in range(0,len(myuse)):
1218                                 # Need to check for "||" manually as no portage
1219                                 # function will remove it without removing values.
1220                                 if myuse[mypos] not in liclist and myuse[mypos] != "||":
1221                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1222                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1223
1224                 #keyword checks
1225                 myuse = myaux["KEYWORDS"].split()
1226                 for mykey in myuse:
1227                         myskey=mykey[:]
1228                         if myskey[0]=="-":
1229                                 myskey=myskey[1:]
1230                         if myskey[0]=="~":
1231                                 myskey=myskey[1:]
1232                         if mykey!="-*":
1233                                 if myskey not in kwlist:
1234                                         stats["KEYWORDS.invalid"] += 1
1235                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1236                                 elif not profiles.has_key(myskey):
1237                                         stats["KEYWORDS.invalid"] += 1
1238                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1239
1240                 #restrict checks
1241                 myrestrict = None
1242                 try:
1243                         myrestrict = portage.dep.use_reduce(
1244                                 portage.dep.paren_reduce(myaux["RESTRICT"]), matchall=1)
1245                 except portage.exception.InvalidDependString, e:
1246                         stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1247                         fails["RESTRICT.syntax"].append(mykey+".ebuild RESTRICT: "+str(e))
1248                         del e
1249                 if myrestrict:
1250                         myrestrict = set(portage.flatten(myrestrict))
1251                         mybadrestrict = myrestrict.difference(valid_restrict)
1252                         if mybadrestrict:
1253                                 stats["RESTRICT.invalid"] += len(mybadrestrict)
1254                                 for mybad in mybadrestrict:
1255                                         fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1256                 # Syntax Checks
1257                 relative_path = os.path.join(x, y + ".ebuild")
1258                 full_path = os.path.join(repodir, relative_path)
1259                 f = open(full_path, 'rb')
1260                 try:
1261                         for check_name, e in run_checks(f, os.stat(full_path).st_mtime):
1262                                 stats[check_name] += 1
1263                                 fails[check_name].append(relative_path + ': %s' % e)
1264                 finally:
1265                         f.close()
1266                         del f
1267
1268                 if options.force:
1269                         # The dep_check() calls are the most expensive QA test. If --force
1270                         # is enabled, there's no point in wasting time on these since the
1271                         # user is intent on forcing the commit anyway.
1272                         continue
1273
1274                 for keyword,arch,groups in arches:
1275
1276                         if not profiles.has_key(arch):
1277                                 # A missing profile will create an error further down
1278                                 # during the KEYWORDS verification.
1279                                 continue
1280
1281                         for prof in profiles[arch]:
1282
1283                                 profdir = portdir+"/profiles/"+prof[0]
1284
1285                                 if prof[0] in arch_caches:
1286                                         dep_settings = arch_caches[prof[0]]
1287                                 else:
1288                                         dep_settings = portage.config(
1289                                                 config_profile_path=profdir,
1290                                                 config_incrementals=portage.const.INCREMENTALS,
1291                                                 local_config=False)
1292                                         arch_caches[prof[0]] = dep_settings
1293                                         while True:
1294                                                 try:
1295                                                         # Protect ACCEPT_KEYWORDS from config.regenerate()
1296                                                         # (just in case)
1297                                                         dep_settings.incrementals.remove("ACCEPT_KEYWORDS")
1298                                                 except ValueError:
1299                                                         break
1300
1301                                 xmatch_cache_key = (prof[0], tuple(groups))
1302                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
1303                                 if xcache is None:
1304                                         portdb.melt()
1305                                         portdb.freeze()
1306                                         xcache = portdb.xcache
1307                                         xcache.update(shared_xmatch_caches)
1308                                         arch_xmatch_caches[xmatch_cache_key] = xcache
1309
1310                                 trees["/"]["porttree"].settings = dep_settings
1311                                 portdb.mysettings = dep_settings
1312                                 portdb.xcache = xcache
1313                                 # for package.use.mask support inside dep_check
1314                                 dep_settings.setcpv("/".join((catdir, y)))
1315                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1316                                 # just in case, prevent config.reset() from nuking these.
1317                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1318
1319                                 for myprovide in myaux["PROVIDE"].split():
1320                                         prov_cp = portage.dep_getkey(myprovide)
1321                                         if prov_cp not in dep_settings.getvirtuals():
1322                                                 stats["virtual.unavailable"]+=1
1323                                                 fails["virtual.unavailable"].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+prov_cp)
1324
1325                                 if not baddepsyntax:
1326                                         ismasked = os.path.join(catdir, y) not in \
1327                                                 portdb.xmatch("list-visible", x)
1328                                         if ismasked:
1329                                                 if options.ignore_masked:
1330                                                         continue
1331                                                 #we are testing deps for a masked package; give it some lee-way
1332                                                 suffix="masked"
1333                                                 matchmode = "minimum-all"
1334                                         else:
1335                                                 suffix=""
1336                                                 matchmode = "minimum-visible"
1337
1338                                         if prof[1] == "dev":
1339                                                 suffix=suffix+"indev"
1340
1341                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1342
1343                                                 mykey=mytype+".bad"+suffix
1344                                                 myvalue = myaux[mytype]
1345                                                 if not myvalue:
1346                                                         continue
1347                                                 try:
1348                                                         mydep = portage.dep_check(myvalue, portdb,
1349                                                                 dep_settings, use="all", mode=matchmode,
1350                                                                 trees=trees)
1351                                                 except KeyError, e:
1352                                                         stats[mykey]=stats[mykey]+1
1353                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(e[0]))
1354                                                         continue
1355
1356                                                 if mydep[0]==1:
1357                                                         if mydep[1]!=[]:
1358                                                                 #we have some unsolvable deps
1359                                                                 #remove ! deps, which always show up as unsatisfiable
1360                                                                 d=0
1361                                                                 while d<len(mydep[1]):
1362                                                                         if mydep[1][d][0]=="!":
1363                                                                                 del mydep[1][d]
1364                                                                         else:
1365                                                                                 d += 1
1366                                                                 #if we emptied out our list, continue:
1367                                                                 if not mydep[1]:
1368                                                                         continue
1369                                                                 stats[mykey]=stats[mykey]+1
1370                                                                 fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1371                                                 else:
1372                                                         stats[mykey]=stats[mykey]+1
1373                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1374
1375         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1376         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1377         #if not portage.portdb.xmatch("bestmatch-visible",x):
1378         #    stats["ebuild.nostable"]+=1
1379         #    fails["ebuild.nostable"].append(x)
1380         if allmasked and repolevel == 3:
1381                 stats["ebuild.allmasked"]+=1
1382                 fails["ebuild.allmasked"].append(x)
1383
1384 if options.mode == "manifest":
1385         sys.exit(dofail)
1386
1387 #Pickle and save results for instant reuse in last and lfull
1388 if os.access(portage.const.CACHE_PATH, os.W_OK):
1389         for myobj, fname in (stats, "repo.stats"), (fails, "repo.fails"):
1390                 fpath = os.path.join(portage.const.CACHE_PATH, fname)
1391                 savef = open(fpath, 'w')
1392                 pickle.dump(myobj, savef)
1393                 savef.close()
1394                 portage.apply_secpass_permissions(fpath, gid=portage.portage_gid,
1395                         mode=0664)
1396
1397 # TODO(antarus) This function and last () look familiar ;)
1398
1399 #dofail will be set to 1 if we have failed in at least one non-warning category
1400 dofail=0
1401 #dowarn will be set to 1 if we tripped any warnings
1402 dowarn=0
1403 #dofull will be set if we should print a "repoman full" informational message
1404 dofull = options.mode not in ("full", "lfull")
1405
1406 for x in qacats:
1407         if not stats[x]:
1408                 continue
1409         dowarn = 1
1410         if x not in qawarnings:
1411                 dofail = 1
1412
1413 if dofail or \
1414         (dowarn and not (options.quiet or options.mode == "scan")):
1415         dofull = 0
1416
1417 # Save QA output so that it can be conveniently displayed
1418 # in $EDITOR while the user creates a commit message.
1419 # Otherwise, the user would not be able to see this output
1420 # once the editor has taken over the screen.
1421 qa_output = StringIO.StringIO()
1422 style_file = ConsoleStyleFile(sys.stdout)
1423 if options.mode == 'commit' and \
1424         (not commitmessage or not commitmessage.strip()):
1425         style_file.write_listener = qa_output
1426 console_writer = StyleWriter(file=style_file, maxcol=9999)
1427 console_writer.style_listener = style_file.new_styles
1428
1429 f = formatter.AbstractFormatter(console_writer)
1430
1431 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
1432
1433 style_file.flush()
1434 del console_writer, f, style_file
1435 qa_output = qa_output.getvalue()
1436 qa_output = qa_output.splitlines(True)
1437
1438 def grouplist(mylist,seperator="/"):
1439         """(list,seperator="/") -- Takes a list of elements; groups them into
1440         same initial element categories. Returns a dict of {base:[sublist]}
1441         From: ["blah/foo","spork/spatula","blah/weee/splat"]
1442         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
1443         mygroups={}
1444         for x in mylist:
1445                 xs=x.split(seperator)
1446                 if xs[0]==".":
1447                         xs=xs[1:]
1448                 if xs[0] not in mygroups:
1449                         mygroups[xs[0]]=[seperator.join(xs[1:])]
1450                 else:
1451                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
1452         return mygroups
1453
1454 if options.mode != 'commit':
1455         if dofull:
1456                 print bold("Note: type \"repoman full\" for a complete listing.")
1457         if dowarn and not dofail:
1458                 print green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\""
1459         elif not dofail:
1460                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
1461         elif dofail:
1462                 print turquoise("Please fix these important QA issues first.")
1463                 print green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n"
1464                 sys.exit(1)
1465 else:
1466         if dofail and can_force and options.force and not options.pretend:
1467                 print green("RepoMan sez:") + \
1468                         " \"You want to commit even with these QA issues?\n" + \
1469                         "              I'll take it this time, but I'm not happy.\"\n"
1470         elif dofail:
1471                 if options.force and not can_force:
1472                         print bad("The --force option has been disabled due to extraordinary issues.")
1473                 print turquoise("Please fix these important QA issues first.")
1474                 print green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n"
1475                 sys.exit(1)
1476
1477         if options.pretend:
1478                 print green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n"
1479
1480         if fails["digest.missing"]:
1481                 print green("Creating missing digests...")
1482         for x in fails["digest.missing"]:
1483                 xs=x.split("/")
1484                 del xs[-2]
1485                 myeb="/".join(xs[:-1])+"/"+xs[-1][7:]
1486                 if options.pretend:
1487                         print "(ebuild "+portdir+"/"+myeb+".ebuild digest)"
1488                 else:
1489                         retval=os.system("ebuild "+portdir+"/"+myeb+".ebuild digest")
1490                         if retval:
1491                                 print "!!! Exiting on ebuild digest (shell) error code:",retval
1492                                 sys.exit(retval)
1493
1494         mycvstree=portage.cvstree.getentries("./",recursive=1)
1495         if isCvs and not mycvstree:
1496                 print "!!! It seems we don't have a cvs tree?"
1497                 sys.exit(3)
1498
1499         myunadded=portage.cvstree.findunadded(mycvstree,recursive=1,basedir="./")
1500         myautoadd=[]
1501         if myunadded:
1502                 for x in range(len(myunadded)-1,-1,-1):
1503                         xs=myunadded[x].split("/")
1504                         if xs[-1]=="files":
1505                                 print "!!! files dir is not added! Please correct this."
1506                                 sys.exit(-1)
1507                         elif xs[-1]=="Manifest":
1508                                 # It's a manifest... auto add
1509                                 myautoadd+=[myunadded[x]]
1510                                 del myunadded[x]
1511                         elif len(xs[-1])>=7:
1512                                 if xs[-1][:7]=="digest-":
1513                                         del xs[-2]
1514                                         myeb="/".join(xs[:-1]+[xs[-1][7:]])+".ebuild"
1515                                         if os.path.exists(myeb):
1516                                                 # Ebuild exists for digest... So autoadd it.
1517                                                 myautoadd+=[myunadded[x]]
1518                                                 del myunadded[x]
1519
1520         if myautoadd:
1521                 print ">>> Auto-Adding missing digests..."
1522                 if options.pretend:
1523                         print "(/usr/bin/cvs add "+" ".join(myautoadd)+")"
1524                         retval=0
1525                 else:
1526                         retval=os.system("/usr/bin/cvs add "+" ".join(myautoadd))
1527                 if retval:
1528                         print "!!! Exiting on cvs (shell) error code:",retval
1529                         sys.exit(retval)
1530
1531         if myunadded:
1532                 print red("!!! The following files are in your cvs tree but are not added to the master")
1533                 print red("!!! tree. Please remove them from the cvs tree or add them to the master tree.")
1534                 for x in myunadded:
1535                         print "   ",x
1536                 print
1537                 print
1538                 sys.exit(1)
1539
1540         if isCvs:
1541                 mycvstree=portage.cvstree.getentries("./",recursive=1)
1542                 mychanged=portage.cvstree.findchanged(mycvstree,recursive=1,basedir="./")
1543                 mynew=portage.cvstree.findnew(mycvstree,recursive=1,basedir="./")
1544                 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
1545                 bin_blob_pattern = re.compile("^-kb$")
1546                 bin_blobs = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
1547                         recursive=1, basedir="./"))
1548
1549                 if not (mychanged or mynew or myremoved):
1550                         print green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\""
1551                         print
1552                         print "(Didn't find any changed files...)"
1553                         print
1554                         sys.exit(0)
1555
1556         # Manifests need to be regenerated after all other commits, so don't commit
1557         # them now even if they have changed.
1558         mymanifests = [f for f in mychanged if 'Manifest' == os.path.basename(f)]
1559         mychanged = [f for f in mychanged if 'Manifest' != os.path.basename(f)]
1560         myupdates=mychanged+mynew
1561         myheaders=[]
1562         mydirty=[]
1563         headerstring="'\$(Header|Id)"
1564         headerstring+=".*\$'"
1565         for myfile in myupdates:
1566                 if myfile in bin_blobs:
1567                         continue
1568                 myout=getstatusoutput("egrep -q "+headerstring+" "+myfile)
1569                 if myout[0]==0:
1570                         myheaders.append(myfile)
1571
1572         print "*",green(str(len(myupdates))),"files being committed...",green(str(len(myheaders))),"have headers that will change."
1573         print "*","Files with headers will cause the manifests to be made and recommited."
1574         logging.info("myupdates:", str(myupdates))
1575         logging.info("myheaders:", str(myheaders))
1576
1577         commitmessage = options.commitmsg
1578         if options.commitmsgfile:
1579                 try:
1580                         f = open(options.commitmsgfile)
1581                         commitmessage = f.read()
1582                         f.close()
1583                         del f
1584                 except (IOError, OSError), e:
1585                         if e.errno == errno.ENOENT:
1586                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
1587                         else:
1588                                 raise
1589                 # We've read the content so the file is no longer needed.
1590                 commitmessagefile = None
1591         if not commitmessage or not commitmessage.strip():
1592                 try:
1593                         editor = os.environ.get("EDITOR")
1594                         if editor and utilities.editor_is_executable(editor):
1595                                 commitmessage = utilties.get_commit_message_with_editor(
1596                                         editor, message=qa_output)
1597                         else:
1598                                 commitmessage = utilties.get_commit_message_with_stdin()
1599                 except KeyboardInterrupt:
1600                         exithandler()
1601                 if not commitmessage or not commitmessage.strip():
1602                         print "* no commit message?  aborting commit."
1603                         sys.exit(1)
1604         commitmessage = commitmessage.rstrip()
1605         portage_version = getattr(portage, "VERSION", None)
1606         if portage_version is None:
1607                 sys.stderr.write("Failed to insert portage version in message!\n")
1608                 sys.stderr.flush()
1609                 portage_version = "Unknown"
1610         commitmessage += "\n(Portage version: "+str(portage_version)
1611         if options.force:
1612                 commitmessage += ", RepoMan options: --force"
1613         commitmessage += ")"
1614
1615         if myupdates or myremoved:
1616                 myfiles = myupdates + myremoved
1617                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1618                 mymsg = os.fdopen(fd, "w")
1619                 mymsg.write(commitmessage)
1620                 mymsg.close()
1621
1622                 print
1623                 print green("Using commit message:")
1624                 print green("------------------------------------------------------------------------------")
1625                 print commitmessage
1626                 print green("------------------------------------------------------------------------------")
1627                 print
1628
1629                 retval = None
1630                 if options.pretend:
1631                         print "(/usr/bin/cvs -q commit -F %s %s)" % \
1632                                 (commitmessagefile, " ".join(myfiles))
1633                 else:
1634                         retval = spawn(["/usr/bin/cvs", "-q", "commit",
1635                                 "-F", commitmessagefile] + myfiles,
1636                                 env=os.environ)
1637                 try:
1638                         os.unlink(commitmessagefile)
1639                 except OSError:
1640                         pass
1641                 if retval:
1642                         print "!!! Exiting on cvs (shell) error code:",retval
1643                         sys.exit(retval)
1644
1645         # Setup the GPG commands
1646         def gpgsign(filename):
1647                 if "PORTAGE_GPG_KEY" not in repoman_settings:
1648                         raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
1649                 if "PORTAGE_GPG_DIR" not in repoman_settings:
1650                         if os.environ.has_key("HOME"):
1651                                 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(os.environ["HOME"], ".gnupg")
1652                                 logging.info("Automatically setting PORTAGE_GPG_DIR to %s" % repoman_settings["PORTAGE_GPG_DIR"])
1653                         else:
1654                                 raise portage.exception.MissingParameter("PORTAGE_GPG_DIR is unset!")
1655                 gpg_dir = repoman_settings["PORTAGE_GPG_DIR"]
1656                 if gpg_dir.startswith("~") and "HOME" in os.environ:
1657                         repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(
1658                                 os.environ["HOME"], gpg_dir[1:].lstrip(os.path.sep))
1659                 if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
1660                         raise portage.exception.InvalidLocation(
1661                                 "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
1662                                 repoman_settings["PORTAGE_GPG_DIR"])
1663                 gpgcmd = "gpg --sign --clearsign --yes "
1664                 gpgcmd+= "--default-key "+repoman_settings["PORTAGE_GPG_KEY"]
1665                 if repoman_settings.has_key("PORTAGE_GPG_DIR"):
1666                         gpgcmd += " --homedir "+repoman_settings["PORTAGE_GPG_DIR"]
1667                 if options.pretend:
1668                         print "("+gpgcmd+" "+filename+")"
1669                 else:
1670                         rValue = os.system(gpgcmd+" "+filename)
1671                         if rValue == os.EX_OK:
1672                                 os.rename(filename+".asc", filename)
1673                         else:
1674                                 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
1675
1676         # When files are removed and re-added, the cvs server will put /Attic/
1677         # inside the $Header path. This code detects the problem and corrects it
1678         # so that the Manifest will generate correctly. See bug #169500.
1679         from portage.util import write_atomic
1680         cvs_header = re.compile(r'^#\s*\$Header.*\$$')
1681         for x in myheaders:
1682                 f = open(x)
1683                 mylines = f.readlines()
1684                 f.close()
1685                 modified = False
1686                 for i, line in enumerate(mylines):
1687                         if cvs_header.match(line) and "/Attic/" in line:
1688                                 mylines[i] = line.replace("/Attic/", "/")
1689                                 modified = True
1690                 if modified:
1691                         write_atomic(x, "".join(mylines))
1692
1693         manifest_commit_required = True
1694         if myheaders or myupdates or myremoved or mynew:
1695                 myfiles=myheaders+myupdates+myremoved+mynew
1696                 for x in range(len(myfiles)-1, -1, -1):
1697                         if myfiles[x].count("/") < 4-repolevel:
1698                                 del myfiles[x]
1699                 mydone=[]
1700                 if repolevel==3:   # In a package dir
1701                         repoman_settings["O"] = startdir
1702                         portage.digestgen([], repoman_settings, manifestonly=1,
1703                                 myportdb=portdb)
1704                 elif repolevel==2: # In a category dir
1705                         for x in myfiles:
1706                                 xs=x.split("/")
1707                                 if len(xs) < 4-repolevel:
1708                                         continue
1709                                 if xs[0]==".":
1710                                         xs=xs[1:]
1711                                 if xs[0] in mydone:
1712                                         continue
1713                                 mydone.append(xs[0])
1714                                 repoman_settings["O"] = os.path.join(startdir, xs[0])
1715                                 if not os.path.isdir(repoman_settings["O"]):
1716                                         continue
1717                                 portage.digestgen([], repoman_settings, manifestonly=1,
1718                                         myportdb=portdb)
1719                 elif repolevel==1: # repo-cvsroot
1720                         print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
1721                         for x in myfiles:
1722                                 xs=x.split("/")
1723                                 if len(xs) < 4-repolevel:
1724                                         continue
1725                                 if xs[0]==".":
1726                                         xs=xs[1:]
1727                                 if "/".join(xs[:2]) in mydone:
1728                                         continue
1729                                 mydone.append("/".join(xs[:2]))
1730                                 repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1])
1731                                 if not os.path.isdir(repoman_settings["O"]):
1732                                         continue
1733                                 portage.digestgen([], repoman_settings, manifestonly=1,
1734                                         myportdb=portdb)
1735                 else:
1736                         print red("I'm confused... I don't know where I am!")
1737                         sys.exit(1)
1738
1739                 # Force an unsigned commit when more than one Manifest needs to be signed.
1740                 if repolevel < 3 and "sign" in repoman_settings.features:
1741                         if options.pretend:
1742                                 print "(/usr/bin/cvs -q commit -F commitmessagefile)"
1743                         else:
1744                                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1745                                 mymsg = os.fdopen(fd, "w")
1746                                 mymsg.write(commitmessage)
1747                                 mymsg.write("\n (Unsigned Manifest commit)")
1748                                 mymsg.close()
1749                                 retval=os.system("/usr/bin/cvs -q commit -F "+commitmessagefile)
1750                                 try:
1751                                         os.unlink(commitmessagefile)
1752                                 except OSError:
1753                                         pass
1754                                 if retval:
1755                                         print "!!! Exiting on cvs (shell) error code:",retval
1756                                         sys.exit(retval)
1757                         manifest_commit_required = False
1758
1759         signed = False
1760         if "sign" in repoman_settings.features:
1761                 signed = True
1762                 myfiles = myupdates + myremoved + mymanifests
1763                 try:
1764                         if repolevel==3:   # In a package dir
1765                                 repoman_settings["O"] = "."
1766                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1767                         elif repolevel==2: # In a category dir
1768                                 mydone=[]
1769                                 for x in myfiles:
1770                                         xs=x.split("/")
1771                                         if len(xs) < 4-repolevel:
1772                                                 continue
1773                                         if xs[0]==".":
1774                                                 xs=xs[1:]
1775                                         if xs[0] in mydone:
1776                                                 continue
1777                                         mydone.append(xs[0])
1778                                         repoman_settings["O"] = os.path.join(".", xs[0])
1779                                         if not os.path.isdir(repoman_settings["O"]):
1780                                                 continue
1781                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1782                         elif repolevel==1: # repo-cvsroot
1783                                 print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
1784                                 mydone=[]
1785                                 for x in myfiles:
1786                                         xs=x.split("/")
1787                                         if len(xs) < 4-repolevel:
1788                                                 continue
1789                                         if xs[0]==".":
1790                                                 xs=xs[1:]
1791                                         if "/".join(xs[:2]) in mydone:
1792                                                 continue
1793                                         mydone.append("/".join(xs[:2]))
1794                                         repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
1795                                         if not os.path.isdir(repoman_settings["O"]):
1796                                                 continue
1797                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1798                 except portage.exception.PortageException, e:
1799                         portage.writemsg("!!! %s\n" % str(e))
1800                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
1801                         signed = False
1802
1803         if manifest_commit_required or signed:
1804                 if options.pretend:
1805                         print "(/usr/bin/cvs -q commit -F commitmessagefile)"
1806                 else:
1807                         fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1808                         mymsg = os.fdopen(fd, "w")
1809                         mymsg.write(commitmessage)
1810                         if signed:
1811                                 mymsg.write("\n (Signed Manifest commit)")
1812                         else:
1813                                 mymsg.write("\n (Unsigned Manifest commit)")
1814                         mymsg.close()
1815                         retval=os.system("/usr/bin/cvs -q commit -F "+commitmessagefile)
1816                         try:
1817                                 os.unlink(commitmessagefile)
1818                         except OSError:
1819                                 pass
1820                         if retval:
1821                                 print "!!! Exiting on cvs (shell) error code:",retval
1822                                 sys.exit(retval)
1823
1824         print
1825         if isCvs:
1826                 print "CVS commit complete."
1827         else:
1828                 print "repoman was too scared by not seeing any familiar cvs file that he forgot to commit anything"
1829         print green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n"
1830 sys.exit(0)
1831