bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2006 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 # Next to do: dep syntax checking in mask files
   7 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   8 # that last one is tricky because multiple profiles need to be checked.
   9
  10 import errno, os, shutil, sys
  11 if not hasattr(__builtins__, "set"):
  12         from sets import Set as set
  13 exename=os.path.basename(sys.argv[0])
  14 version="1.2"
  15
  16 allowed_filename_chars="a-zA-Z0-9._-+:"
  17 allowed_filename_chars_set = {}
  18 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('a'), ord('z')+1)))
  19 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('A'), ord('Z')+1)))
  20 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('0'), ord('9')+1)))
  21 map(allowed_filename_chars_set.setdefault, map(chr, map(ord, [".", "-", "_", "+", ":"])))
  22
  23 import signal,re,pickle,tempfile
  24
  25 os.environ["PORTAGE_LEGACY_GLOBALS"] = "false"
  26 try:
  27         import portage
  28 except ImportError:
  29         sys.path.insert(0, "/usr/lib/portage/pym")
  30         import portage
  31 del os.environ["PORTAGE_LEGACY_GLOBALS"]
  32
  33 import portage_checksum
  34 import portage_const
  35 import portage_dep
  36 portage_dep._dep_check_strict = True
  37 import portage_exception
  38 import cvstree
  39 import time
  40 import codecs
  41
  42 from portage_manifest import Manifest
  43 from portage_exception import ParseError
  44 from portage_exec import spawn
  45
  46 from output import bold, darkgreen, darkred, green, nocolor, red, turquoise, yellow
  47
  48 from commands import getstatusoutput
  49 from fileinput import input
  50 from grp import getgrnam
  51 from stat import S_ISDIR, ST_CTIME, ST_GID, ST_MTIME
  52
  53 # A sane umask is needed for files that portage creates.
  54 os.umask(022)
  55 repoman_settings = portage.config(local_config=False,
  56         config_incrementals=portage_const.INCREMENTALS)
  57 repoman_settings.lock()
  58
  59 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
  60         not sys.stdout.isatty():
  61         nocolor()
  62
  63 def warn(txt):
  64         print exename+": "+txt
  65 def err(txt):
  66         warn(txt)
  67         sys.exit(1)
  68
  69 def err_help(txt):
  70         help(exitstatus=-1,helpfulness=0)
  71         warn(txt)
  72         sys.exit(1)
  73
  74 def exithandler(signum=None,frame=None):
  75         sys.stderr.write("\n"+exename+": Interrupted; exiting...\n")
  76         sys.exit(1)
  77         os.kill(0,signal.SIGKILL)
  78 signal.signal(signal.SIGINT,exithandler)
  79
  80 shortmodes={"ci":"commit"}
  81 modeshelp={
  82 "scan"   : "Scan directory tree for QA issues (default)",
  83 "fix"    : "Fix simple QA issues (stray digests, missing digests)",
  84 "full"   : "Scan directory tree for QA issues (full listing)",
  85 "help"   : "Show this screen",
  86 "commit" : "Scan directory tree for QA issues; if OK, commit via cvs",
  87 "last"   : "Remember report from last run",
  88 "lfull"  : "Remember report from last run (full listing)"
  89 }
  90 modes=modeshelp.keys()
  91 repoman_options={
  92 "--commitmsg"      : "Adds a commit message via the command line",
  93 "--commitmsgfile"  : "Adds a commit message from the specified file",
  94 "--help"           : "Show this screen",
  95 "--ignore-arches"  : "Ignore arch-specific failures (where arch != host)",
  96 "--ignore-masked"  : "Ignore masked packages (not allowed with commit mode)",
  97 "--pretend"        : "Don't commit or fix anything; just show what would be done",
  98 "--quiet"          : "Be less verbose about extraneous info",
  99 "--verbose"        : "Displays every package name while checking",
 100 "--version"        : "Show version info",
 101 "--xmlparse"       : "Forces the metadata.xml parse check to be carried out"
 102 }
 103 repoman_shortoptions={
 104 "-h" : "--help",
 105 "-i" : "--ignore-masked",
 106 "-I" : "--ignore-arches",
 107 "-m" : "--commitmsg",
 108 "-M" : "--commitmsgfile",
 109 "-p" : "--pretend",
 110 "-q" : "--quiet",
 111 "-v" : "--verbose",
 112 "-V" : "--version",
 113 "-x" : "--xmlparse"
 114 }
 115 repoman_shortoptions_rev=dict([(v,k) for (k,v) in repoman_shortoptions.items()])
 116 options=repoman_options.keys()
 117
 118 qahelp={
 119         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 120         "digest.partial":"Digest files do not contain all corresponding URI elements",
 121         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 122         "digestentry.unused":"Digest/Manifest entry has no matching SRC_URI entry",
 123         "digest.fail":"Digest does not match the specified local file",
 124         "digest.stray":"Digest files that do not have a corresponding ebuild",
 125         "digest.missing":"Digest files that are missing (ebuild exists, digest doesn't)",
 126         "digest.disjointed":"Digests not added to cvs when the matching ebuild has been added",
 127         "digest.notadded":"Digests that exist but have not been added to cvs",
 128         "digest.unmatch":"Digests which are incomplete (please check if your USE/ARCH includes all files)",
 129         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 130         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 131         "changelog.missing":"Missing ChangeLog files",
 132         "ebuild.disjointed":"Ebuilds not added to cvs when the matching digest has been added",
 133         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 134         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 135         "filedir.missing":"Package lacks a files directory",
 136         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
 137         "file.size":"Files in the files directory must be under 20k",
 138         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 139         "file.UTF8":"File is not UTF8 compliant",
 140         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 141         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 142         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 143         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 144         "SLOT.missing":"Ebuilds that have a missing or empty SLOT variable",
 145         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 146         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 147         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 148         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 149         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 150         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 151         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 152         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 153         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 154         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 155         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 156         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 157         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 158         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 159         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 160         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 161         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 162         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 163         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error",
 164         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 165         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 166         "variable.readonly":"Assigning a readonly variable",
 167         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 168         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file",
 169         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 170         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 171         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 172         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 173         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 174         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 175         "ebuild.badheader":"This ebuild has a malformed header",
 176         "metadata.missing":"Missing metadata.xml files",
 177         "metadata.bad":"Bad metadata.xml files",
 178         "virtual.versioned":"PROVIDE contains virtuals with versions",
 179         "virtual.exists":"PROVIDE contains existing package names",
 180         "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default",
 181         "usage.obsolete":"The ebuild makes use of an obsolete construct"
 182 }
 183
 184 qacats = qahelp.keys()
 185 qacats.sort()
 186
 187 qawarnings=[
 188 "changelog.missing",
 189 "changelog.notadded",
 190 "ebuild.notadded",
 191 "ebuild.nostable",
 192 "ebuild.allmasked",
 193 "ebuild.nesteddie",
 194 "digest.assumed",
 195 "digest.notadded",
 196 "digest.disjointed",
 197 "digest.missing",
 198 "digestentry.unused",
 199 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 200 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 201 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 202 "IUSE.invalid",
 203 "KEYWORDS.stupid",
 204 "KEYWORDS.missing",
 205 "ebuild.minorsyn",
 206 "ebuild.badheader",
 207 "file.size",
 208 "metadata.missing",
 209 "metadata.bad",
 210 "virtual.versioned",
 211 "virtual.exists",
 212 "virtual.unavailable",
 213 "usage.obsolete",
 214 "LIVEVCS.stable"
 215 ]
 216
 217 missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"]
 218 allvars=portage.auxdbkeys
 219 commitmessage=None
 220 commitmessagefile=None
 221 for x in missingvars:
 222         x += ".missing"
 223         if x not in qacats:
 224                 print "* missingvars values need to be added to qahelp ("+x+")"
 225                 qacats.append(x)
 226                 qawarnings.append(x)
 227
 228
 229 verbose=0
 230 quiet=0
 231
 232 def show_version():
 233         print exename+" "+version
 234         sys.exit(0)
 235
 236 def help(exitstatus=1,helpfulness=1):
 237         if quiet:
 238                 helpfulness=0
 239         if helpfulness:
 240                 print
 241                 print green(exename+" "+version)
 242                 print " \"Quality is job zero.\""
 243                 print " Copyright 1999-2006 Gentoo Foundation"
 244                 print " Distributed under the terms of the GNU General Public License v2"
 245                 print
 246         print bold(" Usage:"),turquoise(exename),"[",green("options"),"] [",green("mode"),"]"
 247         if helpfulness:
 248                 print bold(" Modes:"),turquoise("scan (default)"),
 249                 for x in modes[1:]:
 250                         print "|",turquoise(x),
 251                 print
 252         print
 253         print " "+green("Options".ljust(20)+" Description")
 254         for x in options:
 255                 if repoman_shortoptions_rev.has_key(x):
 256                         shopt=repoman_shortoptions_rev[x]+", "+x
 257                 else:
 258                         shopt="    "+x
 259                 print " "+shopt.ljust(20),repoman_options[x]
 260         print
 261         print " "+green("Modes".ljust(20)+" Description")
 262         for x in modes:
 263                 print " "+x.ljust(20),modeshelp[x]
 264         if helpfulness:
 265                 print
 266                 print " "+green("QA keyword".ljust(20)+" Description")
 267                 for x in qacats:
 268                         print " "+x.ljust(20),qahelp[x]
 269                 print
 270         if (exitstatus != -1):
 271                 sys.exit(exitstatus)
 272         else:
 273                 print
 274
 275 def last():
 276         try:
 277                 #Retrieve and unpickle stats and fails from saved files
 278                 savedf=open('/var/cache/edb/repo.stats','r')
 279                 stats = pickle.load(savedf)
 280                 savedf.close()
 281                 savedf=open('/var/cache/edb/repo.fails','r')
 282                 fails = pickle.load(savedf)
 283                 savedf.close()
 284         except SystemExit, e:
 285                 raise  # Need to propogate this
 286         except:
 287                 err("Error retrieving last repoman run data; exiting.")
 288
 289         #dofail will be set to 1 if we have failed in at least one non-warning category
 290         dofail=0
 291         #dowarn will be set to 1 if we tripped any warnings
 292         dowarn=0
 293         #dofull will be set if we should print a "repoman full" informational message
 294         dofull=0
 295
 296         print
 297         print green("RepoMan remembers...")
 298         print
 299         for x in qacats:
 300                 if stats[x]:
 301                         dowarn=1
 302                         if x not in qawarnings:
 303                                 dofail=1
 304                 else:
 305                         if mymode!="lfull":
 306                                 continue
 307                 print "  "+ x.ljust(20),
 308                 if stats[x]==0:
 309                         print green(`stats[x]`)
 310                         continue
 311                 elif x in qawarnings:
 312                         print yellow(`stats[x]`)
 313                 else:
 314                         print red(`stats[x]`)
 315                 if mymode!="lfull":
 316                         if stats[x]<12:
 317                                 for y in fails[x]:
 318                                         print "   "+y
 319                         else:
 320                                 dofull=1
 321                 else:
 322                         for y in fails[x]:
 323                                 print "   "+y
 324         print
 325         if dofull:
 326                 print bold("Note: type \"repoman lfull\" for a complete listing of repomans last run.")
 327                 print
 328         if dowarn and not dofail:
 329                 print green("RepoMan sez:"),"\"You only gave me a partial QA payment last time?\n              I took it, but I wasn't happy.\""
 330         elif not dofail:
 331                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
 332         print
 333         sys.exit(1)
 334
 335 mymode=None
 336 myoptions=[]
 337 if len(sys.argv)>1:
 338         x=1
 339         while x < len(sys.argv):
 340                 if sys.argv[x] in shortmodes.keys():
 341                         sys.argv[x]=shortmodes[sys.argv[x]]
 342                 elif sys.argv[x] in repoman_shortoptions.keys():
 343                         sys.argv[x] = repoman_shortoptions[sys.argv[x]]
 344                 if sys.argv[x] in modes:
 345                         if mymode is None:
 346                                 mymode=sys.argv[x]
 347                         else:
 348                                 err("Please specify either \""+mymode+"\" or \""+sys.argv[x]+"\", but not both.")
 349                 elif sys.argv[x] in options:
 350                         optionx=sys.argv[x]
 351                         if (optionx=="--commitmsg") and (len(sys.argv)>=(x+1)):
 352                                 commitmessage=sys.argv[x+1]
 353                                 x=x+1
 354                         elif (optionx=="--commitmsgfile") and (len(sys.argv)>=(x+1)):
 355                                 commitmessagefile=sys.argv[x+1]
 356                                 x=x+1
 357                         elif (optionx=="--verbose"):
 358                                 verbose+=1
 359                         elif (optionx=="--quiet"):
 360                                 quiet+=1
 361                         elif optionx not in myoptions:
 362                                 myoptions.append(optionx)
 363                 else:
 364                         err_help("\""+sys.argv[x]+"\" is not a valid mode or option.")
 365                 x=x+1
 366 if mymode is None:
 367         mymode="scan"
 368 if mymode=="help" or ("--help" in myoptions):
 369         help(exitstatus=0)
 370 if ("--version" in myoptions):
 371         show_version()
 372 if mymode=="last" or (mymode=="lfull"):
 373         last()
 374 if mymode == "commit":
 375         while "--ignore-masked" in myoptions:
 376                 myoptions.remove("--ignore-masked")
 377
 378 from portage import normalize_path
 379 isCvs=False
 380 myreporoot=None
 381 if os.path.isdir("CVS"):
 382         isCvs = True
 383
 384 if mymode == "commit" and \
 385         not isCvs and \
 386         "--pretend" not in myoptions:
 387         print
 388         print darkgreen("Not in a CVS repository; enabling pretend mode.")
 389         myoptions.append("--pretend");
 390
 391
 392 def have_profile_dir(path, maxdepth=3):
 393         while path != "/" and maxdepth:
 394                 if os.path.exists(path + "/profiles/package.mask"):
 395                         return normalize_path(path)
 396                 path = normalize_path(path + "/..")
 397                 maxdepth -= 1
 398
 399 portdir=None
 400 portdir_overlay=None
 401 mydir=os.getcwd()
 402 if "PWD" in os.environ and os.environ["PWD"] != mydir and \
 403         os.path.realpath(os.environ["PWD"]) == mydir:
 404         # getcwd() returns the canonical path but that makes it hard for repoman to
 405         # orient itself if the user has symlinks in their portage tree structure.
 406         # We use os.environ["PWD"], if available, to get the non-canonical path of
 407         # the current working directory (from the shell).
 408         mydir = os.environ["PWD"]
 409 mydir = normalize_path(mydir)
 410 if mydir[-1] != "/":
 411         mydir += "/"
 412
 413 for overlay in repoman_settings["PORTDIR_OVERLAY"].split():
 414         if overlay[-1] != "/":
 415                 overlay += "/"
 416         if mydir.startswith(overlay):
 417                 portdir_overlay = overlay
 418                 subdir = mydir[len(overlay):]
 419                 if subdir and subdir[-1] != "/":
 420                         subdir += "/"
 421                 if have_profile_dir(mydir, subdir.count("/")):
 422                         portdir = portdir_overlay
 423                 break
 424
 425 if not portdir_overlay:
 426         if (repoman_settings["PORTDIR"] + os.path.sep).startswith(mydir):
 427                 portdir_overlay = repoman_settings["PORTDIR"]
 428         else:
 429                 portdir_overlay = have_profile_dir(mydir)
 430         portdir = portdir_overlay
 431
 432 if not portdir_overlay:
 433         sys.stderr.write("Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY from" + \
 434                 " the current\nworking directory.\n")
 435         sys.exit(1)
 436
 437 if not portdir:
 438         portdir = repoman_settings["PORTDIR"]
 439
 440 portdir = normalize_path(portdir)
 441 portdir_overlay = normalize_path(portdir_overlay)
 442
 443 os.environ["PORTDIR"] = portdir
 444 if portdir_overlay != portdir:
 445         os.environ["PORTDIR_OVERLAY"] = portdir_overlay
 446 else:
 447         os.environ["PORTDIR_OVERLAY"] = ""
 448
 449 if quiet < 2:
 450         print "\nSetting paths:"
 451         print "PORTDIR = \""+os.environ["PORTDIR"]+"\""
 452         print "PORTDIR_OVERLAY = \""+os.environ["PORTDIR_OVERLAY"]+"\""
 453
 454 # Now that PORTDIR_OVERLAY is properly overridden, create the portdb.
 455 repoman_settings = portage.config(local_config=False,
 456         config_incrementals=portage_const.INCREMENTALS)
 457 trees = portage.create_trees()
 458 trees["/"]["porttree"].settings = repoman_settings
 459 portdb = trees["/"]["porttree"].dbapi
 460 portdb.mysettings = repoman_settings
 461 # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman.
 462 del trees["/"]["vartree"]
 463
 464 if not myreporoot:
 465         myreporoot = os.path.basename(portdir_overlay)
 466         myreporoot += mydir[len(portdir_overlay):-1]
 467
 468 reposplit=myreporoot.split("/")
 469 repolevel=len(reposplit)
 470
 471 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 472 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 473 # this check ensure that repoman knows where it is, and the manifest recommit is at least possible.
 474 if mymode == "commit" and repolevel not in [1,2,3]:
 475         print red("***")+" Commit attempts *must* be from within a cvs co, category, or package directory."
 476         print red("***")+" Attempting to commit from a packages files directory will be blocked for instance."
 477         print red("***")+" This is intended behaviour, to ensure the manifest is recommited for a package."
 478         print red("***")
 479         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 480
 481 startdir=os.getcwd()
 482
 483 for x in range(0,repolevel-1):
 484         os.chdir("..")
 485 repodir=os.getcwd()
 486 os.chdir(startdir)
 487
 488 def caterror(mycat):
 489         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 490 if "--pretend" in myoptions:
 491         print green("\nRepoMan does a once-over of the neighborhood...")
 492 elif quiet < 1:
 493         print green("\nRepoMan scours the neighborhood...")
 494
 495 def parse_use_local_desc(mylines, usedict=None):
 496         """returns a dict of the form {cpv:set(flags)}"""
 497         if usedict is None:
 498                 usedict = {}
 499         lineno = 0
 500         for l in mylines:
 501                 lineno += 1
 502                 if not l or l.startswith("#"):
 503                         continue
 504                 mysplit = l.split(None, 1)
 505                 if not mysplit:
 506                         continue
 507                 mysplit = mysplit[0].split(":")
 508                 if len(mysplit) != 2:
 509                         raise ParseError("line %d: Malformed input: '%s'" % \
 510                                 (lineno, l.rstrip("\n")))
 511                 usedict.setdefault(mysplit[0], set())
 512                 usedict[mysplit[0]].add(mysplit[1])
 513         return usedict
 514
 515 # retreive local USE list
 516 luselist={}
 517 try:
 518         f = open(os.path.join(portdir, "profiles", "use.local.desc"))
 519         parse_use_local_desc(f, luselist)
 520         f.close()
 521 except (IOError, OSError, ParseError), e:
 522         print >> sys.stderr, str(e)
 523         err("Couldn't read from use.local.desc")
 524
 525 if portdir_overlay != portdir:
 526         filename = os.path.join(portdir_overlay, "profiles", "use.local.desc")
 527         if os.path.exists(filename):
 528                 try:
 529                         f = open(filename)
 530                         parse_use_local_desc(f, luselist)
 531                         f.close()
 532                 except (IOError, OSError, ParseError), e:
 533                         print >> sys.stderr, str(e)
 534                         err("Couldn't read from '%s'" % filename)
 535         del filename
 536
 537 # setup a uselist from portage
 538 uselist=[]
 539 try:
 540         uselist=portage.grabfile(portdir+"/profiles/use.desc")
 541         for l in range(0,len(uselist)):
 542                 uselist[l]=uselist[l].split()[0]
 543         for var in repoman_settings["USE_EXPAND"].split():
 544                 vardescs = portage.grabfile(portdir+"/profiles/desc/"+var.lower()+".desc")
 545                 for l in range(0, len(vardescs)):
 546                         uselist.append(var.lower() + "_" + vardescs[l].split()[0])
 547 except SystemExit, e:
 548         raise  # Need to propogate this
 549 except:
 550         err("Couldn't read USE flags from use.desc")
 551
 552 # retrieve a list of current licenses in portage
 553 liclist = set(portage.listdir(os.path.join(portdir, "licenses")))
 554 if not liclist:
 555         err("Couldn't find licenses?")
 556 if portdir_overlay != portdir:
 557         liclist.update(portage.listdir(os.path.join(portdir_overlay, "licenses")))
 558
 559 # retrieve list of offical keywords
 560 kwlist = set(portage.grabfile(os.path.join(portdir, "profiles", "arch.list")))
 561 if not kwlist:
 562         err("Couldn't read KEYWORDS from arch.list")
 563
 564 if portdir_overlay != portdir:
 565         kwlist.update(portage.grabfile(
 566                 os.path.join(portdir_overlay, "profiles", "arch.list")))
 567
 568 scanlist=[]
 569 if repolevel==2:
 570         #we are inside a category directory
 571         catdir=reposplit[-1]
 572         if catdir not in repoman_settings.categories:
 573                 caterror(catdir)
 574         mydirlist=os.listdir(startdir)
 575         for x in mydirlist:
 576                 if x == "CVS" or x.startswith("."):
 577                         continue
 578                 if os.path.isdir(startdir+"/"+x):
 579                         scanlist.append(catdir+"/"+x)
 580 elif repolevel==1:
 581         for x in repoman_settings.categories:
 582                 if not os.path.isdir(startdir+"/"+x):
 583                         continue
 584                 for y in os.listdir(startdir+"/"+x):
 585                         if y == "CVS" or y.startswith("."):
 586                                 continue
 587                         if os.path.isdir(startdir+"/"+x+"/"+y):
 588                                 scanlist.append(x+"/"+y)
 589 elif repolevel==3:
 590         catdir = reposplit[-2]
 591         if catdir not in repoman_settings.categories:
 592                 caterror(catdir)
 593         scanlist.append(catdir+"/"+reposplit[-1])
 594
 595 profiles={}
 596 descfile=portdir+"/profiles/profiles.desc"
 597 if os.path.exists(descfile):
 598         for x in portage.grabfile(descfile):
 599                 if x[0]=="#":
 600                         continue
 601                 arch=x.split()
 602                 if len(arch)!=3:
 603                         print "wrong format: \""+red(x)+"\" in "+descfile
 604                         continue
 605                 if not os.path.isdir(portdir+"/profiles/"+arch[1]):
 606                         print "Invalid "+arch[2]+" profile ("+arch[1]+") for arch "+arch[0]
 607                         continue
 608                 if profiles.has_key(arch[0]):
 609                         profiles[arch[0]]+= [[arch[1], arch[2]]]
 610                 else:
 611                         profiles[arch[0]] = [[arch[1], arch[2]]]
 612
 613         for x in repoman_settings.archlist():
 614                 if x[0] == "~":
 615                         continue
 616                 if not profiles.has_key(x):
 617                         print red("\""+x+"\" doesn't have a valid profile listed in profiles.desc.")
 618                         print red("You need to either \"cvs update\" your profiles dir or follow this")
 619                         print red("up with the "+x+" team.")
 620                         print
 621 else:
 622         print red("profiles.desc does not exist: "+descfile)
 623         print red("You need to do \"cvs update\" in profiles dir.")
 624         print
 625         sys.exit(1)
 626
 627
 628 stats={}
 629 fails={}
 630
 631 for x in qacats:
 632         stats[x]=0
 633         fails[x]=[]
 634 xmllint_capable = False
 635 if getstatusoutput('which xmllint')[0] != 0:
 636         print red("!!! xmllint not found. Can't check metadata.xml.\n")
 637         if "--xmlparse" in myoptions or repolevel==3:
 638                 print red("!!!")+" sorry, xmllint is needed.  failing\n"
 639                 sys.exit(1)
 640 else:
 641         #hardcoded paths/urls suck. :-/
 642         must_fetch=1
 643         backup_exists=0
 644         try:
 645                 # if it's been over a week since fetching (or the system clock is fscked), grab an updated copy of metadata.dtd
 646                 # clock is fscked or it's been a week. time to grab a new one.
 647                 ct=os.stat(portage.CACHE_PATH + '/metadata.dtd')[ST_CTIME]
 648                 if abs(time.time() - ct) > (60*60*24*7):
 649                         # don't trap the exception, we're watching for errno 2 (file not found), anything else is a bug.
 650                         backup_exists=1
 651                 else:
 652                         must_fetch=0
 653
 654         except (OSError,IOError), e:
 655                 if e.errno != 2:
 656                         print red("!!!")+" caught exception '%s' for %s/metadata.dtd, bailing" % (str(e), portage.CACHE_PATH)
 657                         sys.exit(1)
 658
 659         if must_fetch:
 660                 print
 661                 print green("***")+" the local copy of metadata.dtd needs to be refetched, doing that now"
 662                 print
 663                 try:
 664                         if os.path.exists(repoman_settings["DISTDIR"]+'/metadata.dtd'):
 665                                 os.remove(repoman_settings["DISTDIR"]+'/metadata.dtd')
 666                         val=portage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoman_settings,fetchonly=0, \
 667                                 try_mirrors=0)
 668                         if val:
 669                                 if backup_exists:
 670                                         os.remove(portage.CACHE_PATH+'/metadata.dtd')
 671                                 shutil.copy(repoman_settings["DISTDIR"]+'/metadata.dtd',portage.CACHE_PATH+'/metadata.dtd')
 672                                 os.chown(portage.CACHE_PATH+'/metadata.dtd',os.getuid(),portage.portage_gid)
 673                                 os.chmod(portage.CACHE_PATH+'/metadata.dtd',0664)
 674
 675
 676                 except SystemExit, e:
 677                         raise  # Need to propogate this
 678                 except Exception,e:
 679                         print
 680                         print red("!!!")+" attempting to fetch 'http://www.gentoo.org/dtd/metadata.dtd', caught"
 681                         print red("!!!")+" exception '%s' though." % str(e)
 682                         val=0
 683                 if not val:
 684                         print red("!!!")+" fetching new metadata.dtd failed, aborting"
 685                         sys.exit(1)
 686         #this can be problematic if xmllint changes their output
 687         xmllint_capable=True
 688
 689
 690 def x11_deprecation_check(depstr):
 691         if depstr.find("virtual/x11") == -1:
 692                 return False
 693         depsplit = depstr.split()
 694         ok_stack = []
 695         ok = False
 696         for token in depsplit:
 697                 if token == "(":
 698                         ok_stack.append(ok)
 699                         ok = False
 700                 elif token == ")":
 701                         ok = ok_stack.pop()
 702                 elif token == "||":
 703                         ok = True
 704                 else:
 705                         ok = False
 706                         if token.find("virtual/x11") != -1 and (not ok_stack or not ok_stack[-1]):
 707                                 return True
 708         return False
 709
 710
 711 arch_caches={}
 712 arch_xmatch_caches = {}
 713 for x in scanlist:
 714         #ebuilds and digests added to cvs respectively.
 715         if verbose:
 716                 print "checking package " + x
 717         eadded=[]
 718         dadded=[]
 719         catdir,pkgdir=x.split("/")
 720         checkdir=repodir+"/"+x
 721         checkdirlist=os.listdir(checkdir)
 722         ebuildlist=[]
 723         for y in checkdirlist:
 724                 if y[-7:]==".ebuild":
 725                         ebuildlist.append(y[:-7])
 726                 if y in ["Manifest","ChangeLog","metadata.xml"]:
 727                         if os.stat(checkdir+"/"+y)[0] & 0x0248:
 728                                 stats["file.executable"] += 1
 729                                 fails["file.executable"].append(checkdir+"/"+y)
 730         digestlist=[]
 731
 732         for y in checkdirlist:
 733                 for c in y.strip(os.path.sep):
 734                         if c not in allowed_filename_chars_set:
 735                                 stats["file.name"] += 1
 736                                 fails["file.name"].append("%s/%s: char '%s'" % (checkdir, y, c))
 737                                 break
 738
 739                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
 740                         continue
 741                 try:
 742                         line = 1
 743                         for l in codecs.open(checkdir+"/"+y, "r", "utf8"):
 744                                 line +=1
 745                 except UnicodeDecodeError, ue:
 746                         stats["file.UTF8"] += 1
 747                         s = ue.object[:ue.start]
 748                         l2 = s.count("\n")
 749                         line += l2
 750                         if l2 != 0:
 751                                 s = s[s.rfind("\n") + 1:]
 752                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
 753
 754         if isCvs:
 755                 try:
 756                         mystat=os.stat(checkdir+"/files")[0]
 757                         if len(ebuildlist) and not S_ISDIR(mystat):
 758                                 raise Exception
 759                 except SystemExit, e:
 760                         raise  # Need to propogate this
 761                 except:
 762                         stats["filedir.missing"] += 1
 763                         fails["filedir.missing"].append(checkdir)
 764                         continue
 765                 try:
 766                         myf=open(checkdir+"/CVS/Entries","r")
 767                         myl=myf.readlines()
 768                         for l in myl:
 769                                 if l[0]!="/":
 770                                         continue
 771                                 splitl=l[1:].split("/")
 772                                 if not len(splitl):
 773                                         continue
 774                                 if splitl[0][-7:]==".ebuild":
 775                                         eadded.append(splitl[0][:-7])
 776                 except IOError:
 777                         if mymode=="commit":
 778                                 stats["CVS/Entries.IO_error"] += 1
 779                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
 780                         continue
 781
 782                 try:
 783                         myf=open(checkdir+"/files/CVS/Entries","r")
 784                         myl=myf.readlines()
 785                         for l in myl:
 786                                 if l[0]!="/":
 787                                         continue
 788                                 splitl=l[1:].split("/")
 789                                 if not len(splitl):
 790                                         continue
 791                                 if splitl[0][:7]=="digest-":
 792                                         dadded.append(splitl[0][7:])
 793                 except IOError:
 794                         if mymode=="commit":
 795                                 stats["CVS/Entries.IO_error"] += 1
 796                                 fails["CVS/Entries.IO_error"].append(checkdir+"/files/CVS/Entries")
 797                         continue
 798
 799         if mymode in ("fix", "commit") and \
 800                 "--pretend" not in myoptions:
 801                 repoman_settings["O"] = checkdir
 802                 if not portage.digestgen([], repoman_settings, myportdb=portdb):
 803                         print "Unable to generate manifest."
 804                         sys.exit(1)
 805
 806         mf = Manifest(checkdir, repoman_settings["DISTDIR"])
 807         mydigests=mf.getTypeDigests("DIST")
 808
 809         if os.path.exists(checkdir+"/files"):
 810                 filesdirlist=os.listdir(checkdir+"/files")
 811                 for y in filesdirlist:
 812                         if y[:7]=="digest-":
 813                                 if y[7:] not in dadded:
 814                                         #digest not added to cvs
 815                                         stats["digest.notadded"]=stats["digest.notadded"]+1
 816                                         fails["digest.notadded"].append(x+"/files/"+y)
 817                                         if y[7:] in eadded:
 818                                                 stats["digest.disjointed"]=stats["digest.disjointed"]+1
 819                                                 fails["digest.disjointed"].append(x+"/files/"+y)
 820
 821                                 if os.stat(checkdir+"/files/"+y)[0] & 0x0248:
 822                                         stats["file.executable"] += 1
 823                                         fails["file.executable"].append(x+"/files/"+y)
 824
 825                                 mykey = catdir + "/" + y[7:]
 826                                 if y[7:] not in ebuildlist:
 827                                         #stray digest
 828                                         if mymode=="fix":
 829                                                 if "--pretend" in myoptions:
 830                                                         print "(cd "+repodir+"/"+x+"/files; cvs rm -f "+y+")"
 831                                                 else:
 832                                                         os.system("(cd "+repodir+"/"+x+"/files; cvs rm -f "+y+")")
 833                                         else:
 834                                                 stats["digest.stray"]=stats["digest.stray"]+1
 835                                                 fails["digest.stray"].append(x+"/files/"+y)
 836                                 else:
 837                                         # We have an ebuild
 838                                         myuris, myfiles = portdb.getfetchlist(mykey, all=True)
 839
 840                                         uri_dict = {}
 841                                         for myu in myuris:
 842                                                 myubn = os.path.basename(myu)
 843                                                 if myubn not in uri_dict:
 844                                                         uri_dict[myubn]  = [myu]
 845                                                 else:
 846                                                         uri_dict[myubn] += [myu]
 847
 848                                         for myf in uri_dict:
 849                                                 myff = repoman_settings["DISTDIR"] + "/" + myf
 850                                                 if not mydigests.has_key(myf):
 851                                                         uri_settings = portage.config(clone=repoman_settings)
 852                                                         if mymode == "fix":
 853                                                                 if not portage.fetch(uri_dict[myf], uri_settings):
 854                                                                         stats["digest.unmatch"] += 1
 855                                                                         fails["digest.unmatch"].append(y+"::"+myf)
 856                                                                 else:
 857                                                                         eb_name = portdb.findname2(mykey)[0]
 858                                                                         portage.doebuild(eb_name, "digest", "/",
 859                                                                                 uri_settings, tree="porttree",
 860                                                                                 mydbapi=portdb)
 861                                                         else:
 862                                                                 stats["digest.partial"] += 1
 863                                                                 fails["digest.partial"].append(y+"::"+myf)
 864                                                 elif "assume-digests" not in repoman_settings.features:
 865                                                         if os.path.exists(myff):
 866                                                                 if not portage_checksum.verify_all(myff, mydigests[myf]):
 867                                                                         stats["digest.fail"] += 1
 868                                                                         fails["digest.fail"].append(y+"::"+myf)
 869                                                         elif repolevel == 3:
 870                                                                 stats["digest.assumed"] += 1
 871                                                                 fails["digest.assumed"].append(y+"::"+myf)
 872
 873                 # recurse through files directory
 874                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
 875                 while filesdirlist:
 876                         y = filesdirlist.pop(0)
 877                         try:
 878                                 mystat = os.stat(checkdir+"/files/"+y)
 879                         except OSError, oe:
 880                                 if oe.errno == 2:
 881                                         # don't worry about it.  it likely was removed via fix above.
 882                                         continue
 883                                 else:
 884                                         raise oe
 885                         if S_ISDIR(mystat.st_mode):
 886                                 if y == "CVS":
 887                                         continue
 888                                 for z in os.listdir(checkdir+"/files/"+y):
 889                                         if z == "CVS":
 890                                                 continue
 891                                         filesdirlist.append(y+"/"+z)
 892                         # current policy is no files over 20k, this is the check.
 893                         elif mystat.st_size > 20480:
 894                                 stats["file.size"] += 1
 895                                 fails["file.size"].append("("+ str(mystat.st_size/1024) + "K) "+x+"/files/"+y)
 896
 897                         for c in os.path.basename(y.rstrip(os.path.sep)):
 898                                 if c not in allowed_filename_chars_set:
 899                                         stats["file.name"] += 1
 900                                         fails["file.name"].append("%s/files/%s: char '%s'" % (checkdir, y, c))
 901                                         break
 902
 903         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
 904         myfiles_all = []
 905         for myfiles in fetchlist_dict.itervalues():
 906                 myfiles_all.extend(myfiles)
 907                 del myfiles
 908         del fetchlist_dict
 909         myfiles_all = set(myfiles_all)
 910         for entry in mydigests:
 911                 if entry not in myfiles_all:
 912                         stats["digestentry.unused"] += 1
 913                         fails["digestentry.unused"].append(checkdir+"::"+entry)
 914         del mydigests, myfiles_all
 915
 916         if "ChangeLog" not in checkdirlist:
 917                 stats["changelog.missing"]+=1
 918                 fails["changelog.missing"].append(x+"/ChangeLog")
 919
 920         #metadata.xml file check
 921         if "metadata.xml" not in checkdirlist:
 922                 stats["metadata.missing"]+=1
 923                 fails["metadata.missing"].append(x+"/metadata.xml")
 924         #metadata.xml parse check
 925         else:
 926                 #Only carry out if in package directory or check forced
 927                 if xmllint_capable:
 928                         st=getstatusoutput("xmllint --nonet --noout --dtdvalid %s/metadata.dtd %s/metadata.xml" % (portage.CACHE_PATH, checkdir))
 929                         if st[0] != 0:
 930                                 for z in st[1].split("\n"):
 931                                         print red("!!! ")+z
 932                                 stats["metadata.bad"]+=1
 933                                 fails["metadata.bad"].append(x+"/metadata.xml")
 934
 935         allmasked = True
 936
 937         for y in ebuildlist:
 938                 if os.stat(checkdir+"/"+y+".ebuild")[0] & 0x0248:
 939                         stats["file.executable"] += 1
 940                         fails["file.executable"].append(x+"/"+y+".ebuild")
 941                 if y not in eadded:
 942                         #ebuild not added to cvs
 943                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
 944                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
 945                         if y in dadded:
 946                                 stats["ebuild.disjointed"]=stats["ebuild.disjointed"]+1
 947                                 fails["ebuild.disjointed"].append(x+"/"+y+".ebuild")
 948                 if not os.path.exists(checkdir+"/files/digest-"+y):
 949                         if mymode=="fix":
 950                                 if "--pretend" in myoptions:
 951                                         print "You will need to run:"
 952                                         print "  /usr/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest"
 953                                 else:
 954                                         retval=os.system("/usr/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest")
 955                                         if retval:
 956                                                 print "!!! Exiting on ebuild digest (shell) error code:",retval
 957                                                 sys.exit(retval)
 958                         else:
 959                                 stats["digest.missing"]=stats["digest.missing"]+1
 960                                 fails["digest.missing"].append(x+"/files/digest-"+y)
 961                 myesplit=portage.pkgsplit(y)
 962                 if myesplit is None or myesplit[0] != x.split("/")[-1]:
 963                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
 964                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
 965                         continue
 966                 elif myesplit[0]!=pkgdir:
 967                         print pkgdir,myesplit[0]
 968                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
 969                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
 970                         continue
 971                 try:
 972                         myaux = dict(
 973                                 zip(allvars, portdb.aux_get(os.path.join(catdir, y), allvars)))
 974                 except KeyError:
 975                         stats["ebuild.syntax"]=stats["ebuild.syntax"]+1
 976                         fails["ebuild.syntax"].append(x+"/"+y+".ebuild")
 977                         continue
 978                 except IOError:
 979                         stats["ebuild.output"]=stats["ebuild.output"]+1
 980                         fails["ebuild.output"].append(x+"/"+y+".ebuild")
 981                         continue
 982
 983                 # Test for negative logic and bad words in the RESTRICT var.
 984                 #for x in myaux[allvars.index("RESTRICT")].split():
 985                 #       if x.startswith("no"):
 986                 #               print "Bad RESTRICT value: %s" % x
 987
 988                 myaux["PROVIDE"] = portage_dep.use_reduce(portage_dep.paren_reduce(myaux["PROVIDE"]), matchall=1)
 989                 myaux["PROVIDE"] = " ".join(portage.flatten(myaux["PROVIDE"]))
 990                 for myprovide in myaux["PROVIDE"].split():
 991                         prov_cp = portage.dep_getkey(myprovide)
 992                         if prov_cp != myprovide:
 993                                 stats["virtual.versioned"]+=1
 994                                 fails["virtual.versioned"].append(x+"/"+y+".ebuild: "+myprovide)
 995                         prov_pkg = portage.dep_getkey(
 996                                 portage.best(portdb.xmatch("match-all", prov_cp)))
 997                         if prov_cp == prov_pkg:
 998                                 stats["virtual.exists"]+=1
 999                                 fails["virtual.exists"].append(x+"/"+y+".ebuild: "+prov_cp)
1000
1001                 for pos in range(0,len(missingvars)):
1002                         if not myaux[missingvars[pos]]:
1003                                 myqakey=missingvars[pos]+".missing"
1004                                 stats[myqakey]=stats[myqakey]+1
1005                                 fails[myqakey].append(x+"/"+y+".ebuild")
1006
1007                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1008                 if "-*" in myaux["KEYWORDS"].split():
1009                         haskeyword = False
1010                         for kw in myaux["KEYWORDS"].split():
1011                                 if kw[0] == "~":
1012                                         kw = kw[1:]
1013                                 if kw in kwlist:
1014                                         haskeyword = True
1015                         if not haskeyword:
1016                                 stats["KEYWORDS.stupid"] += 1
1017                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1018
1019                 """
1020                 Ebuilds that inherit a "Live" eclasss (darcs,subversion,git,cvs,etc..) should
1021                 not be allowed to be marked stable
1022                 """
1023                 if set(["darcs","cvs","subversion","git"]).intersection(
1024                         myaux["INHERITED"].split()):
1025                         bad_stable_keywords = []
1026                         for keyword in myaux["KEYWORDS"].split():
1027                                 if not keyword.startswith("~") and \
1028                                         not keyword.startswith("-"):
1029                                         bad_stable_keywords.append(keyword)
1030                                 del keyword
1031                         if bad_stable_keywords:
1032                                 stats["LIVEVCS.stable"] += 1
1033                                 fails["LIVEVCS.stable"].append(
1034                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1035                                                 bad_stable_keywords)
1036                         del bad_stable_keywords
1037
1038                 if "--ignore-arches" in myoptions:
1039                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1040                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1041                 else:
1042                         arches=[]
1043                         for keyword in myaux["KEYWORDS"].split():
1044                                 if (keyword[0]=="-"):
1045                                         continue
1046                                 elif (keyword[0]=="~"):
1047                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1048                                 else:
1049                                         arches.append([keyword, keyword, [keyword]])
1050                                         allmasked = False
1051
1052                 baddepsyntax = False
1053                 badlicsyntax = False
1054                 badprovsyntax = False
1055                 catpkg = catdir+"/"+y
1056                 myiuse = set(repoman_settings.archlist())
1057                 for myflag in myaux["IUSE"].split():
1058                         if myflag.startswith("+"):
1059                                 myflag = myflag[1:]
1060                         myiuse.add(myflag)
1061
1062                 operator_tokens = set(["||", "(", ")"])
1063                 type_list, badsyntax = [], []
1064                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND", "LICENSE", "PROVIDE"):
1065                         mydepstr = myaux[mytype]
1066
1067                         if mydepstr.find(" ?") != -1:
1068                                 badsyntax.append("'?' preceded by space")
1069
1070                         try:
1071                                 # Missing closing parenthesis will result in a ValueError
1072                                 mydeplist = portage_dep.paren_reduce(mydepstr)
1073                                 # Missing opening parenthesis will result in a final "" element
1074                                 if "" in mydeplist or "(" in mydeplist:
1075                                         raise ValueError
1076                         except ValueError:
1077                                 badsyntax.append("parenthesis mismatch")
1078                                 mydeplist = []
1079
1080                         try:
1081                                 portage_dep.use_reduce(mydeplist, excludeall=myiuse)
1082                         except portage_exception.InvalidDependString, e:
1083                                 badsyntax.append(str(e))
1084
1085                         for token in operator_tokens:
1086                                 if mydepstr.startswith(token+" "):
1087                                         myteststr = mydepstr[len(token):]
1088                                 else:
1089                                         myteststr = mydepstr
1090                                 if myteststr.endswith(" "+token):
1091                                         myteststr = myteststr[:-len(token)]
1092                                 while myteststr.find(" "+token+" ") != -1:
1093                                         myteststr = " ".join(myteststr.split(" "+token+" ", 1))
1094                                 if myteststr.find(token) != -1:
1095                                         badsyntax.append("'%s' not separated by space" % (token))
1096
1097
1098                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1099                                 for token in mydepstr.split():
1100                                         if token in operator_tokens or \
1101                                                 token.endswith("?"):
1102                                                 continue
1103                                         if not portage.isvalidatom(token, allow_blockers=True) or \
1104                                                 ":" in token and myaux["EAPI"] == "0":
1105                                                 badsyntax.append("'%s' not a valid atom" % token)
1106
1107                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1108
1109                 for m,b in zip(type_list, badsyntax):
1110                         stats[m+".syntax"] += 1
1111                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1112
1113                 badlicsyntax = len(filter(lambda x:x=="LICENSE", type_list))
1114                 badprovsyntax = len(filter(lambda x:x=="PROVIDE", type_list))
1115                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1116                 badlicsyntax = badlicsyntax > 0
1117                 badprovsyntax = badprovsyntax > 0
1118
1119                 if not baddepsyntax:
1120                         if x11_deprecation_check(" ".join([myaux["DEPEND"], myaux["RDEPEND"], myaux["PDEPEND"]])):
1121                                 stats["usage.obsolete"] += 1
1122                                 fails["usage.obsolete"].append("%s/%s.ebuild: not migrated to modular X" % (x, y))
1123
1124                 for keyword,arch,groups in arches:
1125
1126                         if not profiles.has_key(arch):
1127                                 # A missing profile will create an error further down
1128                                 # during the KEYWORDS verification.
1129                                 continue
1130
1131                         for prof in profiles[arch]:
1132
1133                                 profdir = portdir+"/profiles/"+prof[0]
1134
1135                                 if prof[0] in arch_caches:
1136                                         dep_settings = arch_caches[prof[0]]
1137                                 else:
1138                                         dep_settings = portage.config(
1139                                                 config_profile_path=profdir,
1140                                                 config_incrementals=portage_const.INCREMENTALS,
1141                                                 local_config=False)
1142                                         arch_caches[prof[0]] = dep_settings
1143                                         while True:
1144                                                 try:
1145                                                         # Protect ACCEPT_KEYWORDS from config.regenerate()
1146                                                         # (just in case)
1147                                                         dep_settings.incrementals.remove("ACCEPT_KEYWORDS")
1148                                                 except ValueError:
1149                                                         break
1150
1151                                 xmatch_cache_key = (prof[0], tuple(groups))
1152                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
1153                                 if xcache is None:
1154                                         portdb.melt()
1155                                         portdb.freeze()
1156                                         xcache = portdb.xcache
1157                                         arch_xmatch_caches[xmatch_cache_key] = xcache
1158
1159                                 trees["/"]["porttree"].settings = dep_settings
1160                                 portdb.mysettings = dep_settings
1161                                 portdb.xcache = xcache
1162                                 # for package.use.mask support inside dep_check
1163                                 dep_settings.setcpv("/".join((catdir, y)))
1164                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1165                                 # just in case, prevent config.reset() from nuking these.
1166                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1167
1168                                 for myprovide in myaux["PROVIDE"].split():
1169                                         prov_cp = portage.dep_getkey(myprovide)
1170                                         if prov_cp not in dep_settings.getvirtuals():
1171                                                 stats["virtual.unavailable"]+=1
1172                                                 fails["virtual.unavailable"].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+prov_cp)
1173
1174                                 if not baddepsyntax:
1175                                         ismasked = os.path.join(catdir, y) not in \
1176                                                 portdb.xmatch("list-visible", x)
1177                                         if ismasked:
1178                                                 if "--ignore-masked" in myoptions:
1179                                                         continue
1180                                                 #we are testing deps for a masked package; give it some lee-way
1181                                                 suffix="masked"
1182                                                 matchmode="match-all"
1183                                         else:
1184                                                 suffix=""
1185                                                 matchmode="match-visible"
1186
1187                                         if prof[1] == "dev":
1188                                                 suffix=suffix+"indev"
1189
1190                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1191
1192                                                 mykey=mytype+".bad"+suffix
1193                                                 myvalue = myaux[mytype]
1194                                                 if not myvalue:
1195                                                         continue
1196                                                 try:
1197                                                         mydep = portage.dep_check(myvalue, portdb,
1198                                                                 dep_settings, use="all", mode=matchmode,
1199                                                                 trees=trees)
1200                                                 except KeyError, e:
1201                                                         stats[mykey]=stats[mykey]+1
1202                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(e[0]))
1203                                                         continue
1204
1205                                                 if mydep[0]==1:
1206                                                         if mydep[1]!=[]:
1207                                                                 #we have some unsolvable deps
1208                                                                 #remove ! deps, which always show up as unsatisfiable
1209                                                                 d=0
1210                                                                 while d<len(mydep[1]):
1211                                                                         if mydep[1][d][0]=="!":
1212                                                                                 del mydep[1][d]
1213                                                                         else:
1214                                                                                 d += 1
1215                                                                 #if we emptied out our list, continue:
1216                                                                 if not mydep[1]:
1217                                                                         continue
1218                                                                 stats[mykey]=stats[mykey]+1
1219                                                                 fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1220                                                 else:
1221                                                         stats[mykey]=stats[mykey]+1
1222                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1223
1224                 # this check needs work, it won't catch (\ndie)
1225                 if not os.system("egrep '^[^#]*\([^)]*\<die\>' "+checkdir+"/"+y+".ebuild >/dev/null 2>&1"):
1226                         stats["ebuild.nesteddie"]=stats["ebuild.nesteddie"]+1
1227                         fails["ebuild.nesteddie"].append(x+"/"+y+".ebuild")
1228                 # uselist checks - global
1229                 myuse = []
1230                 default_use = []
1231                 for myflag in myaux["IUSE"].split():
1232                         if myflag.startswith("+"):
1233                                 default_use.append(myflag)
1234                                 myflag = myflag[1:]
1235                         myuse.append(myflag)
1236                 for mypos in range(len(myuse)-1,-1,-1):
1237                         if myuse[mypos] and (myuse[mypos] in uselist):
1238                                 del myuse[mypos]
1239                 # uselist checks - local
1240                 mykey = portage.dep_getkey(catpkg)
1241                 if luselist.has_key(mykey):
1242                         for mypos in range(len(myuse)-1,-1,-1):
1243                                 if myuse[mypos] and (myuse[mypos] in luselist[mykey]):
1244                                         del myuse[mypos]
1245                 if default_use and myaux["EAPI"] == "0":
1246                         myuse += default_use
1247                 for mypos in range(len(myuse)):
1248                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1249                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1250
1251                 # license checks
1252                 if not badlicsyntax:
1253                         myuse = myaux["LICENSE"]
1254                         # Parse the LICENSE variable, remove USE conditions and
1255                         # flatten it.
1256                         myuse=portage_dep.use_reduce(portage_dep.paren_reduce(myuse), matchall=1)
1257                         myuse=portage.flatten(myuse)
1258                         # Check each entry to ensure that it exists in PORTDIR's
1259                         # license directory.
1260                         for mypos in range(0,len(myuse)):
1261                                 # Need to check for "||" manually as no portage
1262                                 # function will remove it without removing values.
1263                                 if myuse[mypos] not in liclist and myuse[mypos] != "||":
1264                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1265                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1266
1267                 #keyword checks
1268                 myuse = myaux["KEYWORDS"].split()
1269                 for mykey in myuse:
1270                         myskey=mykey[:]
1271                         if myskey[0]=="-":
1272                                 myskey=myskey[1:]
1273                         if myskey[0]=="~":
1274                                 myskey=myskey[1:]
1275                         if mykey!="-*":
1276                                 if myskey not in kwlist:
1277                                         stats["KEYWORDS.invalid"] += 1
1278                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1279                                 elif not profiles.has_key(myskey):
1280                                         stats["KEYWORDS.invalid"] += 1
1281                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1282
1283                 #syntax checks
1284                 myear = time.gmtime(os.stat(checkdir+"/"+y+".ebuild")[ST_MTIME])[0]
1285                 gentoo_copyright = re.compile(r'^# Copyright ((1999|200\d)-)?' + str(myear) + r' Gentoo Foundation')
1286                 gentoo_license = re.compile(r'^# Distributed under the terms of the GNU General Public License v2$')
1287                 cvs_header = re.compile(r'^#\s*\$Header.*\$$')
1288                 ignore_line = re.compile(r'(^$)|(^(\t)*#)')
1289                 leading_spaces = re.compile(r'^[\S\t]')
1290                 trailing_whitespace = re.compile(r'.*([\S]$)')
1291                 readonly_assignment = re.compile(r'^\s*(export\s+)?(A|CATEGORY|P|PV|PN|PR|PVR|PF|D|WORKDIR|FILESDIR|FEATURES|USE)=')
1292                 line_continuation = re.compile(r'([^#]*\S)(\s+|\t)\\$')
1293                 linenum=0
1294                 previous_line = None
1295                 for line in input(checkdir+"/"+y+".ebuild"):
1296                         linenum += 1
1297                         # Gentoo copyright check
1298                         if linenum == 1:
1299                                 match = gentoo_copyright.match(line)
1300                                 if not match:
1301                                         myerrormsg = "Copyright header Error. Possibly date related."
1302                                         stats["ebuild.badheader"] +=1
1303                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1304                         # Gentoo license check
1305                         elif linenum == 2:
1306                                 match = gentoo_license.match(line)
1307                                 if not match:
1308                                         myerrormsg = "Gentoo License Error."
1309                                         stats["ebuild.badheader"] +=1
1310                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1311                         # CVS Header check
1312                         elif linenum == 3:
1313                                 match = cvs_header.match(line)
1314                                 if not match:
1315                                         myerrormsg = "CVS Header Error."
1316                                         stats["ebuild.badheader"] +=1
1317                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1318                         else:
1319                                 match = ignore_line.match(line)
1320                                 if not match:
1321                                         # Excluded Blank lines and full line comments. Good!
1322                                         # Leading Spaces Check
1323                                         match = leading_spaces.match(line)
1324                                         if not match:
1325                                                 #Line has got leading spaces. Bad!
1326                                                 myerrormsg = "Leading Space Syntax Error. Line %d" % linenum
1327                                                 stats["ebuild.minorsyn"] +=1
1328                                                 fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1329                                         # Trailing whitespace check
1330                                         match = trailing_whitespace.match(line)
1331                                         if not match:
1332                                                 #Line has got trailing whitespace. Bad!
1333                                                 myerrormsg = "Trailing whitespace Syntax Error. Line %d" % linenum
1334                                                 stats["ebuild.minorsyn"] +=1
1335                                                 fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1336                                         # Readonly variable assignment check
1337                                         match = readonly_assignment.match(line)
1338                                         # The regex can give a false positive for continued lines,
1339                                         # so we check the previous line to see if it was continued.
1340                                         if match and (not previous_line or not line_continuation.match(previous_line)):
1341                                                 # invalid assignment, very bad!
1342                                                 myerrormsg = "Readonly variable assignment to %s on line %d" % (match.group(2), linenum)
1343                                                 stats["variable.readonly"] += 1
1344                                                 fails["variable.readonly"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1345                         previous_line = line
1346                 del previous_line
1347
1348         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1349         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1350         #if not portage.portdb.xmatch("bestmatch-visible",x):
1351         #    stats["ebuild.nostable"]+=1
1352         #    fails["ebuild.nostable"].append(x)
1353         if allmasked and repolevel == 3:
1354                 stats["ebuild.allmasked"]+=1
1355                 fails["ebuild.allmasked"].append(x)
1356
1357 #Pickle and save results for instant reuse in last and lfull
1358 if os.access(portage_const.CACHE_PATH, os.W_OK):
1359         for myobj, fname in (stats, "repo.stats"), (fails, "repo.fails"):
1360                 fpath = os.path.join(portage_const.CACHE_PATH, fname)
1361                 savef = open(fpath, 'w')
1362                 pickle.dump(myobj, savef)
1363                 savef.close()
1364                 portage.apply_secpass_permissions(fpath, gid=portage.portage_gid,
1365                         mode=0664)
1366 if quiet < 2:
1367         print
1368 #dofail will be set to 1 if we have failed in at least one non-warning category
1369 dofail=0
1370 #dowarn will be set to 1 if we tripped any warnings
1371 dowarn=0
1372 #dofull will be set if we should print a "repoman full" informational message
1373 dofull=0
1374 for x in qacats:
1375         if not isCvs and x.find("notadded") != -1:
1376                 stats[x] = 0
1377         if stats[x]:
1378                 dowarn=1
1379                 if x not in qawarnings:
1380                         dofail=1
1381         else:
1382                 continue
1383         print "  "+x.ljust(30),
1384         if stats[x]==0:
1385                 print green(`stats[x]`)
1386                 continue
1387         elif x in qawarnings:
1388                 print yellow(`stats[x]`)
1389         else:
1390                 print red(`stats[x]`)
1391         if mymode!="full":
1392                 if stats[x]<12:
1393                         for y in fails[x]:
1394                                 print "   "+y
1395                 else:
1396                         dofull=1
1397         else:
1398                 for y in fails[x]:
1399                         print "   "+y
1400
1401 def grouplist(mylist,seperator="/"):
1402         """(list,seperator="/") -- Takes a list of elements; groups them into
1403         same initial element categories. Returns a dict of {base:[sublist]}
1404         From: ["blah/foo","spork/spatula","blah/weee/splat"]
1405         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
1406         mygroups={}
1407         for x in mylist:
1408                 xs=x.split(seperator)
1409                 if xs[0]==".":
1410                         xs=xs[1:]
1411                 if xs[0] not in mygroups.keys():
1412                         mygroups[xs[0]]=[seperator.join(xs[1:])]
1413                 else:
1414                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
1415         return mygroups
1416
1417 if mymode!="commit":
1418         if dofull:
1419                 print bold("Note: type \"repoman full\" for a complete listing.")
1420                 if quiet < 1:
1421                         print
1422         if dowarn and not dofail:
1423                 if quiet < 2:
1424                         print green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\""
1425                 else:
1426                         print green("RepoMan sez:"),"\"OK for now, but I'll be back ...\""
1427         elif not dofail:
1428                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
1429         if quiet < 1:
1430                 print
1431 else:
1432         if dofail:
1433                 print turquoise("Please fix these important QA issues first.")
1434                 print green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n"
1435                 sys.exit(1)
1436
1437         if "--pretend" in myoptions:
1438                 print green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n"
1439
1440         if fails["digest.missing"]:
1441                 print green("Creating missing digests...")
1442         for x in fails["digest.missing"]:
1443                 xs=x.split("/")
1444                 del xs[-2]
1445                 myeb="/".join(xs[:-1])+"/"+xs[-1][7:]
1446                 if "--pretend" in myoptions:
1447                         print "(ebuild "+portdir+"/"+myeb+".ebuild digest)"
1448                 else:
1449                         retval=os.system("ebuild "+portdir+"/"+myeb+".ebuild digest")
1450                         if retval:
1451                                 print "!!! Exiting on ebuild digest (shell) error code:",retval
1452                                 sys.exit(retval)
1453
1454         mycvstree=cvstree.getentries("./",recursive=1)
1455         if isCvs and not mycvstree:
1456                 print "!!! It seems we don't have a cvs tree?"
1457                 sys.exit(3)
1458
1459         myunadded=cvstree.findunadded(mycvstree,recursive=1,basedir="./")
1460         myautoadd=[]
1461         if myunadded:
1462                 for x in range(len(myunadded)-1,-1,-1):
1463                         xs=myunadded[x].split("/")
1464                         if xs[-1]=="files":
1465                                 print "!!! files dir is not added! Please correct this."
1466                                 sys.exit(-1)
1467                         elif xs[-1]=="Manifest":
1468                                 # It's a manifest... auto add
1469                                 myautoadd+=[myunadded[x]]
1470                                 del myunadded[x]
1471                         elif len(xs[-1])>=7:
1472                                 if xs[-1][:7]=="digest-":
1473                                         del xs[-2]
1474                                         myeb="/".join(xs[:-1]+[xs[-1][7:]])+".ebuild"
1475                                         if os.path.exists(myeb):
1476                                                 # Ebuild exists for digest... So autoadd it.
1477                                                 myautoadd+=[myunadded[x]]
1478                                                 del myunadded[x]
1479
1480         if myautoadd:
1481                 print ">>> Auto-Adding missing digests..."
1482                 if "--pretend" in myoptions:
1483                         print "(/usr/bin/cvs add "+" ".join(myautoadd)+")"
1484                         retval=0
1485                 else:
1486                         retval=os.system("/usr/bin/cvs add "+" ".join(myautoadd))
1487                 if retval:
1488                         print "!!! Exiting on cvs (shell) error code:",retval
1489                         sys.exit(retval)
1490
1491         if myunadded:
1492                 print red("!!! The following files are in your cvs tree but are not added to the master")
1493                 print red("!!! tree. Please remove them from the cvs tree or add them to the master tree.")
1494                 for x in myunadded:
1495                         print "   ",x
1496                 print
1497                 print
1498                 sys.exit(1)
1499
1500         retval=["",""]
1501         if isCvs:
1502                 print "Performing a "+green("cvs -n up")+" with a little magic grep to check for updates."
1503                 retval=getstatusoutput("/usr/bin/cvs -n up 2>&1 | egrep '^[^\?] .*' | egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'")
1504
1505         mylines=retval[1].split("\n")
1506         myupdates=[]
1507         for x in mylines:
1508                 if not x:
1509                         continue
1510                 if x[0] not in "UPMAR": # Updates,Patches,Modified,Added,Removed
1511                         print red("!!! Please fix the following issues reported from cvs: ")+green("(U,P,M,A,R are ok)")
1512                         print red("!!! Note: This is a pretend/no-modify pass...")
1513                         print retval[1]
1514                         print
1515                         sys.exit(1)
1516                 elif x[0] in ["U","P"]:
1517                         myupdates+=[x[2:]]
1518
1519         if myupdates:
1520                 print green("Fetching trivial updates...")
1521                 if "--pretend" in myoptions:
1522                         print "(/usr/bin/cvs up "+" ".join(myupdates)+")"
1523                         retval=0
1524                 else:
1525                         retval=os.system("/usr/bin/cvs up "+" ".join(myupdates))
1526                 if retval!=0:
1527                         print "!!! cvs exited with an error. Terminating."
1528                         sys.exit(retval)
1529
1530         if isCvs:
1531                 mycvstree=cvstree.getentries("./",recursive=1)
1532                 mychanged=cvstree.findchanged(mycvstree,recursive=1,basedir="./")
1533                 mynew=cvstree.findnew(mycvstree,recursive=1,basedir="./")
1534                 myremoved=cvstree.findremoved(mycvstree,recursive=1,basedir="./")
1535                 if not (mychanged or mynew or myremoved):
1536                         print
1537                         print green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"\n"
1538                         print
1539                         print "(Didn't find any changed files...)"
1540                         print
1541                         sys.exit(0)
1542
1543         # Manifests need to be regenerated after all other commits, so don't commit
1544         # them now even if they have changed.
1545         mychanged = [f for f in mychanged if "Manifest" != os.path.basename(f)]
1546         myupdates=mychanged+mynew
1547         myheaders=[]
1548         mydirty=[]
1549         headerstring="'\$(Header|Id)"
1550         headerstring+=".*\$'"
1551         for myfile in myupdates:
1552                 myout=getstatusoutput("egrep -q "+headerstring+" "+myfile)
1553                 if myout[0]==0:
1554                         myheaders.append(myfile)
1555
1556         print "*",green(str(len(myupdates))),"files being committed...",green(str(len(myheaders))),"have headers that will change."
1557         print "*","Files with headers will cause the manifests to be made and recommited."
1558         if quiet == 0:
1559                 print "myupdates:",myupdates
1560                 print "myheaders:",myheaders
1561                 print
1562
1563         if commitmessagefile:
1564                 try:
1565                         f = open(commitmessagefile)
1566                         commitmessage = f.read()
1567                         f.close()
1568                         del f
1569                 except (IOError, OSError), e:
1570                         if e.errno == errno.ENOENT:
1571                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % commitmessagefile)
1572                         else:
1573                                 raise
1574                 # We've read the content so the file is no longer needed.
1575                 commitmessagefile = None
1576         if not commitmessage:
1577                 print "Please enter a CVS commit message at the prompt:"
1578                 while not commitmessage:
1579                         try:
1580                                 commitmessage=raw_input(green("> "))
1581                         except KeyboardInterrupt:
1582                                 exithandler()
1583         try:
1584                 commitmessage+="\n(Portage version: "+str(portage.VERSION)+")"
1585         except AttributeError:
1586                 print "Failed to insert portage version in message!"
1587                 commitmessage+="\n(Portage version: Unknown)"
1588         if myupdates or myremoved:
1589                 myfiles = myupdates + myremoved
1590                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1591                 mymsg = os.fdopen(fd, "w")
1592                 mymsg.write(commitmessage)
1593                 mymsg.close()
1594
1595                 print
1596                 print green("Using commit message:")
1597                 print green("------------------------------------------------------------------------------")
1598                 print commitmessage
1599                 print green("------------------------------------------------------------------------------")
1600                 print
1601
1602                 retval = None
1603                 if "--pretend" in myoptions:
1604                         print "(/usr/bin/cvs -q commit -F %s %s)" % \
1605                                 (commitmessagefile, " ".join(myfiles))
1606                 else:
1607                         retval = spawn(["/usr/bin/cvs", "-q", "commit",
1608                                 "-F", commitmessagefile] + myfiles,
1609                                 env=os.environ)
1610                 try:
1611                         os.unlink(commitmessagefile)
1612                 except OSError:
1613                         pass
1614                 if retval:
1615                         print "!!! Exiting on cvs (shell) error code:",retval
1616                         sys.exit(retval)
1617
1618         # Setup the GPG commands
1619         def gpgsign(filename):
1620                 if "PORTAGE_GPG_KEY" not in repoman_settings:
1621                         raise portage_exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
1622                 if "PORTAGE_GPG_DIR" not in repoman_settings:
1623                         if os.environ.has_key("HOME"):
1624                                 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(os.environ["HOME"], ".gnupg")
1625                                 if quiet < 1:
1626                                         print "Automatically setting PORTAGE_GPG_DIR to",repoman_settings["PORTAGE_GPG_DIR"]
1627                         else:
1628                                 raise portage_exception.MissingParameter("PORTAGE_GPG_DIR is unset!")
1629                 gpg_dir = repoman_settings["PORTAGE_GPG_DIR"]
1630                 if gpg_dir.startswith("~") and "HOME" in os.environ:
1631                         repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(
1632                                 os.environ["HOME"], gpg_dir[1:].lstrip(os.path.sep))
1633                 if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
1634                         raise portage_exception.InvalidLocation(
1635                                 "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
1636                                 repoman_settings["PORTAGE_GPG_DIR"])
1637                 gpgcmd = "gpg --sign --clearsign --yes "
1638                 gpgcmd+= "--default-key "+repoman_settings["PORTAGE_GPG_KEY"]
1639                 if repoman_settings.has_key("PORTAGE_GPG_DIR"):
1640                         gpgcmd += " --homedir "+repoman_settings["PORTAGE_GPG_DIR"]
1641                 if "--pretend" in myoptions:
1642                         print "("+gpgcmd+" "+filename+")"
1643                 else:
1644                         rValue = os.system(gpgcmd+" "+filename)
1645                         if rValue == os.EX_OK:
1646                                 os.rename(filename+".asc", filename)
1647                         else:
1648                                 raise portage_exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
1649
1650         manifest_commit_required = True
1651         if myheaders or myupdates or myremoved or mynew:
1652                 myfiles=myheaders+myupdates+myremoved+mynew
1653                 for x in range(len(myfiles)-1, -1, -1):
1654                         if myfiles[x].count("/") < 4-repolevel:
1655                                 del myfiles[x]
1656                 mydone=[]
1657                 if repolevel==3:   # In a package dir
1658                         repoman_settings["O"]=os.getcwd()
1659                         portage.digestgen([], repoman_settings, manifestonly=1,
1660                                 myportdb=portdb)
1661                 elif repolevel==2: # In a category dir
1662                         for x in myfiles:
1663                                 xs=x.split("/")
1664                                 if xs[0]==".":
1665                                         xs=xs[1:]
1666                                 if xs[0] in mydone:
1667                                         continue
1668                                 mydone.append(xs[0])
1669                                 repoman_settings["O"]=os.path.join(os.getcwd(), xs[0])
1670                                 portage.digestgen([], repoman_settings, manifestonly=1,
1671                                         myportdb=portdb)
1672                 elif repolevel==1: # repo-cvsroot
1673                         print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
1674                         for x in myfiles:
1675                                 xs=x.split("/")
1676                                 if xs[0]==".":
1677                                         xs=xs[1:]
1678                                 if "/".join(xs[:2]) in mydone:
1679                                         continue
1680                                 mydone.append("/".join(xs[:2]))
1681                                 repoman_settings["O"]=os.path.join(os.getcwd(), x[0], x[1])
1682                                 portage.digestgen([], repoman_settings, manifestonly=1,
1683                                         myportdb=portdb)
1684                 else:
1685                         print red("I'm confused... I don't know where I am!")
1686                         sys.exit(1)
1687
1688                 # Force an unsigned commit when more than one Manifest needs to be signed.
1689                 if repolevel < 3 and "sign" in repoman_settings.features:
1690                         if "--pretend" in myoptions:
1691                                 print "(/usr/bin/cvs -q commit -F "+commitmessagefile+")"
1692                         else:
1693                                 mymsg=open(commitmessagefile,"w")
1694                                 mymsg.write(commitmessage)
1695                                 mymsg.write("\n (Unsigned Manifest commit)")
1696                                 mymsg.close()
1697                                 retval=os.system("/usr/bin/cvs -q commit -F "+commitmessagefile)
1698                                 if retval:
1699                                         print "!!! Exiting on cvs (shell) error code:",retval
1700                                         sys.exit(retval)
1701                         manifest_commit_required = False
1702
1703         signed = False
1704         if "sign" in repoman_settings.features:
1705                 signed = True
1706                 try:
1707                         if repolevel==3:   # In a package dir
1708                                 repoman_settings["O"] = "."
1709                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1710                         elif repolevel==2: # In a category dir
1711                                 mydone=[]
1712                                 for x in myfiles:
1713                                         xs=x.split("/")
1714                                         if xs[0]==".":
1715                                                 xs=xs[1:]
1716                                         if xs[0] in mydone:
1717                                                 continue
1718                                         mydone.append(xs[0])
1719                                         repoman_settings["O"] = os.path.join(".", xs[0])
1720                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1721                         elif repolevel==1: # repo-cvsroot
1722                                 print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
1723                                 mydone=[]
1724                                 for x in myfiles:
1725                                         xs=x.split("/")
1726                                         if xs[0]==".":
1727                                                 xs=xs[1:]
1728                                         if "/".join(xs[:2]) in mydone:
1729                                                 continue
1730                                         mydone.append("/".join(xs[:2]))
1731                                         repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
1732                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1733                 except portage_exception.PortageException, e:
1734                         portage.writemsg("!!! %s\n" % str(e))
1735                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
1736                         signed = False
1737
1738         if manifest_commit_required or signed:
1739                 if "--pretend" in myoptions:
1740                         print "(/usr/bin/cvs -q commit -F "+commitmessagefile+")"
1741                 else:
1742                         fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1743                         mymsg = os.fdopen(fd, "w")
1744                         mymsg.write(commitmessage)
1745                         if signed:
1746                                 mymsg.write("\n (Signed Manifest commit)")
1747                         else:
1748                                 mymsg.write("\n (Unsigned Manifest commit)")
1749                         mymsg.close()
1750                         retval=os.system("/usr/bin/cvs -q commit -F "+commitmessagefile)
1751                         try:
1752                                 os.unlink(commitmessagefile)
1753                         except OSError:
1754                                 pass
1755                         if retval:
1756                                 print "!!! Exiting on cvs (shell) error code:",retval
1757                                 sys.exit(retval)
1758
1759         print
1760         if isCvs:
1761                 print "CVS commit complete."
1762         else:
1763                 print "repoman was too scared by not seeing any familiar cvs file that he forgot to commit anything"
1764         print green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n"
1765 sys.exit(0)
1766