bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2006 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 # Next to do: dep syntax checking in mask files
   7 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   8 # that last one is tricky because multiple profiles need to be checked.
   9
  10 import errno, os, shutil, sys
  11 if not hasattr(__builtins__, "set"):
  12         from sets import Set as set
  13 exename=os.path.basename(sys.argv[0])
  14 version="1.2"
  15
  16 allowed_filename_chars="a-zA-Z0-9._-+:"
  17 allowed_filename_chars_set = {}
  18 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('a'), ord('z')+1)))
  19 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('A'), ord('Z')+1)))
  20 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('0'), ord('9')+1)))
  21 map(allowed_filename_chars_set.setdefault, map(chr, map(ord, [".", "-", "_", "+", ":"])))
  22
  23 import string,signal,re,pickle,tempfile
  24
  25 os.environ["PORTAGE_LEGACY_GLOBALS"] = "false"
  26 try:
  27         import portage
  28 except ImportError:
  29         sys.path.insert(0, "/usr/lib/portage/pym")
  30         import portage
  31 del os.environ["PORTAGE_LEGACY_GLOBALS"]
  32
  33 import portage_checksum
  34 import portage_const
  35 import portage_dep
  36 import portage_exception
  37 import cvstree
  38 import time
  39 import codecs
  40
  41 from portage_manifest import Manifest
  42 from portage_exception import ParseError
  43
  44 from output import bold, darkgreen, darkred, green, nocolor, red, turquoise, yellow
  45
  46 from commands import getstatusoutput
  47 from fileinput import input
  48 from grp import getgrnam
  49 from stat import S_ISDIR, ST_CTIME, ST_GID, ST_MTIME
  50
  51 # A sane umask is needed for files that portage creates.
  52 os.umask(022)
  53 repoman_settings = portage.config(local_config=False,
  54         config_incrementals=portage_const.INCREMENTALS)
  55 repoman_settings.lock()
  56
  57 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
  58         not sys.stdout.isatty():
  59         nocolor()
  60
  61 def warn(txt):
  62         print exename+": "+txt
  63 def err(txt):
  64         warn(txt)
  65         sys.exit(1)
  66
  67 def err_help(txt):
  68         help(exitstatus=-1,helpfulness=0)
  69         warn(txt)
  70         sys.exit(1)
  71
  72 def exithandler(signum=None,frame=None):
  73         sys.stderr.write("\n"+exename+": Interrupted; exiting...\n")
  74         sys.exit(1)
  75         os.kill(0,signal.SIGKILL)
  76 signal.signal(signal.SIGINT,exithandler)
  77
  78 shortmodes={"ci":"commit"}
  79 modeshelp={
  80 "scan"   : "Scan directory tree for QA issues (default)",
  81 "fix"    : "Fix simple QA issues (stray digests, missing digests)",
  82 "full"   : "Scan directory tree for QA issues (full listing)",
  83 "help"   : "Show this screen",
  84 "commit" : "Scan directory tree for QA issues; if OK, commit via cvs",
  85 "last"   : "Remember report from last run",
  86 "lfull"  : "Remember report from last run (full listing)"
  87 }
  88 modes=modeshelp.keys()
  89 repoman_options={
  90 "--commitmsg"      : "Adds a commit message via the command line",
  91 "--commitmsgfile"  : "Adds a commit message from the specified file",
  92 "--help"           : "Show this screen",
  93 "--ignore-arches"  : "Ignore arch-specific failures (where arch != host)",
  94 "--ignore-masked"  : "Ignore masked packages (not allowed with commit mode)",
  95 "--pretend"        : "Don't commit or fix anything; just show what would be done",
  96 "--quiet"          : "Be less verbose about extraneous info",
  97 "--verbose"        : "Displays every package name while checking",
  98 "--version"        : "Show version info",
  99 "--xmlparse"       : "Forces the metadata.xml parse check to be carried out"
 100 }
 101 repoman_shortoptions={
 102 "-h" : "--help",
 103 "-i" : "--ignore-masked",
 104 "-I" : "--ignore-arches",
 105 "-m" : "--commitmsg",
 106 "-M" : "--commitmsgfile",
 107 "-p" : "--pretend",
 108 "-q" : "--quiet",
 109 "-v" : "--verbose",
 110 "-V" : "--version",
 111 "-x" : "--xmlparse"
 112 }
 113 repoman_shortoptions_rev=dict([(v,k) for (k,v) in repoman_shortoptions.items()])
 114 options=repoman_options.keys()
 115
 116 qahelp={
 117         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 118         "digest.partial":"Digest files do not contain all corresponding URI elements",
 119         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 120         "digestentry.unused":"Digest/Manifest entry has no matching SRC_URI entry",
 121         "digest.fail":"Digest does not match the specified local file",
 122         "digest.stray":"Digest files that do not have a corresponding ebuild",
 123         "digest.missing":"Digest files that are missing (ebuild exists, digest doesn't)",
 124         "digest.disjointed":"Digests not added to cvs when the matching ebuild has been added",
 125         "digest.notadded":"Digests that exist but have not been added to cvs",
 126         "digest.unmatch":"Digests which are incomplete (please check if your USE/ARCH includes all files)",
 127         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 128         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 129         "changelog.missing":"Missing ChangeLog files",
 130         "ebuild.disjointed":"Ebuilds not added to cvs when the matching digest has been added",
 131         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 132         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 133         "filedir.missing":"Package lacks a files directory",
 134         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
 135         "file.size":"Files in the files directory must be under 20k",
 136         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 137         "file.UTF8":"File is not UTF8 compliant",
 138         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 139         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 140         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 141         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 142         "SLOT.missing":"Ebuilds that have a missing or empty SLOT variable",
 143         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 144         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 145         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 146         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 147         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 148         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 149         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 150         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 151         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 152         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 153         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 154         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 155         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 156         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 157         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 158         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 159         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 160         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 161         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error",
 162         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 163         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 164         "variable.readonly":"Assigning a readonly variable",
 165         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 166         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file",
 167         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 168         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 169         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 170         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 171         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 172         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 173         "ebuild.badheader":"This ebuild has a malformed header",
 174         "metadata.missing":"Missing metadata.xml files",
 175         "metadata.bad":"Bad metadata.xml files",
 176         "virtual.versioned":"PROVIDE contains virtuals with versions",
 177         "virtual.exists":"PROVIDE contains existing package names",
 178         "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default",
 179         "usage.obsolete":"The ebuild makes use of an obsolete construct"
 180 }
 181
 182 qacats = qahelp.keys()
 183 qacats.sort()
 184
 185 qawarnings=[
 186 "changelog.missing",
 187 "changelog.notadded",
 188 "ebuild.notadded",
 189 "ebuild.nostable",
 190 "ebuild.allmasked",
 191 "ebuild.nesteddie",
 192 "digest.assumed",
 193 "digest.notadded",
 194 "digest.disjointed",
 195 "digest.missing",
 196 "digestentry.unused",
 197 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 198 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 199 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 200 "IUSE.invalid",
 201 "KEYWORDS.stupid",
 202 "ebuild.minorsyn",
 203 "ebuild.badheader",
 204 "file.size",
 205 "metadata.missing",
 206 "metadata.bad",
 207 "virtual.versioned",
 208 "virtual.exists",
 209 "virtual.unavailable",
 210 "usage.obsolete",
 211 "LIVEVCS.stable"
 212 ]
 213
 214 missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"]
 215 allvars=portage.auxdbkeys
 216 commitmessage=None
 217 commitmessagefile=None
 218 for x in missingvars:
 219         x += ".missing"
 220         if x not in qacats:
 221                 print "* missingvars values need to be added to qahelp ("+x+")"
 222                 qacats.append(x)
 223                 qawarnings.append(x)
 224
 225
 226 ven_cat = r'[\w0-9-]+'                                 # Category
 227 ven_nam = r'([+a-z0-9-]+(?:[+_a-z0-9-]*[+a-z0-9-]+)*)' # Name
 228 ven_ver = r'((?:\d+\.)*\d+[a-z]?)'                     # Version
 229 ven_suf = r'(_(alpha\d*|beta\d*|pre\d*|rc\d*|p\d+))?'  # Suffix
 230 ven_rev = r'(-r\d+)?'                                  # Revision
 231
 232 ven_string=ven_cat+'/'+ven_nam+'-'+ven_ver+ven_suf+ven_rev
 233 valid_ebuild_name_re=re.compile(ven_string+'$', re.I)
 234 valid_ebuild_filename_re=re.compile(ven_string+'\.ebuild$', re.I)
 235
 236 verbose=0
 237 quiet=0
 238
 239 def valid_ebuild_name(name):
 240         """(name) --- Checks to ensure that the package name meets portage specs.
 241         Return 1 if valid, 0 if not."""
 242         # Handle either a path to the ebuild, or cat/pkg-ver string
 243         if (len(name) > 7) and (name[-7:] == ".ebuild"):
 244                 if valid_ebuild_filename_re.match(name):
 245                         return 1
 246         else:
 247                 if valid_ebuild_name_re.match(name):
 248                         return 1
 249         return 0
 250
 251
 252 def show_version():
 253         print exename+" "+version
 254         sys.exit(0)
 255
 256 def help(exitstatus=1,helpfulness=1):
 257         if quiet:
 258                 helpfulness=0
 259         if helpfulness:
 260                 print
 261                 print green(exename+" "+version)
 262                 print " \"Quality is job zero.\""
 263                 print " Copyright 1999-2006 Gentoo Foundation"
 264                 print " Distributed under the terms of the GNU General Public License v2"
 265                 print
 266         print bold(" Usage:"),turquoise(exename),"[",green("options"),"] [",green("mode"),"]"
 267         if helpfulness:
 268                 print bold(" Modes:"),turquoise("scan (default)"),
 269                 for x in modes[1:]:
 270                         print "|",turquoise(x),
 271                 print
 272         print
 273         print " "+green(string.ljust("Options",20)+" Description")
 274         for x in options:
 275                 if repoman_shortoptions_rev.has_key(x):
 276                         shopt=repoman_shortoptions_rev[x]+", "+x
 277                 else:
 278                         shopt="    "+x
 279                 print " "+string.ljust(shopt,20),repoman_options[x]
 280         print
 281         print " "+green(string.ljust("Modes",20)+" Description")
 282         for x in modes:
 283                 print " "+string.ljust(x,20),modeshelp[x]
 284         if helpfulness:
 285                 print
 286                 print " "+green(string.ljust("QA keyword",20)+" Description")
 287                 for x in qacats:
 288                         print " "+string.ljust(x,20),qahelp[x]
 289                 print
 290         if (exitstatus != -1):
 291                 sys.exit(exitstatus)
 292         else:
 293                 print
 294
 295 def last():
 296         try:
 297                 #Retrieve and unpickle stats and fails from saved files
 298                 savedf=open('/var/cache/edb/repo.stats','r')
 299                 stats = pickle.load(savedf)
 300                 savedf.close()
 301                 savedf=open('/var/cache/edb/repo.fails','r')
 302                 fails = pickle.load(savedf)
 303                 savedf.close()
 304         except SystemExit, e:
 305                 raise  # Need to propogate this
 306         except:
 307                 err("Error retrieving last repoman run data; exiting.")
 308
 309         #dofail will be set to 1 if we have failed in at least one non-warning category
 310         dofail=0
 311         #dowarn will be set to 1 if we tripped any warnings
 312         dowarn=0
 313         #dofull will be set if we should print a "repoman full" informational message
 314         dofull=0
 315
 316         print
 317         print green("RepoMan remembers...")
 318         print
 319         for x in qacats:
 320                 if stats[x]:
 321                         dowarn=1
 322                         if x not in qawarnings:
 323                                 dofail=1
 324                 else:
 325                         if mymode!="lfull":
 326                                 continue
 327                 print "  "+string.ljust(x,20),
 328                 if stats[x]==0:
 329                         print green(`stats[x]`)
 330                         continue
 331                 elif x in qawarnings:
 332                         print yellow(`stats[x]`)
 333                 else:
 334                         print red(`stats[x]`)
 335                 if mymode!="lfull":
 336                         if stats[x]<12:
 337                                 for y in fails[x]:
 338                                         print "   "+y
 339                         else:
 340                                 dofull=1
 341                 else:
 342                         for y in fails[x]:
 343                                 print "   "+y
 344         print
 345         if dofull:
 346                 print bold("Note: type \"repoman lfull\" for a complete listing of repomans last run.")
 347                 print
 348         if dowarn and not dofail:
 349                 print green("RepoMan sez:"),"\"You only gave me a partial QA payment last time?\n              I took it, but I wasn't happy.\""
 350         elif not dofail:
 351                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
 352         print
 353         sys.exit(1)
 354
 355 mymode=None
 356 myoptions=[]
 357 if len(sys.argv)>1:
 358         x=1
 359         while x < len(sys.argv):
 360                 if sys.argv[x] in shortmodes.keys():
 361                         sys.argv[x]=shortmodes[sys.argv[x]]
 362                 elif sys.argv[x] in repoman_shortoptions.keys():
 363                         sys.argv[x] = repoman_shortoptions[sys.argv[x]]
 364                 if sys.argv[x] in modes:
 365                         if mymode is None:
 366                                 mymode=sys.argv[x]
 367                         else:
 368                                 err("Please specify either \""+mymode+"\" or \""+sys.argv[x]+"\", but not both.")
 369                 elif sys.argv[x] in options:
 370                         optionx=sys.argv[x]
 371                         if (optionx=="--commitmsg") and (len(sys.argv)>=(x+1)):
 372                                 commitmessage=sys.argv[x+1]
 373                                 x=x+1
 374                         elif (optionx=="--commitmsgfile") and (len(sys.argv)>=(x+1)):
 375                                 commitmessagefile=sys.argv[x+1]
 376                                 x=x+1
 377                         elif (optionx=="--verbose"):
 378                                 verbose+=1
 379                         elif (optionx=="--quiet"):
 380                                 quiet+=1
 381                         elif optionx not in myoptions:
 382                                 myoptions.append(optionx)
 383                 else:
 384                         err_help("\""+sys.argv[x]+"\" is not a valid mode or option.")
 385                 x=x+1
 386 if mymode is None:
 387         mymode="scan"
 388 if mymode=="help" or ("--help" in myoptions):
 389         help(exitstatus=0)
 390 if ("--version" in myoptions):
 391         show_version()
 392 if mymode=="last" or (mymode=="lfull"):
 393         last()
 394 if mymode == "commit":
 395         while "--ignore-masked" in myoptions:
 396                 myoptions.remove("--ignore-masked")
 397
 398 isCvs=False
 399 myreporoot=None
 400 if os.path.isdir("CVS"):
 401         if "cvs" not in repoman_settings.features:
 402                 print
 403                 print
 404                 print red('!!! You do not have ')+bold('FEATURES="cvs" ')+red("enabled...")
 405                 print red("!!! ")+bold("Adding \"cvs\" to FEATURES")
 406                 print
 407                 os.environ["FEATURES"]=repoman_settings["FEATURES"]+" cvs"
 408         isCvs = True
 409
 410 if not "--pretend" in myoptions and not isCvs:
 411         print
 412         print darkgreen("Not in a CVS repository; enabling pretend mode.")
 413         myoptions.append("--pretend");
 414
 415
 416 def have_profile_dir(path, maxdepth=3):
 417         while path != "/" and maxdepth:
 418                 if os.path.exists(path + "/profiles/package.mask"):
 419                         return path
 420                 path = os.path.normpath(path + "/..")
 421                 maxdepth -= 1
 422
 423 portdir=None
 424 portdir_overlay=None
 425 mydir=os.getcwd()
 426 if "PWD" in os.environ and os.environ["PWD"] != mydir and \
 427         os.path.realpath(os.environ["PWD"]) == mydir:
 428         # getcwd() returns the canonical path but that makes it hard for repoman to
 429         # orient itself if the user has symlinks in their portage tree structure.
 430         # We use os.environ["PWD"], if available, to get the non-canonical path of
 431         # the current working directory (from the shell).
 432         mydir = os.path.normpath(os.environ["PWD"])
 433 if mydir[-1] != "/":
 434         mydir += "/"
 435
 436 for overlay in repoman_settings["PORTDIR_OVERLAY"].split():
 437         if overlay[-1] != "/":
 438                 overlay += "/"
 439         if mydir.startswith(overlay):
 440                 portdir_overlay = overlay
 441                 subdir = mydir[len(overlay):]
 442                 if subdir and subdir[-1] != "/":
 443                         subdir += "/"
 444                 if have_profile_dir(mydir, subdir.count("/")):
 445                         portdir = portdir_overlay
 446                 break
 447
 448 if not portdir_overlay:
 449         if (repoman_settings["PORTDIR"] + os.path.sep).startswith(mydir):
 450                 portdir_overlay = repoman_settings["PORTDIR"]
 451         else:
 452                 portdir_overlay = have_profile_dir(mydir)
 453         portdir = portdir_overlay
 454
 455 if not portdir_overlay:
 456         sys.stderr.write("Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY from" + \
 457                 " the current\nworking directory.\n")
 458         sys.exit(1)
 459
 460 if not portdir:
 461         portdir = repoman_settings["PORTDIR"]
 462
 463 if portdir[-1] == "/":
 464         portdir = portdir[:-1]
 465 if portdir_overlay[-1] == "/":
 466         portdir_overlay = portdir_overlay[:-1]
 467
 468 os.environ["PORTDIR"] = portdir
 469 if portdir_overlay != portdir:
 470         os.environ["PORTDIR_OVERLAY"] = portdir_overlay
 471 else:
 472         os.environ["PORTDIR_OVERLAY"] = ""
 473
 474 if quiet < 2:
 475         print "\nSetting paths:"
 476         print "PORTDIR = \""+os.environ["PORTDIR"]+"\""
 477         print "PORTDIR_OVERLAY = \""+os.environ["PORTDIR_OVERLAY"]+"\""
 478
 479 # Now that PORTDIR_OVERLAY is properly overridden, create the portdb.
 480 repoman_settings = portage.config(local_config=False,
 481         config_incrementals=portage_const.INCREMENTALS)
 482 trees = portage.create_trees()
 483 trees["/"]["porttree"].settings = repoman_settings
 484 portdb = trees["/"]["porttree"].dbapi
 485 portdb.mysettings = repoman_settings
 486 # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman.
 487 trees["/"]["vartree"].dbapi = portage.fakedbapi(settings=repoman_settings)
 488
 489 if not myreporoot:
 490         myreporoot = os.path.basename(portdir_overlay)
 491         myreporoot += mydir[len(portdir_overlay):-1]
 492
 493 reposplit=string.split(myreporoot,"/")
 494 repolevel=len(reposplit)
 495
 496 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 497 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 498 # this check ensure that repoman knows where it is, and the manifest recommit is at least possible.
 499 if mymode == "commit" and repolevel not in [1,2,3]:
 500         print red("***")+" Commit attempts *must* be from within a cvs co, category, or package directory."
 501         print red("***")+" Attempting to commit from a packages files directory will be blocked for instance."
 502         print red("***")+" This is intended behaviour, to ensure the manifest is recommited for a package."
 503         print red("***")
 504         err("Unable to identify level we're commiting from for %s" % string.join(reposplit,'/'))
 505
 506 startdir=os.getcwd()
 507
 508 for x in range(0,repolevel-1):
 509         os.chdir("..")
 510 repodir=os.getcwd()
 511 os.chdir(startdir)
 512
 513 def caterror(mycat):
 514         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 515 if "--pretend" in myoptions:
 516         print green("\nRepoMan does a once-over of the neighborhood...")
 517 elif quiet < 1:
 518         print green("\nRepoMan scours the neighborhood...")
 519
 520 def parse_use_local_desc(mylines, usedict=None):
 521         """returns a dict of the form {cpv:set(flags)}"""
 522         if usedict is None:
 523                 usedict = {}
 524         lineno = 0
 525         for l in mylines:
 526                 lineno += 1
 527                 if not l or l.startswith("#"):
 528                         continue
 529                 mysplit = l.split(None, 1)
 530                 if not mysplit:
 531                         continue
 532                 mysplit = mysplit[0].split(":")
 533                 if len(mysplit) != 2:
 534                         raise ParseError("line %d: Malformed input: '%s'" % \
 535                                 (lineno, l.rstrip("\n")))
 536                 usedict.setdefault(mysplit[0], set())
 537                 usedict[mysplit[0]].add(mysplit[1])
 538         return usedict
 539
 540 # retreive local USE list
 541 luselist={}
 542 try:
 543         f = open(os.path.join(portdir, "profiles", "use.local.desc"))
 544         parse_use_local_desc(f, luselist)
 545         f.close()
 546 except (IOError, OSError, ParseError), e:
 547         print >> sys.stderr, str(e)
 548         err("Couldn't read from use.local.desc")
 549
 550 if portdir_overlay != portdir:
 551         filename = os.path.join(portdir_overlay, "profiles", "use.local.desc")
 552         if os.path.exists(filename):
 553                 try:
 554                         f = open(filename)
 555                         parse_use_local_desc(f, luselist)
 556                         f.close()
 557                 except (IOError, OSError, ParseError), e:
 558                         print >> sys.stderr, str(e)
 559                         err("Couldn't read from '%s'" % filename)
 560         del filename
 561
 562 # setup a uselist from portage
 563 uselist=[]
 564 try:
 565         uselist=portage.grabfile(portdir+"/profiles/use.desc")
 566         for l in range(0,len(uselist)):
 567                 uselist[l]=string.split(uselist[l])[0]
 568         for var in string.split(repoman_settings["USE_EXPAND"]):
 569                 vardescs = portage.grabfile(portdir+"/profiles/desc/"+var.lower()+".desc")
 570                 for l in range(0, len(vardescs)):
 571                         uselist.append(var.lower() + "_" + string.split(vardescs[l])[0])
 572 except SystemExit, e:
 573         raise  # Need to propogate this
 574 except:
 575         err("Couldn't read USE flags from use.desc")
 576
 577 # retrieve a list of current licenses in portage
 578 liclist = set(portage.listdir(os.path.join(portdir, "licenses")))
 579 if not liclist:
 580         err("Couldn't find licenses?")
 581 if portdir_overlay != portdir:
 582         liclist.update(portage.listdir(os.path.join(portdir_overlay, "licenses")))
 583
 584 # retrieve list of offical keywords
 585 kwlist = set(portage.grabfile(os.path.join(portdir, "profiles", "arch.list")))
 586 if not kwlist:
 587         err("Couldn't read KEYWORDS from arch.list")
 588
 589 if portdir_overlay != portdir:
 590         kwlist.update(portage.grabfile(
 591                 os.path.join(portdir_overlay, "profiles", "arch.list")))
 592
 593 scanlist=[]
 594 if repolevel==2:
 595         #we are inside a category directory
 596         catdir=reposplit[-1]
 597         if catdir not in repoman_settings.categories:
 598                 caterror(catdir)
 599         mydirlist=os.listdir(startdir)
 600         for x in mydirlist:
 601                 if x == "CVS" or x.startswith("."):
 602                         continue
 603                 if os.path.isdir(startdir+"/"+x):
 604                         scanlist.append(catdir+"/"+x)
 605 elif repolevel==1:
 606         for x in repoman_settings.categories:
 607                 if not os.path.isdir(startdir+"/"+x):
 608                         continue
 609                 for y in os.listdir(startdir+"/"+x):
 610                         if y == "CVS" or y.startswith("."):
 611                                 continue
 612                         if os.path.isdir(startdir+"/"+x+"/"+y):
 613                                 scanlist.append(x+"/"+y)
 614 elif repolevel==3:
 615         catdir = reposplit[-2]
 616         if catdir not in repoman_settings.categories:
 617                 caterror(catdir)
 618         scanlist.append(catdir+"/"+reposplit[-1])
 619
 620 profiles={}
 621 descfile=portdir+"/profiles/profiles.desc"
 622 if os.path.exists(descfile):
 623         for x in portage.grabfile(descfile):
 624                 if x[0]=="#":
 625                         continue
 626                 arch=string.split(x)
 627                 if len(arch)!=3:
 628                         print "wrong format: \""+red(x)+"\" in "+descfile
 629                         continue
 630                 if not os.path.isdir(portdir+"/profiles/"+arch[1]):
 631                         print "Invalid "+arch[2]+" profile ("+arch[1]+") for arch "+arch[0]
 632                         continue
 633                 if profiles.has_key(arch[0]):
 634                         profiles[arch[0]]+= [[arch[1], arch[2]]]
 635                 else:
 636                         profiles[arch[0]] = [[arch[1], arch[2]]]
 637
 638         for x in repoman_settings.archlist():
 639                 if x[0] == "~":
 640                         continue
 641                 if not profiles.has_key(x):
 642                         print red("\""+x+"\" doesn't have a valid profile listed in profiles.desc.")
 643                         print red("You need to either \"cvs update\" your profiles dir or follow this")
 644                         print red("up with the "+x+" team.")
 645                         print
 646 else:
 647         print red("profiles.desc does not exist: "+descfile)
 648         print red("You need to do \"cvs update\" in profiles dir.")
 649         print
 650         sys.exit(1)
 651
 652
 653 stats={}
 654 fails={}
 655
 656 for x in qacats:
 657         stats[x]=0
 658         fails[x]=[]
 659 xmllint_capable = False
 660 if getstatusoutput('which xmllint')[0] != 0:
 661         print red("!!! xmllint not found. Can't check metadata.xml.\n")
 662         if "--xmlparse" in myoptions or repolevel==3:
 663                 print red("!!!")+" sorry, xmllint is needed.  failing\n"
 664                 sys.exit(1)
 665 else:
 666         #hardcoded paths/urls suck. :-/
 667         must_fetch=1
 668         backup_exists=0
 669         try:
 670                 # if it's been over a week since fetching (or the system clock is fscked), grab an updated copy of metadata.dtd
 671                 # clock is fscked or it's been a week. time to grab a new one.
 672                 ct=os.stat(portage.CACHE_PATH + '/metadata.dtd')[ST_CTIME]
 673                 if abs(time.time() - ct) > (60*60*24*7):
 674                         # don't trap the exception, we're watching for errno 2 (file not found), anything else is a bug.
 675                         backup_exists=1
 676                 else:
 677                         must_fetch=0
 678
 679         except (OSError,IOError), e:
 680                 if e.errno != 2:
 681                         print red("!!!")+" caught exception '%s' for %s/metadata.dtd, bailing" % (str(e), portage.CACHE_PATH)
 682                         sys.exit(1)
 683
 684         if must_fetch:
 685                 print
 686                 print green("***")+" the local copy of metadata.dtd needs to be refetched, doing that now"
 687                 print
 688                 try:
 689                         if os.path.exists(repoman_settings["DISTDIR"]+'/metadata.dtd'):
 690                                 os.remove(repoman_settings["DISTDIR"]+'/metadata.dtd')
 691                         val=portage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoman_settings,fetchonly=0, \
 692                                 try_mirrors=0)
 693                         if val:
 694                                 if backup_exists:
 695                                         os.remove(portage.CACHE_PATH+'/metadata.dtd')
 696                                 shutil.copy(repoman_settings["DISTDIR"]+'/metadata.dtd',portage.CACHE_PATH+'/metadata.dtd')
 697                                 os.chown(portage.CACHE_PATH+'/metadata.dtd',os.getuid(),portage.portage_gid)
 698                                 os.chmod(portage.CACHE_PATH+'/metadata.dtd',0664)
 699
 700
 701                 except SystemExit, e:
 702                         raise  # Need to propogate this
 703                 except Exception,e:
 704                         print
 705                         print red("!!!")+" attempting to fetch 'http://www.gentoo.org/dtd/metadata.dtd', caught"
 706                         print red("!!!")+" exception '%s' though." % str(e)
 707                         val=0
 708                 if not val:
 709                         print red("!!!")+" fetching new metadata.dtd failed, aborting"
 710                         sys.exit(1)
 711         #this can be problematic if xmllint changes their output
 712         xmllint_capable=True
 713
 714
 715 def x11_deprecation_check(depstr):
 716         if depstr.find("virtual/x11") == -1:
 717                 return False
 718         depsplit = depstr.split()
 719         ok_stack = []
 720         ok = False
 721         for token in depsplit:
 722                 if token == "(":
 723                         ok_stack.append(ok)
 724                         ok = False
 725                 elif token == ")":
 726                         ok = ok_stack.pop()
 727                 elif token == "||":
 728                         ok = True
 729                 else:
 730                         ok = False
 731                         if token.find("virtual/x11") != -1 and (not ok_stack or not ok_stack[-1]):
 732                                 return True
 733         return False
 734
 735
 736 arch_caches={}
 737 for x in scanlist:
 738         #ebuilds and digests added to cvs respectively.
 739         if verbose:
 740                 print "checking package " + x
 741         eadded=[]
 742         dadded=[]
 743         catdir,pkgdir=x.split("/")
 744         checkdir=repodir+"/"+x
 745         checkdirlist=os.listdir(checkdir)
 746         ebuildlist=[]
 747         for y in checkdirlist:
 748                 if y[-7:]==".ebuild":
 749                         ebuildlist.append(y[:-7])
 750                 if y in ["Manifest","ChangeLog","metadata.xml"]:
 751                         if os.stat(checkdir+"/"+y)[0] & 0x0248:
 752                                 stats["file.executable"] += 1
 753                                 fails["file.executable"].append(checkdir+"/"+y)
 754         digestlist=[]
 755
 756         for y in checkdirlist:
 757                 for c in y.strip(os.path.sep):
 758                         if c not in allowed_filename_chars_set:
 759                                 stats["file.name"] += 1
 760                                 fails["file.name"].append("%s/%s: char '%s'" % (checkdir, y, c))
 761                                 break
 762
 763                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
 764                         continue
 765                 try:
 766                         line = 1
 767                         for l in codecs.open(checkdir+"/"+y, "r", "utf8"):
 768                                 line +=1
 769                 except UnicodeDecodeError, ue:
 770                         stats["file.UTF8"] += 1
 771                         s = ue.object[:ue.start]
 772                         l2 = s.count("\n")
 773                         line += l2
 774                         if l2 != 0:
 775                                 s = s[s.rfind("\n") + 1:]
 776                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
 777
 778         if isCvs:
 779                 try:
 780                         mystat=os.stat(checkdir+"/files")[0]
 781                         if len(ebuildlist) and not S_ISDIR(mystat):
 782                                 raise Exception
 783                 except SystemExit, e:
 784                         raise  # Need to propogate this
 785                 except:
 786                         stats["filedir.missing"] += 1
 787                         fails["filedir.missing"].append(checkdir)
 788                         continue
 789                 try:
 790                         myf=open(checkdir+"/CVS/Entries","r")
 791                         myl=myf.readlines()
 792                         for l in myl:
 793                                 if l[0]!="/":
 794                                         continue
 795                                 splitl=l[1:].split("/")
 796                                 if not len(splitl):
 797                                         continue
 798                                 if splitl[0][-7:]==".ebuild":
 799                                         eadded.append(splitl[0][:-7])
 800                 except IOError:
 801                         if mymode=="commit":
 802                                 stats["CVS/Entries.IO_error"] += 1
 803                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
 804                         continue
 805
 806                 try:
 807                         myf=open(checkdir+"/files/CVS/Entries","r")
 808                         myl=myf.readlines()
 809                         for l in myl:
 810                                 if l[0]!="/":
 811                                         continue
 812                                 splitl=l[1:].split("/")
 813                                 if not len(splitl):
 814                                         continue
 815                                 if splitl[0][:7]=="digest-":
 816                                         dadded.append(splitl[0][7:])
 817                 except IOError:
 818                         if mymode=="commit":
 819                                 stats["CVS/Entries.IO_error"] += 1
 820                                 fails["CVS/Entries.IO_error"].append(checkdir+"/files/CVS/Entries")
 821                         continue
 822
 823         if mymode == "commit":
 824                 repoman_settings["O"] = checkdir
 825                 portage.digestgen(
 826                         [], repoman_settings, manifestonly=1, myportdb=portdb)
 827
 828         mf = Manifest(checkdir, repoman_settings["DISTDIR"])
 829         mydigests=mf.getTypeDigests("DIST")
 830         myfiles_all = []
 831
 832         if os.path.exists(checkdir+"/files"):
 833                 filesdirlist=os.listdir(checkdir+"/files")
 834                 for y in filesdirlist:
 835                         if y[:7]=="digest-":
 836                                 if y[7:] not in dadded:
 837                                         #digest not added to cvs
 838                                         stats["digest.notadded"]=stats["digest.notadded"]+1
 839                                         fails["digest.notadded"].append(x+"/files/"+y)
 840                                         if y[7:] in eadded:
 841                                                 stats["digest.disjointed"]=stats["digest.disjointed"]+1
 842                                                 fails["digest.disjointed"].append(x+"/files/"+y)
 843
 844                                 if os.stat(checkdir+"/files/"+y)[0] & 0x0248:
 845                                         stats["file.executable"] += 1
 846                                         fails["file.executable"].append(x+"/files/"+y)
 847
 848                                 mykey = catdir + "/" + y[7:]
 849                                 if y[7:] not in ebuildlist:
 850                                         #stray digest
 851                                         if mymode=="fix":
 852                                                 if "--pretend" in myoptions:
 853                                                         print "(cd "+repodir+"/"+x+"/files; cvs rm -f "+y+")"
 854                                                 else:
 855                                                         os.system("(cd "+repodir+"/"+x+"/files; cvs rm -f "+y+")")
 856                                         else:
 857                                                 stats["digest.stray"]=stats["digest.stray"]+1
 858                                                 fails["digest.stray"].append(x+"/files/"+y)
 859                                 else:
 860                                         # We have an ebuild
 861                                         myuris, myfiles = portdb.getfetchlist(mykey, all=True)
 862                                         myfiles_all.extend(myfiles)
 863
 864                                         uri_dict = {}
 865                                         for myu in myuris:
 866                                                 myubn = os.path.basename(myu)
 867                                                 if myubn not in uri_dict:
 868                                                         uri_dict[myubn]  = [myu]
 869                                                 else:
 870                                                         uri_dict[myubn] += [myu]
 871
 872                                         for myf in uri_dict:
 873                                                 myff = repoman_settings["DISTDIR"] + "/" + myf
 874                                                 if not mydigests.has_key(myf):
 875                                                         uri_settings = portage.config(clone=repoman_settings)
 876                                                         if mymode == "fix":
 877                                                                 if not portage.fetch(uri_dict[myf], uri_settings):
 878                                                                         stats["digest.unmatch"] += 1
 879                                                                         fails["digest.unmatch"].append(y+"::"+myf)
 880                                                                 else:
 881                                                                         eb_name = portdb.findname2(mykey)[0]
 882                                                                         portage.doebuild(eb_name, "digest", "/",
 883                                                                                 uri_settings, tree="porttree",
 884                                                                                 mydbapi=portdb)
 885                                                         else:
 886                                                                 stats["digest.partial"] += 1
 887                                                                 fails["digest.partial"].append(y+"::"+myf)
 888                                                 elif "assume-digests" not in repoman_settings.features:
 889                                                         if os.path.exists(myff):
 890                                                                 if not portage_checksum.verify_all(myff, mydigests[myf]):
 891                                                                         stats["digest.fail"] += 1
 892                                                                         fails["digest.fail"].append(y+"::"+myf)
 893                                                         elif repolevel == 3:
 894                                                                 stats["digest.assumed"] += 1
 895                                                                 fails["digest.assumed"].append(y+"::"+myf)
 896
 897                 # recurse through files directory
 898                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
 899                 while filesdirlist:
 900                         y = filesdirlist.pop(0)
 901                         try:
 902                                 mystat = os.stat(checkdir+"/files/"+y)
 903                         except OSError, oe:
 904                                 if oe.errno == 2:
 905                                         # don't worry about it.  it likely was removed via fix above.
 906                                         continue
 907                                 else:
 908                                         raise oe
 909                         if S_ISDIR(mystat.st_mode):
 910                                 if y == "CVS":
 911                                         continue
 912                                 for z in os.listdir(checkdir+"/files/"+y):
 913                                         if z == "CVS":
 914                                                 continue
 915                                         filesdirlist.append(y+"/"+z)
 916                         # current policy is no files over 20k, this is the check.
 917                         elif mystat.st_size > 20480:
 918                                 stats["file.size"] += 1
 919                                 fails["file.size"].append("("+ str(mystat.st_size/1024) + "K) "+x+"/files/"+y)
 920
 921                         for c in os.path.basename(y.rstrip(os.path.sep)):
 922                                 if c not in allowed_filename_chars_set:
 923                                         stats["file.name"] += 1
 924                                         fails["file.name"].append("%s/files/%s: char '%s'" % (checkdir, y, c))
 925                                         break
 926
 927         for entry in mydigests.keys():
 928                 if entry not in myfiles_all:
 929                         stats["digestentry.unused"] += 1
 930                         fails["digestentry.unused"].append(checkdir+"::"+entry)
 931
 932
 933         if "ChangeLog" not in checkdirlist:
 934                 stats["changelog.missing"]+=1
 935                 fails["changelog.missing"].append(x+"/ChangeLog")
 936
 937         #metadata.xml file check
 938         if "metadata.xml" not in checkdirlist:
 939                 stats["metadata.missing"]+=1
 940                 fails["metadata.missing"].append(x+"/metadata.xml")
 941         #metadata.xml parse check
 942         else:
 943                 #Only carry out if in package directory or check forced
 944                 if xmllint_capable:
 945                         st=getstatusoutput("xmllint --nonet --noout --dtdvalid %s/metadata.dtd %s/metadata.xml" % (portage.CACHE_PATH, checkdir))
 946                         if st[0] != 0:
 947                                 for z in st[1].split("\n"):
 948                                         print red("!!! ")+z
 949                                 stats["metadata.bad"]+=1
 950                                 fails["metadata.bad"].append(x+"/metadata.xml")
 951
 952         allmasked = True
 953
 954         for y in ebuildlist:
 955                 if os.stat(checkdir+"/"+y+".ebuild")[0] & 0x0248:
 956                         stats["file.executable"] += 1
 957                         fails["file.executable"].append(x+"/"+y+".ebuild")
 958                 if y not in eadded:
 959                         #ebuild not added to cvs
 960                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
 961                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
 962                         if y in dadded:
 963                                 stats["ebuild.disjointed"]=stats["ebuild.disjointed"]+1
 964                                 fails["ebuild.disjointed"].append(x+"/"+y+".ebuild")
 965                 if not os.path.exists(checkdir+"/files/digest-"+y):
 966                         if mymode=="fix":
 967                                 if "--pretend" in myoptions:
 968                                         print "You will need to run:"
 969                                         print "  /usr/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest"
 970                                 else:
 971                                         retval=os.system("/usr/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest")
 972                                         if retval:
 973                                                 print "!!! Exiting on ebuild digest (shell) error code:",retval
 974                                                 sys.exit(retval)
 975                         else:
 976                                 stats["digest.missing"]=stats["digest.missing"]+1
 977                                 fails["digest.missing"].append(x+"/files/digest-"+y)
 978                 myesplit=portage.pkgsplit(y)
 979                 if myesplit is None or not valid_ebuild_name(x.split("/")[0]+"/"+y):
 980                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
 981                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
 982                         continue
 983                 elif myesplit[0]!=pkgdir:
 984                         print pkgdir,myesplit[0]
 985                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
 986                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
 987                         continue
 988                 try:
 989                         myaux = dict(
 990                                 zip(allvars, portdb.aux_get(os.path.join(catdir, y), allvars)))
 991                 except KeyError:
 992                         stats["ebuild.syntax"]=stats["ebuild.syntax"]+1
 993                         fails["ebuild.syntax"].append(x+"/"+y+".ebuild")
 994                         continue
 995                 except IOError:
 996                         stats["ebuild.output"]=stats["ebuild.output"]+1
 997                         fails["ebuild.output"].append(x+"/"+y+".ebuild")
 998                         continue
 999
1000                 # Test for negative logic and bad words in the RESTRICT var.
1001                 #for x in myaux[allvars.index("RESTRICT")].split():
1002                 #       if x.startswith("no"):
1003                 #               print "Bad RESTRICT value: %s" % x
1004
1005                 myaux["PROVIDE"] = portage_dep.use_reduce(portage_dep.paren_reduce(myaux["PROVIDE"]), matchall=1)
1006                 myaux["PROVIDE"] = " ".join(portage.flatten(myaux["PROVIDE"]))
1007                 for myprovide in myaux["PROVIDE"].split():
1008                         prov_cp = portage.dep_getkey(myprovide)
1009                         if prov_cp != myprovide:
1010                                 stats["virtual.versioned"]+=1
1011                                 fails["virtual.versioned"].append(x+"/"+y+".ebuild: "+myprovide)
1012                         prov_pkg = portage.dep_getkey(
1013                                 portage.best(portdb.xmatch("match-all", prov_cp)))
1014                         if prov_cp == prov_pkg:
1015                                 stats["virtual.exists"]+=1
1016                                 fails["virtual.exists"].append(x+"/"+y+".ebuild: "+prov_cp)
1017
1018                 for pos in range(0,len(missingvars)):
1019                         if not myaux[missingvars[pos]]:
1020                                 myqakey=missingvars[pos]+".missing"
1021                                 stats[myqakey]=stats[myqakey]+1
1022                                 fails[myqakey].append(x+"/"+y+".ebuild")
1023
1024                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1025                 if "-*" in myaux["KEYWORDS"].split():
1026                         haskeyword = False
1027                         for kw in myaux["KEYWORDS"].split():
1028                                 if kw[0] == "~":
1029                                         kw = kw[1:]
1030                                 if kw in kwlist:
1031                                         haskeyword = True
1032                         if not haskeyword:
1033                                 stats["KEYWORDS.stupid"] += 1
1034                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1035
1036                 """
1037                 Ebuilds that inherit a "Live" eclasss (darcs,subversion,git,cvs,etc..) should
1038                 not be allowed to be marked stable
1039                 """
1040                 if set(["darcs","cvs","subversion","git"]).intersection(myaux["INHERITED"].split()):
1041                         bad_stable_keywords = [keyword for keyword in myaux["KEYWORDS"].split() if not keyword.startswith("~") and not keyword.startswith("-")]
1042                         if bad_stable_keywords:
1043                                 stats["LIVEVCS.stable"] += 1
1044                                 fails["LIVEVCS.stable"].append(x+"/"+y+".ebuild with stable keywords:%s " % bad_stable_keywords)
1045                         del keyword, bad_stable_keywords
1046
1047                 if "--ignore-arches" in myoptions:
1048                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1049                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1050                 else:
1051                         arches=[]
1052                         for keyword in myaux["KEYWORDS"].split():
1053                                 if (keyword[0]=="-"):
1054                                         continue
1055                                 elif (keyword[0]=="~"):
1056                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1057                                 else:
1058                                         arches.append([keyword, keyword, [keyword]])
1059                                         allmasked = False
1060
1061                 baddepsyntax = False
1062                 badlicsyntax = False
1063                 badprovsyntax = False
1064                 catpkg = catdir+"/"+y
1065                 myiuse = set(repoman_settings.archlist())
1066                 for myflag in myaux["IUSE"].split():
1067                         if myflag.startswith("+"):
1068                                 myflag = myflag[1:]
1069                         myiuse.add(myflag)
1070
1071                 type_list, badsyntax = [], []
1072                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND", "LICENSE", "PROVIDE"):
1073                         mydepstr = myaux[mytype]
1074
1075                         if (string.find(mydepstr, " ?") != -1):
1076                                 badsyntax.append("'?' preceded by space")
1077
1078                         try:
1079                                 # Missing closing parenthesis will result in a ValueError
1080                                 mydeplist = portage_dep.paren_reduce(mydepstr)
1081                                 # Missing opening parenthesis will result in a final "" element
1082                                 if "" in mydeplist or "(" in mydeplist:
1083                                         raise ValueError
1084                         except ValueError:
1085                                 badsyntax.append("parenthesis mismatch")
1086                                 mydeplist = []
1087
1088                         try:
1089                                 portage_dep.use_reduce(mydeplist, excludeall=myiuse)
1090                         except portage_exception.InvalidDependString, e:
1091                                 badsyntax.append(str(e))
1092
1093                         for token in ("||", "(", ")"):
1094                                 if mydepstr.startswith(token+" "):
1095                                         myteststr = mydepstr[len(token):]
1096                                 else:
1097                                         myteststr = mydepstr
1098                                 if myteststr.endswith(" "+token):
1099                                         myteststr = myteststr[:-len(token)]
1100                                 while myteststr.find(" "+token+" ") != -1:
1101                                         myteststr = " ".join(myteststr.split(" "+token+" ", 1))
1102                                 if myteststr.find(token) != -1:
1103                                         badsyntax.append("'%s' not separated by space" % (token))
1104
1105
1106                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1107                                 for token in filter(lambda x: not (x.endswith("?") or x.strip() in ("||", "&&", "(", ")")), mydepstr.split()):
1108                                         if not "/" in token or \
1109                                                 ":" in token and myaux["EAPI"] == "0":
1110                                                 badsyntax.append("'%s' not a valid atom" % token)
1111
1112                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1113
1114                 for m,b in zip(type_list, badsyntax):
1115                         stats[m+".syntax"] += 1
1116                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1117
1118                 badlicsyntax = len(filter(lambda x:x=="LICENSE", type_list))
1119                 badprovsyntax = len(filter(lambda x:x=="PROVIDE", type_list))
1120                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1121                 badlicsyntax = badlicsyntax > 0
1122                 badprovsyntax = badprovsyntax > 0
1123
1124                 if not baddepsyntax:
1125                         if x11_deprecation_check(" ".join([myaux["DEPEND"], myaux["RDEPEND"], myaux["PDEPEND"]])):
1126                                 stats["usage.obsolete"] += 1
1127                                 fails["usage.obsolete"].append("%s/%s.ebuild: not migrated to modular X" % (x, y))
1128
1129                 for keyword,arch,groups in arches:
1130
1131                         if not profiles.has_key(arch):
1132                                 # A missing profile will create an error further down
1133                                 # during the KEYWORDS verification.
1134                                 continue
1135
1136                         for prof in profiles[arch]:
1137
1138                                 profdir = portdir+"/profiles/"+prof[0]
1139
1140                                 if prof[0] in arch_caches:
1141                                         dep_settings = arch_caches[prof[0]]
1142                                 else:
1143                                         dep_settings = portage.config(
1144                                                 config_profile_path=profdir,
1145                                                 config_incrementals=portage_const.INCREMENTALS,
1146                                                 local_config=False)
1147                                         arch_caches[prof[0]] = dep_settings
1148                                         while True:
1149                                                 try:
1150                                                         # Protect ACCEPT_KEYWORDS from config.regenerate()
1151                                                         # (just in case)
1152                                                         dep_settings.incrementals.remove("ACCEPT_KEYWORDS")
1153                                                 except ValueError:
1154                                                         break
1155
1156                                 trees["/"]["porttree"].settings = dep_settings
1157                                 portdb.mysettings = dep_settings
1158                                 # for package.use.mask support inside dep_check
1159                                 dep_settings.setcpv("/".join((catdir, y)))
1160                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1161                                 # just in case, prevent config.reset() from nuking these.
1162                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1163
1164                                 for myprovide in myaux["PROVIDE"].split():
1165                                         prov_cp = portage.dep_getkey(myprovide)
1166                                         if prov_cp not in dep_settings.getvirtuals():
1167                                                 stats["virtual.unavailable"]+=1
1168                                                 fails["virtual.unavailable"].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+prov_cp)
1169
1170                                 if not baddepsyntax:
1171                                         ismasked = os.path.join(catdir, y) not in \
1172                                                 portdb.xmatch("list-visible", x)
1173                                         if ismasked:
1174                                                 if "--ignore-masked" in myoptions:
1175                                                         continue
1176                                                 #we are testing deps for a masked package; give it some lee-way
1177                                                 suffix="masked"
1178                                                 matchmode="match-all"
1179                                         else:
1180                                                 suffix=""
1181                                                 matchmode="match-visible"
1182
1183                                         if prof[1] == "dev":
1184                                                 suffix=suffix+"indev"
1185
1186                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1187
1188                                                 mykey=mytype+".bad"+suffix
1189                                                 myvalue = myaux[mytype]
1190                                                 if not myvalue:
1191                                                         continue
1192                                                 try:
1193                                                         mydep = portage.dep_check(myvalue, portdb,
1194                                                                 dep_settings, use="all", mode=matchmode,
1195                                                                 trees=trees)
1196                                                 except KeyError, e:
1197                                                         stats[mykey]=stats[mykey]+1
1198                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(e[0]))
1199                                                         continue
1200
1201                                                 if mydep[0]==1:
1202                                                         if mydep[1]!=[]:
1203                                                                 #we have some unsolvable deps
1204                                                                 #remove ! deps, which always show up as unsatisfiable
1205                                                                 d=0
1206                                                                 while d<len(mydep[1]):
1207                                                                         if mydep[1][d][0]=="!":
1208                                                                                 del mydep[1][d]
1209                                                                         else:
1210                                                                                 d += 1
1211                                                                 #if we emptied out our list, continue:
1212                                                                 if not mydep[1]:
1213                                                                         continue
1214                                                                 stats[mykey]=stats[mykey]+1
1215                                                                 fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1216                                                 else:
1217                                                         stats[mykey]=stats[mykey]+1
1218                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1219
1220                 # this check needs work, it won't catch (\ndie)
1221                 if not os.system("egrep '^[^#]*\([^)]*\<die\>' "+checkdir+"/"+y+".ebuild >/dev/null 2>&1"):
1222                         stats["ebuild.nesteddie"]=stats["ebuild.nesteddie"]+1
1223                         fails["ebuild.nesteddie"].append(x+"/"+y+".ebuild")
1224                 # uselist checks - global
1225                 myuse = []
1226                 default_use = []
1227                 for myflag in myaux["IUSE"].split():
1228                         if myflag.startswith("+"):
1229                                 default_use.append(myflag)
1230                                 myflag = myflag[1:]
1231                         myuse.append(myflag)
1232                 for mypos in range(len(myuse)-1,-1,-1):
1233                         if myuse[mypos] and (myuse[mypos] in uselist):
1234                                 del myuse[mypos]
1235                 # uselist checks - local
1236                 mykey = portage.dep_getkey(catpkg)
1237                 if luselist.has_key(mykey):
1238                         for mypos in range(len(myuse)-1,-1,-1):
1239                                 if myuse[mypos] and (myuse[mypos] in luselist[mykey]):
1240                                         del myuse[mypos]
1241                 if default_use and myaux["EAPI"] == "0":
1242                         myuse += default_use
1243                 for mypos in range(len(myuse)):
1244                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1245                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1246
1247                 # license checks
1248                 if not badlicsyntax:
1249                         myuse = myaux["LICENSE"]
1250                         # Parse the LICENSE variable, remove USE conditions and
1251                         # flatten it.
1252                         myuse=portage_dep.use_reduce(portage_dep.paren_reduce(myuse), matchall=1)
1253                         myuse=portage.flatten(myuse)
1254                         # Check each entry to ensure that it exists in PORTDIR's
1255                         # license directory.
1256                         for mypos in range(0,len(myuse)):
1257                                 # Need to check for "||" manually as no portage
1258                                 # function will remove it without removing values.
1259                                 if myuse[mypos] not in liclist and myuse[mypos] != "||":
1260                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1261                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1262
1263                 #keyword checks
1264                 myuse = myaux["KEYWORDS"].split()
1265                 for mykey in myuse:
1266                         myskey=mykey[:]
1267                         if myskey[0]=="-":
1268                                 myskey=myskey[1:]
1269                         if myskey[0]=="~":
1270                                 myskey=myskey[1:]
1271                         if mykey!="-*":
1272                                 if myskey not in kwlist:
1273                                         stats["KEYWORDS.invalid"] += 1
1274                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1275                                 elif not profiles.has_key(myskey):
1276                                         stats["KEYWORDS.invalid"] += 1
1277                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1278
1279                 #syntax checks
1280                 myear = time.gmtime(os.stat(checkdir+"/"+y+".ebuild")[ST_MTIME])[0]
1281                 gentoo_copyright = re.compile(r'^# Copyright ((1999|200\d)-)?' + str(myear) + r' Gentoo Foundation')
1282                 gentoo_license = re.compile(r'^# Distributed under the terms of the GNU General Public License v2$')
1283                 cvs_header = re.compile(r'^#\s*\$Header.*\$$')
1284                 ignore_line = re.compile(r'(^$)|(^(\t)*#)')
1285                 leading_spaces = re.compile(r'^[\S\t]')
1286                 trailing_whitespace = re.compile(r'.*([\S]$)')
1287                 readonly_assignment = re.compile(r'^\s*(export\s+)?(A|CATEGORY|P|PV|PN|PR|PVR|PF|D|WORKDIR|FILESDIR|FEATURES|USE)=')
1288                 continuation_symbol = re.compile(r'(.*[ ]+[\\][ ].*)')
1289                 line_continuation_quoted = re.compile(r'(\"|\')(([\w ,:;#\[\]\.`=/|\$\^\*{}()\'-])|(\\.))*\1')
1290                 line_continuation = re.compile(r'([^#]*\S)(\s+|\t)\\$')
1291                 linenum=0
1292                 previous_line = None
1293                 for line in input(checkdir+"/"+y+".ebuild"):
1294                         linenum += 1
1295                         # Gentoo copyright check
1296                         if linenum == 1:
1297                                 match = gentoo_copyright.match(line)
1298                                 if not match:
1299                                         myerrormsg = "Copyright header Error. Possibly date related."
1300                                         stats["ebuild.badheader"] +=1
1301                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1302                         # Gentoo license check
1303                         elif linenum == 2:
1304                                 match = gentoo_license.match(line)
1305                                 if not match:
1306                                         myerrormsg = "Gentoo License Error."
1307                                         stats["ebuild.badheader"] +=1
1308                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1309                         # CVS Header check
1310                         elif linenum == 3:
1311                                 match = cvs_header.match(line)
1312                                 if not match:
1313                                         myerrormsg = "CVS Header Error."
1314                                         stats["ebuild.badheader"] +=1
1315                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1316                         else:
1317                                 match = ignore_line.match(line)
1318                                 if not match:
1319                                         # Excluded Blank lines and full line comments. Good!
1320                                         # Leading Spaces Check
1321                                         match = leading_spaces.match(line)
1322                                         if not match:
1323                                                 #Line has got leading spaces. Bad!
1324                                                 myerrormsg = "Leading Space Syntax Error. Line %d" % linenum
1325                                                 stats["ebuild.minorsyn"] +=1
1326                                                 fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1327                                         # Trailing whitespace check
1328                                         match = trailing_whitespace.match(line)
1329                                         if not match:
1330                                                 #Line has got trailing whitespace. Bad!
1331                                                 myerrormsg = "Trailing whitespace Syntax Error. Line %d" % linenum
1332                                                 stats["ebuild.minorsyn"] +=1
1333                                                 fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1334                                         # Readonly variable assignment check
1335                                         match = readonly_assignment.match(line)
1336                                         # The regex can give a false positive for continued lines,
1337                                         # so we check the previous line to see if it was continued.
1338                                         if match and (not previous_line or not line_continuation.match(previous_line)):
1339                                                 # invalid assignment, very bad!
1340                                                 myerrormsg = "Readonly variable assignment to %s on line %d" % (match.group(2), linenum)
1341                                                 stats["variable.readonly"] += 1
1342                                                 fails["variable.readonly"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1343                                         # Line continuation check
1344                                         match = continuation_symbol.match(line)
1345                                         if match:
1346                                                 #Excluded lines not even containing a " \" match. Good!
1347                                                 line = re.sub(line_continuation_quoted,"\"\"",line)
1348                                                 #line has been edited to collapsed "" and '' quotes to "". Good!
1349                                                 match = continuation_symbol.match(line)
1350                                                 if match:
1351                                                         #Again exclude lines not even containing a " \" match. Good!
1352                                                         #This repetition is done for a slight performance increase
1353                                                         match = line_continuation.match(line)
1354                                                         if not match:
1355                                                                 #Line has a line continuation error. Bad!
1356                                                                 myerrormsg = "Line continuation (\"\\\") Syntax Error. Line %d" % linenum
1357                                                                 stats["ebuild.majorsyn"] +=1
1358                                                                 fails["ebuild.majorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1359                         previous_line = line
1360                 del previous_line
1361
1362         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1363         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1364         #if not portage.portdb.xmatch("bestmatch-visible",x):
1365         #    stats["ebuild.nostable"]+=1
1366         #    fails["ebuild.nostable"].append(x)
1367         if allmasked and repolevel == 3:
1368                 stats["ebuild.allmasked"]+=1
1369                 fails["ebuild.allmasked"].append(x)
1370
1371 #Pickle and save results for instant reuse in last and lfull
1372 savef=open('/var/cache/edb/repo.stats','w')
1373 pickle.dump(stats,savef)
1374 savef.close()
1375 savef=open('/var/cache/edb/repo.fails','w')
1376 pickle.dump(fails,savef)
1377 savef.close()
1378 if not (os.stat('/var/cache/edb/repo.stats')[ST_GID] == getgrnam('portage')[2]):
1379         os.chown('/var/cache/edb/repo.stats',os.geteuid(),getgrnam('portage')[2])
1380         os.chmod('/var/cache/edb/repo.stats',0664)
1381 if not (os.stat('/var/cache/edb/repo.fails')[ST_GID] == getgrnam('portage')[2]):
1382         os.chown('/var/cache/edb/repo.fails',os.geteuid(),getgrnam('portage')[2])
1383         os.chmod('/var/cache/edb/repo.fails',0664)
1384 if quiet < 2:
1385         print
1386 #dofail will be set to 1 if we have failed in at least one non-warning category
1387 dofail=0
1388 #dowarn will be set to 1 if we tripped any warnings
1389 dowarn=0
1390 #dofull will be set if we should print a "repoman full" informational message
1391 dofull=0
1392 for x in qacats:
1393         if not isCvs and (string.find(x, "notadded") != -1):
1394                 stats[x] = 0
1395         if stats[x]:
1396                 dowarn=1
1397                 if x not in qawarnings:
1398                         dofail=1
1399         else:
1400                 continue
1401         print "  "+string.ljust(x,30),
1402         if stats[x]==0:
1403                 print green(`stats[x]`)
1404                 continue
1405         elif x in qawarnings:
1406                 print yellow(`stats[x]`)
1407         else:
1408                 print red(`stats[x]`)
1409         if mymode!="full":
1410                 if stats[x]<12:
1411                         for y in fails[x]:
1412                                 print "   "+y
1413                 else:
1414                         dofull=1
1415         else:
1416                 for y in fails[x]:
1417                         print "   "+y
1418
1419 def grouplist(mylist,seperator="/"):
1420         """(list,seperator="/") -- Takes a list of elements; groups them into
1421         same initial element categories. Returns a dict of {base:[sublist]}
1422         From: ["blah/foo","spork/spatula","blah/weee/splat"]
1423         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
1424         mygroups={}
1425         for x in mylist:
1426                 xs=string.split(x,seperator)
1427                 if xs[0]==".":
1428                         xs=xs[1:]
1429                 if xs[0] not in mygroups.keys():
1430                         mygroups[xs[0]]=[string.join(xs[1:],seperator)]
1431                 else:
1432                         mygroups[xs[0]]+=[string.join(xs[1:],seperator)]
1433         return mygroups
1434
1435 if mymode!="commit":
1436         if dofull:
1437                 print bold("Note: type \"repoman full\" for a complete listing.")
1438                 if quiet < 1:
1439                         print
1440         if dowarn and not dofail:
1441                 if quiet < 2:
1442                         print green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\""
1443                 else:
1444                         print green("RepoMan sez:"),"\"OK for now, but I'll be back ...\""
1445         elif not dofail:
1446                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
1447         if quiet < 1:
1448                 print
1449 else:
1450         if dofail:
1451                 print turquoise("Please fix these important QA issues first.")
1452                 print green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n"
1453                 sys.exit(1)
1454
1455         if "--pretend" in myoptions:
1456                 print green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n"
1457
1458         if fails["digest.missing"]:
1459                 print green("Creating missing digests...")
1460         for x in fails["digest.missing"]:
1461                 xs=string.split(x,"/")
1462                 del xs[-2]
1463                 myeb=string.join(xs[:-1],"/")+"/"+xs[-1][7:]
1464                 if "--pretend" in myoptions:
1465                         print "(ebuild "+portdir+"/"+myeb+".ebuild digest)"
1466                 else:
1467                         retval=os.system("ebuild "+portdir+"/"+myeb+".ebuild digest")
1468                         if retval:
1469                                 print "!!! Exiting on ebuild digest (shell) error code:",retval
1470                                 sys.exit(retval)
1471
1472         mycvstree=cvstree.getentries("./",recursive=1)
1473         if isCvs and not mycvstree:
1474                 print "!!! It seems we don't have a cvs tree?"
1475                 sys.exit(3)
1476
1477         myunadded=cvstree.findunadded(mycvstree,recursive=1,basedir="./")
1478         myautoadd=[]
1479         if myunadded:
1480                 for x in range(len(myunadded)-1,-1,-1):
1481                         xs=string.split(myunadded[x],"/")
1482                         if xs[-1]=="files":
1483                                 print "!!! files dir is not added! Please correct this."
1484                                 sys.exit(-1)
1485                         elif xs[-1]=="Manifest":
1486                                 # It's a manifest... auto add
1487                                 myautoadd+=[myunadded[x]]
1488                                 del myunadded[x]
1489                         elif len(xs[-1])>=7:
1490                                 if xs[-1][:7]=="digest-":
1491                                         del xs[-2]
1492                                         myeb=string.join(xs[:-1]+[xs[-1][7:]],"/")+".ebuild"
1493                                         if os.path.exists(myeb):
1494                                                 # Ebuild exists for digest... So autoadd it.
1495                                                 myautoadd+=[myunadded[x]]
1496                                                 del myunadded[x]
1497
1498         if myautoadd:
1499                 print ">>> Auto-Adding missing digests..."
1500                 if "--pretend" in myoptions:
1501                         print "(/usr/bin/cvs add "+string.join(myautoadd)+")"
1502                         retval=0
1503                 else:
1504                         retval=os.system("/usr/bin/cvs add "+string.join(myautoadd))
1505                 if retval:
1506                         print "!!! Exiting on cvs (shell) error code:",retval
1507                         sys.exit(retval)
1508
1509         if myunadded:
1510                 print red("!!! The following files are in your cvs tree but are not added to the master")
1511                 print red("!!! tree. Please remove them from the cvs tree or add them to the master tree.")
1512                 for x in myunadded:
1513                         print "   ",x
1514                 print
1515                 print
1516                 sys.exit(1)
1517
1518         retval=["",""]
1519         if isCvs:
1520                 print "Performing a "+green("cvs -n up")+" with a little magic grep to check for updates."
1521                 retval=getstatusoutput("/usr/bin/cvs -n up 2>&1 | egrep '^[^\?] .*' | egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'")
1522
1523         mylines=string.split(retval[1], "\n")
1524         myupdates=[]
1525         for x in mylines:
1526                 if not x:
1527                         continue
1528                 if x[0] not in "UPMAR": # Updates,Patches,Modified,Added,Removed
1529                         print red("!!! Please fix the following issues reported from cvs: ")+green("(U,P,M,A,R are ok)")
1530                         print red("!!! Note: This is a pretend/no-modify pass...")
1531                         print retval[1]
1532                         print
1533                         sys.exit(1)
1534                 elif x[0] in ["U","P"]:
1535                         myupdates+=[x[2:]]
1536
1537         if myupdates:
1538                 print green("Fetching trivial updates...")
1539                 if "--pretend" in myoptions:
1540                         print "(/usr/bin/cvs up "+string.join(myupdates)+")"
1541                         retval=0
1542                 else:
1543                         retval=os.system("/usr/bin/cvs up "+string.join(myupdates))
1544                 if retval!=0:
1545                         print "!!! cvs exited with an error. Terminating."
1546                         sys.exit(retval)
1547
1548         if isCvs:
1549                 mycvstree=cvstree.getentries("./",recursive=1)
1550                 mychanged=cvstree.findchanged(mycvstree,recursive=1,basedir="./")
1551                 for manifest in [file for file in mychanged if '/Manifest' in file]:
1552                         mychanged.remove(manifest)
1553                 mynew=cvstree.findnew(mycvstree,recursive=1,basedir="./")
1554                 myremoved=cvstree.findremoved(mycvstree,recursive=1,basedir="./")
1555                 if not (mychanged or mynew or myremoved):
1556                         print
1557                         print green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"\n"
1558                         print
1559                         print "(Didn't find any changed files...)"
1560                         print
1561                         sys.exit(0)
1562
1563         myupdates=mychanged+mynew
1564         myheaders=[]
1565         mydirty=[]
1566         headerstring="'\$(Header|Id)"
1567         headerstring+=".*\$'"
1568         for myfile in myupdates:
1569                 myout=getstatusoutput("egrep -q "+headerstring+" "+myfile)
1570                 if myout[0]==0:
1571                         myheaders.append(myfile)
1572
1573         print "*",green(str(len(myupdates))),"files being committed...",green(str(len(myheaders))),"have headers that will change."
1574         print "*","Files with headers will cause the manifests to be made and recommited."
1575         if quiet == 0:
1576                 print "myupdates:",myupdates
1577                 print "myheaders:",myheaders
1578                 print
1579         unlinkfile=0
1580         if commitmessagefile:
1581                 try:
1582                         f = open(commitmessagefile)
1583                         commitmessage = f.read()
1584                         f.close()
1585                         del f
1586                 except (IOError, OSError), e:
1587                         if e.errno == errno.ENOENT:
1588                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % commitmessagefile)
1589                         else:
1590                                 raise
1591                 # We've read the content so the file is no longer needed.
1592                 commitmessagefile = None
1593         if not commitmessage:
1594                 print "Please enter a CVS commit message at the prompt:"
1595                 while not commitmessage:
1596                         try:
1597                                 commitmessage=raw_input(green("> "))
1598                         except KeyboardInterrupt:
1599                                 exithandler()
1600         try:
1601                 commitmessage+="\n(Portage version: "+str(portage.VERSION)+")"
1602         except AttributeError:
1603                 print "Failed to insert portage version in message!"
1604                 commitmessage+="\n(Portage version: Unknown)"
1605         if not commitmessagefile:
1606                 unlinkfile=1
1607                 commitmessagefile=tempfile.mktemp(".repoman.msg")
1608                 if os.path.exists(commitmessagefile):
1609                         os.unlink(commitmessagefile)
1610                 mymsg=open(commitmessagefile,"w")
1611                 mymsg.write(commitmessage)
1612                 mymsg.close()
1613
1614                 print
1615                 print green("Using commit message:")
1616                 print green("------------------------------------------------------------------------------")
1617                 print commitmessage
1618                 print green("------------------------------------------------------------------------------")
1619                 print
1620
1621                 retval = None
1622                 if "--pretend" in myoptions:
1623                         print "(/usr/bin/cvs -q commit -F "+ commitmessagefile +" "+ string.join(myupdates," ")+")"
1624                 else:
1625                         retval=os.system("/usr/bin/cvs -q commit -F "+ commitmessagefile + " " +string.join(myupdates, " "))
1626                         if retval:
1627                                 print "!!! Exiting on cvs (shell) error code:",retval
1628                                 sys.exit(retval)
1629
1630         # Setup the GPG commands
1631         def gpgsign(filename):
1632                 if "PORTAGE_GPG_KEY" not in repoman_settings:
1633                         raise portage_exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
1634                 if "PORTAGE_GPG_DIR" not in repoman_settings:
1635                         if os.environ.has_key("HOME"):
1636                                 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(os.environ["HOME"], ".gnupg")
1637                                 if quiet < 1:
1638                                         print "Automatically setting PORTAGE_GPG_DIR to",repoman_settings["PORTAGE_GPG_DIR"]
1639                         else:
1640                                 raise portage_exception.MissingParameter("PORTAGE_GPG_DIR is unset!")
1641                 gpg_dir = repoman_settings["PORTAGE_GPG_DIR"]
1642                 if gpg_dir.startswith("~") and "HOME" in os.environ:
1643                         repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(
1644                                 os.environ["HOME"], gpg_dir[1:].lstrip(os.path.sep))
1645                 if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
1646                         raise portage_exception.InvalidLocation(
1647                                 "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
1648                                 repoman_settings["PORTAGE_GPG_DIR"])
1649                 gpgcmd = "gpg --sign --clearsign --yes "
1650                 gpgcmd+= "--default-key "+repoman_settings["PORTAGE_GPG_KEY"]
1651                 if repoman_settings.has_key("PORTAGE_GPG_DIR"):
1652                         gpgcmd += " --homedir "+repoman_settings["PORTAGE_GPG_DIR"]
1653                 if "--pretend" in myoptions:
1654                         print "("+gpgcmd+" "+filename+")"
1655                 else:
1656                         rValue = os.system(gpgcmd+" "+filename)
1657                         if rValue == os.EX_OK:
1658                                 os.rename(filename+".asc", filename)
1659                         else:
1660                                 raise portage_exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
1661
1662         need_commit = False
1663         if myheaders or myupdates or myremoved or mynew:
1664                 myfiles=myheaders+myupdates+myremoved+mynew
1665                 for x in range(len(myfiles)-1, -1, -1):
1666                         if myfiles[x].count("/") < 4-repolevel:
1667                                 del myfiles[x]
1668                 mydone=[]
1669                 if repolevel==3:   # In a package dir
1670                         repoman_settings["O"]=os.getcwd()
1671                         portage.digestgen([], repoman_settings, manifestonly=1,
1672                                 myportdb=portdb)
1673                 elif repolevel==2: # In a category dir
1674                         for x in myfiles:
1675                                 xs=string.split(x,"/")
1676                                 if xs[0]==".":
1677                                         xs=xs[1:]
1678                                 if xs[0] in mydone:
1679                                         continue
1680                                 mydone.append(xs[0])
1681                                 repoman_settings["O"]=os.path.join(os.getcwd(), xs[0])
1682                                 portage.digestgen([], repoman_settings, manifestonly=1,
1683                                         myportdb=portdb)
1684                 elif repolevel==1: # repo-cvsroot
1685                         print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
1686                         for x in myfiles:
1687                                 xs=string.split(x,"/")
1688                                 if xs[0]==".":
1689                                         xs=xs[1:]
1690                                 if string.join(xs[:2],"/") in mydone:
1691                                         continue
1692                                 mydone.append(string.join(xs[:2],"/"))
1693                                 repoman_settings["O"]=os.path.join(os.getcwd(), x[0], x[1])
1694                                 portage.digestgen([], repoman_settings, manifestonly=1,
1695                                         myportdb=portdb)
1696                 else:
1697                         print red("I'm confused... I don't know where I am!")
1698                         sys.exit(1)
1699
1700                 # Force an unsigned commit when more than one Manifest needs to be signed.
1701                 if repolevel < 3 and "sign" in repoman_settings.features:
1702                         if "--pretend" in myoptions:
1703                                 print "(/usr/bin/cvs -q commit -F "+commitmessagefile+")"
1704                         else:
1705                                 mymsg=open(commitmessagefile,"w")
1706                                 mymsg.write(commitmessage)
1707                                 mymsg.write("\n (Unsigned Manifest commit)")
1708                                 mymsg.close()
1709                                 retval=os.system("/usr/bin/cvs -q commit -F "+commitmessagefile)
1710                                 if retval:
1711                                         print "!!! Exiting on cvs (shell) error code:",retval
1712                                         sys.exit(retval)
1713                 else:
1714                         need_commit = True
1715
1716         signed = False
1717         if "sign" in repoman_settings.features:
1718                 signed = True
1719                 try:
1720                         if repolevel==3:   # In a package dir
1721                                 repoman_settings["O"] = "."
1722                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1723                         elif repolevel==2: # In a category dir
1724                                 mydone=[]
1725                                 for x in myfiles:
1726                                         xs=string.split(x,"/")
1727                                         if xs[0]==".":
1728                                                 xs=xs[1:]
1729                                         if xs[0] in mydone:
1730                                                 continue
1731                                         mydone.append(xs[0])
1732                                         repoman_settings["O"] = os.path.join(".", xs[0])
1733                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1734                         elif repolevel==1: # repo-cvsroot
1735                                 print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
1736                                 mydone=[]
1737                                 for x in myfiles:
1738                                         xs=string.split(x,"/")
1739                                         if xs[0]==".":
1740                                                 xs=xs[1:]
1741                                         if string.join(xs[:2],"/") in mydone:
1742                                                 continue
1743                                         mydone.append(string.join(xs[:2],"/"))
1744                                         repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
1745                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1746                 except portage_exception.PortageException, e:
1747                         portage.writemsg("!!! %s\n" % str(e))
1748                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
1749                         signed = False
1750
1751         if need_commit or signed:
1752                 if "--pretend" in myoptions:
1753                         print "(/usr/bin/cvs -q commit -F "+commitmessagefile+")"
1754                 else:
1755                         mymsg=open(commitmessagefile,"w")
1756                         mymsg.write(commitmessage)
1757                         if signed:
1758                                 mymsg.write("\n (Signed Manifest commit)")
1759                         else:
1760                                 mymsg.write("\n (Unsigned Manifest commit)")
1761                         mymsg.close()
1762                         retval=os.system("/usr/bin/cvs -q commit -F "+commitmessagefile)
1763                         if retval:
1764                                 print "!!! Exiting on cvs (shell) error code:",retval
1765                                 sys.exit(retval)
1766
1767         if unlinkfile:
1768                 os.unlink(commitmessagefile)
1769         print
1770         if isCvs:
1771                 print "CVS commit complete."
1772         else:
1773                 print "repoman was too scared by not seeing any familiar cvs file that he forgot to commit anything"
1774         print green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n"
1775 sys.exit(0)
1776