bin/repoman

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2006 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 # Next to do: dep syntax checking in mask files
   7 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
   8 # that last one is tricky because multiple profiles need to be checked.
   9
  10 import errno, os, shutil, sys
  11 if not hasattr(__builtins__, "set"):
  12         from sets import Set as set
  13 exename=os.path.basename(sys.argv[0])
  14 version="1.2"
  15
  16 allowed_filename_chars="a-zA-Z0-9._-+:"
  17 allowed_filename_chars_set = {}
  18 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('a'), ord('z')+1)))
  19 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('A'), ord('Z')+1)))
  20 map(allowed_filename_chars_set.setdefault, map(chr, range(ord('0'), ord('9')+1)))
  21 map(allowed_filename_chars_set.setdefault, map(chr, map(ord, [".", "-", "_", "+", ":"])))
  22
  23 import signal,re,pickle,tempfile
  24
  25 os.environ["PORTAGE_LEGACY_GLOBALS"] = "false"
  26 try:
  27         import portage
  28 except ImportError:
  29         from os import path as osp
  30         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  31         import portage
  32 del os.environ["PORTAGE_LEGACY_GLOBALS"]
  33
  34 import portage_checksum
  35 import portage_const
  36 import portage_dep
  37 portage_dep._dep_check_strict = True
  38 import portage_exception
  39 import cvstree
  40 import time
  41 import codecs
  42
  43 from portage_manifest import Manifest
  44 from portage_exception import ParseError
  45 from portage_exec import spawn
  46
  47 from output import bold, darkgreen, darkred, green, nocolor, red, turquoise, yellow
  48
  49 from commands import getstatusoutput
  50 from fileinput import input
  51 from grp import getgrnam
  52 from stat import S_ISDIR, ST_CTIME, ST_GID, ST_MTIME
  53
  54 # A sane umask is needed for files that portage creates.
  55 os.umask(022)
  56 repoman_settings = portage.config(local_config=False,
  57         config_incrementals=portage_const.INCREMENTALS)
  58 repoman_settings.lock()
  59
  60 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
  61         not sys.stdout.isatty():
  62         nocolor()
  63
  64 def warn(txt):
  65         print exename+": "+txt
  66 def err(txt):
  67         warn(txt)
  68         sys.exit(1)
  69
  70 def err_help(txt):
  71         help(exitstatus=-1,helpfulness=0)
  72         warn(txt)
  73         sys.exit(1)
  74
  75 def exithandler(signum=None,frame=None):
  76         sys.stderr.write("\n"+exename+": Interrupted; exiting...\n")
  77         sys.exit(1)
  78         os.kill(0,signal.SIGKILL)
  79 signal.signal(signal.SIGINT,exithandler)
  80
  81 shortmodes={"ci":"commit"}
  82 modeshelp={
  83 "scan"   : "Scan directory tree for QA issues (default)",
  84 "fix"    : "Fix simple QA issues (stray digests, missing digests)",
  85 "full"   : "Scan directory tree for QA issues (full listing)",
  86 "help"   : "Show this screen",
  87 "commit" : "Scan directory tree for QA issues; if OK, commit via cvs",
  88 "last"   : "Remember report from last run",
  89 "lfull"  : "Remember report from last run (full listing)"
  90 }
  91 modes=modeshelp.keys()
  92 repoman_options={
  93 "--commitmsg"      : "Adds a commit message via the command line",
  94 "--commitmsgfile"  : "Adds a commit message from the specified file",
  95 "--help"           : "Show this screen",
  96 "--ignore-arches"  : "Ignore arch-specific failures (where arch != host)",
  97 "--ignore-masked"  : "Ignore masked packages (not allowed with commit mode)",
  98 "--pretend"        : "Don't commit or fix anything; just show what would be done",
  99 "--quiet"          : "Be less verbose about extraneous info",
 100 "--verbose"        : "Displays every package name while checking",
 101 "--version"        : "Show version info",
 102 "--xmlparse"       : "Forces the metadata.xml parse check to be carried out"
 103 }
 104 repoman_shortoptions={
 105 "-h" : "--help",
 106 "-i" : "--ignore-masked",
 107 "-I" : "--ignore-arches",
 108 "-m" : "--commitmsg",
 109 "-M" : "--commitmsgfile",
 110 "-p" : "--pretend",
 111 "-q" : "--quiet",
 112 "-v" : "--verbose",
 113 "-V" : "--version",
 114 "-x" : "--xmlparse"
 115 }
 116 repoman_shortoptions_rev=dict([(v,k) for (k,v) in repoman_shortoptions.items()])
 117 options=repoman_options.keys()
 118
 119 qahelp={
 120         "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
 121         "digest.partial":"Digest files do not contain all corresponding URI elements",
 122         "digest.assumed":"Existing digest must be assumed correct (Package level only)",
 123         "digestentry.unused":"Digest/Manifest entry has no matching SRC_URI entry",
 124         "digest.fail":"Digest does not match the specified local file",
 125         "digest.stray":"Digest files that do not have a corresponding ebuild",
 126         "digest.missing":"Digest files that are missing (ebuild exists, digest doesn't)",
 127         "digest.disjointed":"Digests not added to cvs when the matching ebuild has been added",
 128         "digest.notadded":"Digests that exist but have not been added to cvs",
 129         "digest.unmatch":"Digests which are incomplete (please check if your USE/ARCH includes all files)",
 130         "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
 131         "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
 132         "changelog.missing":"Missing ChangeLog files",
 133         "ebuild.disjointed":"Ebuilds not added to cvs when the matching digest has been added",
 134         "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
 135         "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
 136         "filedir.missing":"Package lacks a files directory",
 137         "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
 138         "file.size":"Files in the files directory must be under 20k",
 139         "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
 140         "file.UTF8":"File is not UTF8 compliant",
 141         "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
 142         "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
 143         "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
 144         "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
 145         "SLOT.missing":"Ebuilds that have a missing or empty SLOT variable",
 146         "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
 147         "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
 148         "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
 149         "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
 150         "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
 151         "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
 152         "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
 153         "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
 154         "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
 155         "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
 156         "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
 157         "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
 158         "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
 159         "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
 160         "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
 161         "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
 162         "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
 163         "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
 164         "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
 165         "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error",
 166         "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
 167         "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
 168         "variable.readonly":"Assigning a readonly variable",
 169         "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
 170         "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file",
 171         "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
 172         "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
 173         "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
 174         "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
 175         "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
 176         "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
 177         "ebuild.badheader":"This ebuild has a malformed header",
 178         "metadata.missing":"Missing metadata.xml files",
 179         "metadata.bad":"Bad metadata.xml files",
 180         "virtual.versioned":"PROVIDE contains virtuals with versions",
 181         "virtual.exists":"PROVIDE contains existing package names",
 182         "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default",
 183         "usage.obsolete":"The ebuild makes use of an obsolete construct"
 184 }
 185
 186 qacats = qahelp.keys()
 187 qacats.sort()
 188
 189 qawarnings=[
 190 "changelog.missing",
 191 "changelog.notadded",
 192 "ebuild.notadded",
 193 "ebuild.nostable",
 194 "ebuild.allmasked",
 195 "ebuild.nesteddie",
 196 "digest.assumed",
 197 "digest.notadded",
 198 "digest.disjointed",
 199 "digest.missing",
 200 "digestentry.unused",
 201 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
 202 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
 203 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
 204 "IUSE.invalid",
 205 "KEYWORDS.stupid",
 206 "KEYWORDS.missing",
 207 "ebuild.minorsyn",
 208 "ebuild.badheader",
 209 "file.size",
 210 "metadata.missing",
 211 "metadata.bad",
 212 "virtual.versioned",
 213 "virtual.exists",
 214 "virtual.unavailable",
 215 "usage.obsolete",
 216 "LIVEVCS.stable"
 217 ]
 218
 219 missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"]
 220 allvars=portage.auxdbkeys
 221 commitmessage=None
 222 commitmessagefile=None
 223 for x in missingvars:
 224         x += ".missing"
 225         if x not in qacats:
 226                 print "* missingvars values need to be added to qahelp ("+x+")"
 227                 qacats.append(x)
 228                 qawarnings.append(x)
 229
 230
 231 verbose=0
 232 quiet=0
 233
 234 def show_version():
 235         print exename+" "+version
 236         sys.exit(0)
 237
 238 def help(exitstatus=1,helpfulness=1):
 239         if quiet:
 240                 helpfulness=0
 241         if helpfulness:
 242                 print
 243                 print green(exename+" "+version)
 244                 print " \"Quality is job zero.\""
 245                 print " Copyright 1999-2006 Gentoo Foundation"
 246                 print " Distributed under the terms of the GNU General Public License v2"
 247                 print
 248         print bold(" Usage:"),turquoise(exename),"[",green("options"),"] [",green("mode"),"]"
 249         if helpfulness:
 250                 print bold(" Modes:"),turquoise("scan (default)"),
 251                 for x in modes[1:]:
 252                         print "|",turquoise(x),
 253                 print
 254         print
 255         print " "+green("Options".ljust(20)+" Description")
 256         for x in options:
 257                 if repoman_shortoptions_rev.has_key(x):
 258                         shopt=repoman_shortoptions_rev[x]+", "+x
 259                 else:
 260                         shopt="    "+x
 261                 print " "+shopt.ljust(20),repoman_options[x]
 262         print
 263         print " "+green("Modes".ljust(20)+" Description")
 264         for x in modes:
 265                 print " "+x.ljust(20),modeshelp[x]
 266         if helpfulness:
 267                 print
 268                 print " "+green("QA keyword".ljust(20)+" Description")
 269                 for x in qacats:
 270                         print " "+x.ljust(20),qahelp[x]
 271                 print
 272         if (exitstatus != -1):
 273                 sys.exit(exitstatus)
 274         else:
 275                 print
 276
 277 def last():
 278         try:
 279                 #Retrieve and unpickle stats and fails from saved files
 280                 savedf=open('/var/cache/edb/repo.stats','r')
 281                 stats = pickle.load(savedf)
 282                 savedf.close()
 283                 savedf=open('/var/cache/edb/repo.fails','r')
 284                 fails = pickle.load(savedf)
 285                 savedf.close()
 286         except SystemExit, e:
 287                 raise  # Need to propogate this
 288         except:
 289                 err("Error retrieving last repoman run data; exiting.")
 290
 291         #dofail will be set to 1 if we have failed in at least one non-warning category
 292         dofail=0
 293         #dowarn will be set to 1 if we tripped any warnings
 294         dowarn=0
 295         #dofull will be set if we should print a "repoman full" informational message
 296         dofull=0
 297
 298         print
 299         print green("RepoMan remembers...")
 300         print
 301         for x in qacats:
 302                 if stats[x]:
 303                         dowarn=1
 304                         if x not in qawarnings:
 305                                 dofail=1
 306                 else:
 307                         continue
 308                 print "  "+ x.ljust(20),
 309                 if stats[x]==0:
 310                         print green(`stats[x]`)
 311                         continue
 312                 elif x in qawarnings:
 313                         print yellow(`stats[x]`)
 314                 else:
 315                         print red(`stats[x]`)
 316                 if mymode!="lfull":
 317                         if stats[x]<12:
 318                                 for y in fails[x]:
 319                                         print "   "+y
 320                         else:
 321                                 dofull=1
 322                 else:
 323                         for y in fails[x]:
 324                                 print "   "+y
 325         print
 326         if dofull:
 327                 print bold("Note: type \"repoman lfull\" for a complete listing of repomans last run.")
 328                 print
 329         if dowarn and not dofail:
 330                 print green("RepoMan sez:"),"\"You only gave me a partial QA payment last time?\n              I took it, but I wasn't happy.\""
 331         elif not dofail:
 332                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
 333         print
 334         sys.exit(1)
 335
 336 mymode=None
 337 myoptions=[]
 338 if len(sys.argv)>1:
 339         x=1
 340         while x < len(sys.argv):
 341                 if sys.argv[x] in shortmodes.keys():
 342                         sys.argv[x]=shortmodes[sys.argv[x]]
 343                 elif sys.argv[x] in repoman_shortoptions.keys():
 344                         sys.argv[x] = repoman_shortoptions[sys.argv[x]]
 345                 if sys.argv[x] in modes:
 346                         if mymode is None:
 347                                 mymode=sys.argv[x]
 348                         else:
 349                                 err("Please specify either \""+mymode+"\" or \""+sys.argv[x]+"\", but not both.")
 350                 elif sys.argv[x] in options:
 351                         optionx=sys.argv[x]
 352                         if (optionx=="--commitmsg") and (len(sys.argv)>=(x+1)):
 353                                 commitmessage=sys.argv[x+1]
 354                                 x=x+1
 355                         elif (optionx=="--commitmsgfile") and (len(sys.argv)>=(x+1)):
 356                                 commitmessagefile=sys.argv[x+1]
 357                                 x=x+1
 358                         elif (optionx=="--verbose"):
 359                                 verbose+=1
 360                         elif (optionx=="--quiet"):
 361                                 quiet+=1
 362                         elif optionx not in myoptions:
 363                                 myoptions.append(optionx)
 364                 else:
 365                         err_help("\""+sys.argv[x]+"\" is not a valid mode or option.")
 366                 x=x+1
 367 if mymode is None:
 368         mymode="scan"
 369 if mymode=="help" or ("--help" in myoptions):
 370         help(exitstatus=0)
 371 if ("--version" in myoptions):
 372         show_version()
 373 if mymode=="last" or (mymode=="lfull"):
 374         last()
 375 if mymode == "commit":
 376         while "--ignore-masked" in myoptions:
 377                 myoptions.remove("--ignore-masked")
 378
 379 from portage import normalize_path
 380 isCvs=False
 381 myreporoot=None
 382 if os.path.isdir("CVS"):
 383         isCvs = True
 384
 385 if mymode == "commit" and \
 386         not isCvs and \
 387         "--pretend" not in myoptions:
 388         print
 389         print darkgreen("Not in a CVS repository; enabling pretend mode.")
 390         myoptions.append("--pretend");
 391
 392
 393 def have_profile_dir(path, maxdepth=3):
 394         while path != "/" and maxdepth:
 395                 if os.path.exists(path + "/profiles/package.mask"):
 396                         return normalize_path(path)
 397                 path = normalize_path(path + "/..")
 398                 maxdepth -= 1
 399
 400 portdir=None
 401 portdir_overlay=None
 402 mydir=os.getcwd()
 403 if "PWD" in os.environ and os.environ["PWD"] != mydir and \
 404         os.path.realpath(os.environ["PWD"]) == mydir:
 405         # getcwd() returns the canonical path but that makes it hard for repoman to
 406         # orient itself if the user has symlinks in their portage tree structure.
 407         # We use os.environ["PWD"], if available, to get the non-canonical path of
 408         # the current working directory (from the shell).
 409         mydir = os.environ["PWD"]
 410 mydir = normalize_path(mydir)
 411 if mydir[-1] != "/":
 412         mydir += "/"
 413
 414 for overlay in repoman_settings["PORTDIR_OVERLAY"].split():
 415         if overlay[-1] != "/":
 416                 overlay += "/"
 417         if mydir.startswith(overlay):
 418                 portdir_overlay = overlay
 419                 subdir = mydir[len(overlay):]
 420                 if subdir and subdir[-1] != "/":
 421                         subdir += "/"
 422                 if have_profile_dir(mydir, subdir.count("/")):
 423                         portdir = portdir_overlay
 424                 break
 425
 426 if not portdir_overlay:
 427         if (repoman_settings["PORTDIR"] + os.path.sep).startswith(mydir):
 428                 portdir_overlay = repoman_settings["PORTDIR"]
 429         else:
 430                 portdir_overlay = have_profile_dir(mydir)
 431         portdir = portdir_overlay
 432
 433 if not portdir_overlay:
 434         sys.stderr.write("Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY from" + \
 435                 " the current\nworking directory.\n")
 436         sys.exit(1)
 437
 438 if not portdir:
 439         portdir = repoman_settings["PORTDIR"]
 440
 441 portdir = normalize_path(portdir)
 442 portdir_overlay = normalize_path(portdir_overlay)
 443
 444 os.environ["PORTDIR"] = portdir
 445 if portdir_overlay != portdir:
 446         os.environ["PORTDIR_OVERLAY"] = portdir_overlay
 447 else:
 448         os.environ["PORTDIR_OVERLAY"] = ""
 449
 450 if quiet < 2:
 451         print "\nSetting paths:"
 452         print "PORTDIR = \""+os.environ["PORTDIR"]+"\""
 453         print "PORTDIR_OVERLAY = \""+os.environ["PORTDIR_OVERLAY"]+"\""
 454
 455 # Now that PORTDIR_OVERLAY is properly overridden, create the portdb.
 456 repoman_settings = portage.config(local_config=False,
 457         config_incrementals=portage_const.INCREMENTALS)
 458 trees = portage.create_trees()
 459 trees["/"]["porttree"].settings = repoman_settings
 460 portdb = trees["/"]["porttree"].dbapi
 461 portdb.mysettings = repoman_settings
 462 # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman.
 463 del trees["/"]["vartree"]
 464
 465 if not myreporoot:
 466         myreporoot = os.path.basename(portdir_overlay)
 467         myreporoot += mydir[len(portdir_overlay):-1]
 468
 469 reposplit=myreporoot.split("/")
 470 repolevel=len(reposplit)
 471
 472 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 473 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
 474 # this check ensure that repoman knows where it is, and the manifest recommit is at least possible.
 475 if mymode == "commit" and repolevel not in [1,2,3]:
 476         print red("***")+" Commit attempts *must* be from within a cvs co, category, or package directory."
 477         print red("***")+" Attempting to commit from a packages files directory will be blocked for instance."
 478         print red("***")+" This is intended behaviour, to ensure the manifest is recommited for a package."
 479         print red("***")
 480         err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
 481
 482 startdir = normalize_path(mydir)
 483 repodir = startdir
 484 for x in range(0,repolevel-1):
 485         repodir = os.path.dirname(repodir)
 486
 487 def caterror(mycat):
 488         err(mycat+" is not an official category.  Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
 489 if "--pretend" in myoptions:
 490         print green("\nRepoMan does a once-over of the neighborhood...")
 491 elif quiet < 1:
 492         print green("\nRepoMan scours the neighborhood...")
 493
 494 def parse_use_local_desc(mylines, usedict=None):
 495         """returns a dict of the form {cpv:set(flags)}"""
 496         if usedict is None:
 497                 usedict = {}
 498         lineno = 0
 499         for l in mylines:
 500                 lineno += 1
 501                 if not l or l.startswith("#"):
 502                         continue
 503                 mysplit = l.split(None, 1)
 504                 if not mysplit:
 505                         continue
 506                 mysplit = mysplit[0].split(":")
 507                 if len(mysplit) != 2:
 508                         raise ParseError("line %d: Malformed input: '%s'" % \
 509                                 (lineno, l.rstrip("\n")))
 510                 usedict.setdefault(mysplit[0], set())
 511                 usedict[mysplit[0]].add(mysplit[1])
 512         return usedict
 513
 514 # retreive local USE list
 515 luselist={}
 516 try:
 517         f = open(os.path.join(portdir, "profiles", "use.local.desc"))
 518         parse_use_local_desc(f, luselist)
 519         f.close()
 520 except (IOError, OSError, ParseError), e:
 521         print >> sys.stderr, str(e)
 522         err("Couldn't read from use.local.desc")
 523
 524 if portdir_overlay != portdir:
 525         filename = os.path.join(portdir_overlay, "profiles", "use.local.desc")
 526         if os.path.exists(filename):
 527                 try:
 528                         f = open(filename)
 529                         parse_use_local_desc(f, luselist)
 530                         f.close()
 531                 except (IOError, OSError, ParseError), e:
 532                         print >> sys.stderr, str(e)
 533                         err("Couldn't read from '%s'" % filename)
 534         del filename
 535
 536 # setup a uselist from portage
 537 uselist=[]
 538 try:
 539         uselist=portage.grabfile(portdir+"/profiles/use.desc")
 540         for l in range(0,len(uselist)):
 541                 uselist[l]=uselist[l].split()[0]
 542         for var in repoman_settings["USE_EXPAND"].split():
 543                 vardescs = portage.grabfile(portdir+"/profiles/desc/"+var.lower()+".desc")
 544                 for l in range(0, len(vardescs)):
 545                         uselist.append(var.lower() + "_" + vardescs[l].split()[0])
 546 except SystemExit, e:
 547         raise  # Need to propogate this
 548 except:
 549         err("Couldn't read USE flags from use.desc")
 550
 551 # retrieve a list of current licenses in portage
 552 liclist = set(portage.listdir(os.path.join(portdir, "licenses")))
 553 if not liclist:
 554         err("Couldn't find licenses?")
 555 if portdir_overlay != portdir:
 556         liclist.update(portage.listdir(os.path.join(portdir_overlay, "licenses")))
 557
 558 # retrieve list of offical keywords
 559 kwlist = set(portage.grabfile(os.path.join(portdir, "profiles", "arch.list")))
 560 if not kwlist:
 561         err("Couldn't read KEYWORDS from arch.list")
 562
 563 manifest1_compat = not os.path.exists(
 564         os.path.join(portdir, "manifest1_obsolete"))
 565 if portdir_overlay != portdir:
 566         kwlist.update(portage.grabfile(
 567                 os.path.join(portdir_overlay, "profiles", "arch.list")))
 568         manifest1_compat = not os.path.exists(
 569                 os.path.join(portdir_overlay, "manifest1_obsolete"))
 570
 571 scanlist=[]
 572 if repolevel==2:
 573         #we are inside a category directory
 574         catdir=reposplit[-1]
 575         if catdir not in repoman_settings.categories:
 576                 caterror(catdir)
 577         mydirlist=os.listdir(startdir)
 578         for x in mydirlist:
 579                 if x == "CVS" or x.startswith("."):
 580                         continue
 581                 if os.path.isdir(startdir+"/"+x):
 582                         scanlist.append(catdir+"/"+x)
 583 elif repolevel==1:
 584         for x in repoman_settings.categories:
 585                 if not os.path.isdir(startdir+"/"+x):
 586                         continue
 587                 for y in os.listdir(startdir+"/"+x):
 588                         if y == "CVS" or y.startswith("."):
 589                                 continue
 590                         if os.path.isdir(startdir+"/"+x+"/"+y):
 591                                 scanlist.append(x+"/"+y)
 592 elif repolevel==3:
 593         catdir = reposplit[-2]
 594         if catdir not in repoman_settings.categories:
 595                 caterror(catdir)
 596         scanlist.append(catdir+"/"+reposplit[-1])
 597
 598 profiles={}
 599 descfile=portdir+"/profiles/profiles.desc"
 600 if os.path.exists(descfile):
 601         for x in portage.grabfile(descfile):
 602                 if x[0]=="#":
 603                         continue
 604                 arch=x.split()
 605                 if len(arch)!=3:
 606                         print "wrong format: \""+red(x)+"\" in "+descfile
 607                         continue
 608                 if not os.path.isdir(portdir+"/profiles/"+arch[1]):
 609                         print "Invalid "+arch[2]+" profile ("+arch[1]+") for arch "+arch[0]
 610                         continue
 611                 if profiles.has_key(arch[0]):
 612                         profiles[arch[0]]+= [[arch[1], arch[2]]]
 613                 else:
 614                         profiles[arch[0]] = [[arch[1], arch[2]]]
 615
 616         for x in repoman_settings.archlist():
 617                 if x[0] == "~":
 618                         continue
 619                 if not profiles.has_key(x):
 620                         print red("\""+x+"\" doesn't have a valid profile listed in profiles.desc.")
 621                         print red("You need to either \"cvs update\" your profiles dir or follow this")
 622                         print red("up with the "+x+" team.")
 623                         print
 624 else:
 625         print red("profiles.desc does not exist: "+descfile)
 626         print red("You need to do \"cvs update\" in profiles dir.")
 627         print
 628         sys.exit(1)
 629
 630
 631 stats={}
 632 fails={}
 633
 634 for x in qacats:
 635         stats[x]=0
 636         fails[x]=[]
 637 xmllint_capable = False
 638 if getstatusoutput('which xmllint')[0] != 0:
 639         print red("!!! xmllint not found. Can't check metadata.xml.\n")
 640         if "--xmlparse" in myoptions or repolevel==3:
 641                 print red("!!!")+" sorry, xmllint is needed.  failing\n"
 642                 sys.exit(1)
 643 else:
 644         #hardcoded paths/urls suck. :-/
 645         must_fetch=1
 646         backup_exists=0
 647         try:
 648                 # if it's been over a week since fetching (or the system clock is fscked), grab an updated copy of metadata.dtd
 649                 # clock is fscked or it's been a week. time to grab a new one.
 650                 ct=os.stat(portage.CACHE_PATH + '/metadata.dtd')[ST_CTIME]
 651                 if abs(time.time() - ct) > (60*60*24*7):
 652                         # don't trap the exception, we're watching for errno 2 (file not found), anything else is a bug.
 653                         backup_exists=1
 654                 else:
 655                         must_fetch=0
 656
 657         except (OSError,IOError), e:
 658                 if e.errno != 2:
 659                         print red("!!!")+" caught exception '%s' for %s/metadata.dtd, bailing" % (str(e), portage.CACHE_PATH)
 660                         sys.exit(1)
 661
 662         if must_fetch:
 663                 print
 664                 print green("***")+" the local copy of metadata.dtd needs to be refetched, doing that now"
 665                 print
 666                 try:
 667                         if os.path.exists(repoman_settings["DISTDIR"]+'/metadata.dtd'):
 668                                 os.remove(repoman_settings["DISTDIR"]+'/metadata.dtd')
 669                         val=portage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoman_settings,fetchonly=0, \
 670                                 try_mirrors=0)
 671                         if val:
 672                                 if backup_exists:
 673                                         os.remove(portage.CACHE_PATH+'/metadata.dtd')
 674                                 shutil.copy(repoman_settings["DISTDIR"]+'/metadata.dtd',portage.CACHE_PATH+'/metadata.dtd')
 675                                 os.chown(portage.CACHE_PATH+'/metadata.dtd',os.getuid(),portage.portage_gid)
 676                                 os.chmod(portage.CACHE_PATH+'/metadata.dtd',0664)
 677
 678
 679                 except SystemExit, e:
 680                         raise  # Need to propogate this
 681                 except Exception,e:
 682                         print
 683                         print red("!!!")+" attempting to fetch 'http://www.gentoo.org/dtd/metadata.dtd', caught"
 684                         print red("!!!")+" exception '%s' though." % str(e)
 685                         val=0
 686                 if not val:
 687                         print red("!!!")+" fetching new metadata.dtd failed, aborting"
 688                         sys.exit(1)
 689         #this can be problematic if xmllint changes their output
 690         xmllint_capable=True
 691
 692
 693 def x11_deprecation_check(depstr):
 694         if depstr.find("virtual/x11") == -1:
 695                 return False
 696         depsplit = depstr.split()
 697         ok_stack = []
 698         ok = False
 699         for token in depsplit:
 700                 if token == "(":
 701                         ok_stack.append(ok)
 702                         ok = False
 703                 elif token == ")":
 704                         ok = ok_stack.pop()
 705                 elif token == "||":
 706                         ok = True
 707                 else:
 708                         ok = False
 709                         if token.find("virtual/x11") != -1 and (not ok_stack or not ok_stack[-1]):
 710                                 return True
 711         return False
 712
 713
 714 arch_caches={}
 715 arch_xmatch_caches = {}
 716 for x in scanlist:
 717         #ebuilds and digests added to cvs respectively.
 718         if verbose:
 719                 print "checking package " + x
 720         eadded=[]
 721         dadded=[]
 722         catdir,pkgdir=x.split("/")
 723         checkdir=repodir+"/"+x
 724         checkdirlist=os.listdir(checkdir)
 725         ebuildlist=[]
 726         for y in checkdirlist:
 727                 if y[-7:]==".ebuild":
 728                         ebuildlist.append(y[:-7])
 729                 if y in ["Manifest","ChangeLog","metadata.xml"]:
 730                         if os.stat(checkdir+"/"+y)[0] & 0x0248:
 731                                 stats["file.executable"] += 1
 732                                 fails["file.executable"].append(checkdir+"/"+y)
 733         digestlist=[]
 734
 735         for y in checkdirlist:
 736                 for c in y.strip(os.path.sep):
 737                         if c not in allowed_filename_chars_set:
 738                                 stats["file.name"] += 1
 739                                 fails["file.name"].append("%s/%s: char '%s'" % (checkdir, y, c))
 740                                 break
 741
 742                 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
 743                         continue
 744                 try:
 745                         line = 1
 746                         for l in codecs.open(checkdir+"/"+y, "r", "utf8"):
 747                                 line +=1
 748                 except UnicodeDecodeError, ue:
 749                         stats["file.UTF8"] += 1
 750                         s = ue.object[:ue.start]
 751                         l2 = s.count("\n")
 752                         line += l2
 753                         if l2 != 0:
 754                                 s = s[s.rfind("\n") + 1:]
 755                         fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
 756
 757         if isCvs:
 758                 try:
 759                         mystat=os.stat(checkdir+"/files")[0]
 760                         if len(ebuildlist) and not S_ISDIR(mystat):
 761                                 raise Exception
 762                 except SystemExit, e:
 763                         raise  # Need to propogate this
 764                 except:
 765                         stats["filedir.missing"] += 1
 766                         fails["filedir.missing"].append(checkdir)
 767                         continue
 768                 try:
 769                         myf=open(checkdir+"/CVS/Entries","r")
 770                         myl=myf.readlines()
 771                         for l in myl:
 772                                 if l[0]!="/":
 773                                         continue
 774                                 splitl=l[1:].split("/")
 775                                 if not len(splitl):
 776                                         continue
 777                                 if splitl[0][-7:]==".ebuild":
 778                                         eadded.append(splitl[0][:-7])
 779                 except IOError:
 780                         if mymode=="commit":
 781                                 stats["CVS/Entries.IO_error"] += 1
 782                                 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
 783                         continue
 784
 785                 try:
 786                         myf=open(checkdir+"/files/CVS/Entries","r")
 787                         myl=myf.readlines()
 788                         for l in myl:
 789                                 if l[0]!="/":
 790                                         continue
 791                                 splitl=l[1:].split("/")
 792                                 if not len(splitl):
 793                                         continue
 794                                 if splitl[0][:7]=="digest-":
 795                                         dadded.append(splitl[0][7:])
 796                 except IOError:
 797                         if mymode=="commit":
 798                                 stats["CVS/Entries.IO_error"] += 1
 799                                 fails["CVS/Entries.IO_error"].append(checkdir+"/files/CVS/Entries")
 800                         continue
 801
 802         if mymode in ("fix", "commit") and \
 803                 "--pretend" not in myoptions:
 804                 repoman_settings["O"] = checkdir
 805                 if not portage.digestgen([], repoman_settings, myportdb=portdb):
 806                         print "Unable to generate manifest."
 807                         sys.exit(1)
 808
 809         mf = Manifest(checkdir, repoman_settings["DISTDIR"])
 810         mydigests=mf.getTypeDigests("DIST")
 811
 812         fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
 813         myfiles_all = []
 814         for mykey in fetchlist_dict:
 815                 try:
 816                         myfiles_all.extend(fetchlist_dict[mykey])
 817                 except portage_exception.InvalidDependString, e:
 818                         stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
 819                         fails["SRC_URI.syntax"].append(mykey+".ebuild SRC_URI: "+str(e))
 820         del fetchlist_dict
 821         myfiles_all = set(myfiles_all)
 822         for entry in mydigests:
 823                 if entry not in myfiles_all:
 824                         stats["digestentry.unused"] += 1
 825                         fails["digestentry.unused"].append(checkdir+"::"+entry)
 826         del myfiles_all
 827
 828         if os.path.exists(checkdir+"/files"):
 829                 filesdirlist=os.listdir(checkdir+"/files")
 830                 if manifest1_compat:
 831                         for y in filesdirlist:
 832                                 if not y.startswith("digest-"):
 833                                         continue
 834                                 if y[7:] not in dadded:
 835                                         #digest not added to cvs
 836                                         stats["digest.notadded"]=stats["digest.notadded"]+1
 837                                         fails["digest.notadded"].append(x+"/files/"+y)
 838                                         if y[7:] in eadded:
 839                                                 stats["digest.disjointed"]=stats["digest.disjointed"]+1
 840                                                 fails["digest.disjointed"].append(x+"/files/"+y)
 841
 842                                 if os.stat(checkdir+"/files/"+y)[0] & 0x0248:
 843                                         stats["file.executable"] += 1
 844                                         fails["file.executable"].append(x+"/files/"+y)
 845
 846                                 mykey = catdir + "/" + y[7:]
 847                                 if y[7:] not in ebuildlist:
 848                                         #stray digest
 849                                         if mymode=="fix":
 850                                                 if "--pretend" in myoptions:
 851                                                         print "(cd "+repodir+"/"+x+"/files; cvs rm -f "+y+")"
 852                                                 else:
 853                                                         os.system("(cd "+repodir+"/"+x+"/files; cvs rm -f "+y+")")
 854                                         else:
 855                                                 stats["digest.stray"]=stats["digest.stray"]+1
 856                                                 fails["digest.stray"].append(x+"/files/"+y)
 857                                 else:
 858                                         # We have an ebuild
 859                                         try:
 860                                                 myuris, myfiles = portdb.getfetchlist(mykey, all=True)
 861                                         except portage_exception.InvalidDependString, e:
 862                                                 # Already handled above.
 863                                                 continue
 864
 865                                         uri_dict = {}
 866                                         for myu in myuris:
 867                                                 myubn = os.path.basename(myu)
 868                                                 if myubn not in uri_dict:
 869                                                         uri_dict[myubn]  = [myu]
 870                                                 else:
 871                                                         uri_dict[myubn] += [myu]
 872
 873                                         for myf in uri_dict:
 874                                                 myff = repoman_settings["DISTDIR"] + "/" + myf
 875                                                 if not mydigests.has_key(myf):
 876                                                         uri_settings = portage.config(clone=repoman_settings)
 877                                                         if mymode == "fix":
 878                                                                 if not portage.fetch(uri_dict[myf], uri_settings):
 879                                                                         stats["digest.unmatch"] += 1
 880                                                                         fails["digest.unmatch"].append(y+"::"+myf)
 881                                                                 else:
 882                                                                         eb_name = portdb.findname2(mykey)[0]
 883                                                                         portage.doebuild(eb_name, "digest", "/",
 884                                                                                 uri_settings, tree="porttree",
 885                                                                                 mydbapi=portdb)
 886                                                         else:
 887                                                                 stats["digest.partial"] += 1
 888                                                                 fails["digest.partial"].append(y+"::"+myf)
 889                                                 elif "assume-digests" not in repoman_settings.features:
 890                                                         if os.path.exists(myff):
 891                                                                 if not portage_checksum.verify_all(myff, mydigests[myf]):
 892                                                                         stats["digest.fail"] += 1
 893                                                                         fails["digest.fail"].append(y+"::"+myf)
 894                                                         elif repolevel == 3:
 895                                                                 stats["digest.assumed"] += 1
 896                                                                 fails["digest.assumed"].append(y+"::"+myf)
 897
 898                 # recurse through files directory
 899                 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
 900                 while filesdirlist:
 901                         y = filesdirlist.pop(0)
 902                         try:
 903                                 mystat = os.stat(checkdir+"/files/"+y)
 904                         except OSError, oe:
 905                                 if oe.errno == 2:
 906                                         # don't worry about it.  it likely was removed via fix above.
 907                                         continue
 908                                 else:
 909                                         raise oe
 910                         if S_ISDIR(mystat.st_mode):
 911                                 if y == "CVS":
 912                                         continue
 913                                 for z in os.listdir(checkdir+"/files/"+y):
 914                                         if z == "CVS":
 915                                                 continue
 916                                         filesdirlist.append(y+"/"+z)
 917                         # current policy is no files over 20k, this is the check.
 918                         elif mystat.st_size > 20480:
 919                                 stats["file.size"] += 1
 920                                 fails["file.size"].append("("+ str(mystat.st_size/1024) + "K) "+x+"/files/"+y)
 921
 922                         for c in os.path.basename(y.rstrip(os.path.sep)):
 923                                 if c not in allowed_filename_chars_set:
 924                                         stats["file.name"] += 1
 925                                         fails["file.name"].append("%s/files/%s: char '%s'" % (checkdir, y, c))
 926                                         break
 927
 928         del mydigests
 929
 930         if "ChangeLog" not in checkdirlist:
 931                 stats["changelog.missing"]+=1
 932                 fails["changelog.missing"].append(x+"/ChangeLog")
 933
 934         #metadata.xml file check
 935         if "metadata.xml" not in checkdirlist:
 936                 stats["metadata.missing"]+=1
 937                 fails["metadata.missing"].append(x+"/metadata.xml")
 938         #metadata.xml parse check
 939         else:
 940                 #Only carry out if in package directory or check forced
 941                 if xmllint_capable:
 942                         st=getstatusoutput("xmllint --nonet --noout --dtdvalid %s/metadata.dtd %s/metadata.xml" % (portage.CACHE_PATH, checkdir))
 943                         if st[0] != 0:
 944                                 for z in st[1].split("\n"):
 945                                         print red("!!! ")+z
 946                                 stats["metadata.bad"]+=1
 947                                 fails["metadata.bad"].append(x+"/metadata.xml")
 948
 949         allmasked = True
 950
 951         for y in ebuildlist:
 952                 if os.stat(checkdir+"/"+y+".ebuild")[0] & 0x0248:
 953                         stats["file.executable"] += 1
 954                         fails["file.executable"].append(x+"/"+y+".ebuild")
 955                 if y not in eadded:
 956                         #ebuild not added to cvs
 957                         stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
 958                         fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
 959                         if y in dadded:
 960                                 stats["ebuild.disjointed"]=stats["ebuild.disjointed"]+1
 961                                 fails["ebuild.disjointed"].append(x+"/"+y+".ebuild")
 962                 if manifest1_compat and \
 963                         not os.path.exists(os.path.join(checkdir, "files", "digest-"+y)):
 964                         if mymode=="fix":
 965                                 if "--pretend" in myoptions:
 966                                         print "You will need to run:"
 967                                         print "  /usr/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest"
 968                                 else:
 969                                         retval=os.system("/usr/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest")
 970                                         if retval:
 971                                                 print "!!! Exiting on ebuild digest (shell) error code:",retval
 972                                                 sys.exit(retval)
 973                         else:
 974                                 stats["digest.missing"]=stats["digest.missing"]+1
 975                                 fails["digest.missing"].append(x+"/files/digest-"+y)
 976                 myesplit=portage.pkgsplit(y)
 977                 if myesplit is None or myesplit[0] != x.split("/")[-1]:
 978                         stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
 979                         fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
 980                         continue
 981                 elif myesplit[0]!=pkgdir:
 982                         print pkgdir,myesplit[0]
 983                         stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
 984                         fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
 985                         continue
 986                 try:
 987                         myaux = dict(
 988                                 zip(allvars, portdb.aux_get(os.path.join(catdir, y), allvars)))
 989                 except KeyError:
 990                         stats["ebuild.syntax"]=stats["ebuild.syntax"]+1
 991                         fails["ebuild.syntax"].append(x+"/"+y+".ebuild")
 992                         continue
 993                 except IOError:
 994                         stats["ebuild.output"]=stats["ebuild.output"]+1
 995                         fails["ebuild.output"].append(x+"/"+y+".ebuild")
 996                         continue
 997
 998                 # Test for negative logic and bad words in the RESTRICT var.
 999                 #for x in myaux[allvars.index("RESTRICT")].split():
1000                 #       if x.startswith("no"):
1001                 #               print "Bad RESTRICT value: %s" % x
1002                 try:
1003                         myaux["PROVIDE"] = portage_dep.use_reduce(
1004                                 portage_dep.paren_reduce(myaux["PROVIDE"]), matchall=1)
1005                 except portage_exception.InvalidDependString, e:
1006                         stats["PROVIDE.syntax"] = stats["PROVIDE.syntax"] + 1
1007                         fails["PROVIDE.syntax"].append(mykey+".ebuild PROVIDE: "+str(e))
1008                         del e
1009                         continue
1010                 myaux["PROVIDE"] = " ".join(portage.flatten(myaux["PROVIDE"]))
1011                 for myprovide in myaux["PROVIDE"].split():
1012                         prov_cp = portage.dep_getkey(myprovide)
1013                         if prov_cp != myprovide:
1014                                 stats["virtual.versioned"]+=1
1015                                 fails["virtual.versioned"].append(x+"/"+y+".ebuild: "+myprovide)
1016                         prov_pkg = portage.dep_getkey(
1017                                 portage.best(portdb.xmatch("match-all", prov_cp)))
1018                         if prov_cp == prov_pkg:
1019                                 stats["virtual.exists"]+=1
1020                                 fails["virtual.exists"].append(x+"/"+y+".ebuild: "+prov_cp)
1021
1022                 for pos in range(0,len(missingvars)):
1023                         if not myaux[missingvars[pos]]:
1024                                 myqakey=missingvars[pos]+".missing"
1025                                 stats[myqakey]=stats[myqakey]+1
1026                                 fails[myqakey].append(x+"/"+y+".ebuild")
1027
1028                 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1029                 if "-*" in myaux["KEYWORDS"].split():
1030                         haskeyword = False
1031                         for kw in myaux["KEYWORDS"].split():
1032                                 if kw[0] == "~":
1033                                         kw = kw[1:]
1034                                 if kw in kwlist:
1035                                         haskeyword = True
1036                         if not haskeyword:
1037                                 stats["KEYWORDS.stupid"] += 1
1038                                 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1039
1040                 """
1041                 Ebuilds that inherit a "Live" eclasss (darcs,subversion,git,cvs,etc..) should
1042                 not be allowed to be marked stable
1043                 """
1044                 if set(["darcs","cvs","subversion","git"]).intersection(
1045                         myaux["INHERITED"].split()):
1046                         bad_stable_keywords = []
1047                         for keyword in myaux["KEYWORDS"].split():
1048                                 if not keyword.startswith("~") and \
1049                                         not keyword.startswith("-"):
1050                                         bad_stable_keywords.append(keyword)
1051                                 del keyword
1052                         if bad_stable_keywords:
1053                                 stats["LIVEVCS.stable"] += 1
1054                                 fails["LIVEVCS.stable"].append(
1055                                         x + "/" + y + ".ebuild with stable keywords:%s " % \
1056                                                 bad_stable_keywords)
1057                         del bad_stable_keywords
1058
1059                 if "--ignore-arches" in myoptions:
1060                         arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1061                                 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1062                 else:
1063                         arches=[]
1064                         for keyword in myaux["KEYWORDS"].split():
1065                                 if (keyword[0]=="-"):
1066                                         continue
1067                                 elif (keyword[0]=="~"):
1068                                         arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1069                                 else:
1070                                         arches.append([keyword, keyword, [keyword]])
1071                                         allmasked = False
1072
1073                 baddepsyntax = False
1074                 badlicsyntax = False
1075                 badprovsyntax = False
1076                 catpkg = catdir+"/"+y
1077                 myiuse = set(repoman_settings.archlist())
1078                 for myflag in myaux["IUSE"].split():
1079                         if myflag.startswith("+"):
1080                                 myflag = myflag[1:]
1081                         myiuse.add(myflag)
1082
1083                 operator_tokens = set(["||", "(", ")"])
1084                 type_list, badsyntax = [], []
1085                 for mytype in ("DEPEND", "RDEPEND", "PDEPEND", "LICENSE", "PROVIDE"):
1086                         mydepstr = myaux[mytype]
1087
1088                         if mydepstr.find(" ?") != -1:
1089                                 badsyntax.append("'?' preceded by space")
1090
1091                         try:
1092                                 # Missing closing parenthesis will result in a ValueError
1093                                 mydeplist = portage_dep.paren_reduce(mydepstr)
1094                                 # Missing opening parenthesis will result in a final "" element
1095                                 if "" in mydeplist or "(" in mydeplist:
1096                                         raise ValueError
1097                         except ValueError:
1098                                 badsyntax.append("parenthesis mismatch")
1099                                 mydeplist = []
1100
1101                         try:
1102                                 portage_dep.use_reduce(mydeplist, excludeall=myiuse)
1103                         except portage_exception.InvalidDependString, e:
1104                                 badsyntax.append(str(e))
1105
1106                         for token in operator_tokens:
1107                                 if mydepstr.startswith(token+" "):
1108                                         myteststr = mydepstr[len(token):]
1109                                 else:
1110                                         myteststr = mydepstr
1111                                 if myteststr.endswith(" "+token):
1112                                         myteststr = myteststr[:-len(token)]
1113                                 while myteststr.find(" "+token+" ") != -1:
1114                                         myteststr = " ".join(myteststr.split(" "+token+" ", 1))
1115                                 if myteststr.find(token) != -1:
1116                                         badsyntax.append("'%s' not separated by space" % (token))
1117
1118
1119                         if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1120                                 for token in mydepstr.split():
1121                                         if token in operator_tokens or \
1122                                                 token.endswith("?"):
1123                                                 continue
1124                                         if not portage.isvalidatom(token, allow_blockers=True) or \
1125                                                 ":" in token and myaux["EAPI"] == "0":
1126                                                 badsyntax.append("'%s' not a valid atom" % token)
1127
1128                         type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1129
1130                 for m,b in zip(type_list, badsyntax):
1131                         stats[m+".syntax"] += 1
1132                         fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1133
1134                 badlicsyntax = len(filter(lambda x:x=="LICENSE", type_list))
1135                 badprovsyntax = len(filter(lambda x:x=="PROVIDE", type_list))
1136                 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1137                 badlicsyntax = badlicsyntax > 0
1138                 badprovsyntax = badprovsyntax > 0
1139
1140                 if not baddepsyntax:
1141                         if x11_deprecation_check(" ".join([myaux["DEPEND"], myaux["RDEPEND"], myaux["PDEPEND"]])):
1142                                 stats["usage.obsolete"] += 1
1143                                 fails["usage.obsolete"].append("%s/%s.ebuild: not migrated to modular X" % (x, y))
1144
1145                 for keyword,arch,groups in arches:
1146
1147                         if not profiles.has_key(arch):
1148                                 # A missing profile will create an error further down
1149                                 # during the KEYWORDS verification.
1150                                 continue
1151
1152                         for prof in profiles[arch]:
1153
1154                                 profdir = portdir+"/profiles/"+prof[0]
1155
1156                                 if prof[0] in arch_caches:
1157                                         dep_settings = arch_caches[prof[0]]
1158                                 else:
1159                                         dep_settings = portage.config(
1160                                                 config_profile_path=profdir,
1161                                                 config_incrementals=portage_const.INCREMENTALS,
1162                                                 local_config=False)
1163                                         arch_caches[prof[0]] = dep_settings
1164                                         while True:
1165                                                 try:
1166                                                         # Protect ACCEPT_KEYWORDS from config.regenerate()
1167                                                         # (just in case)
1168                                                         dep_settings.incrementals.remove("ACCEPT_KEYWORDS")
1169                                                 except ValueError:
1170                                                         break
1171
1172                                 xmatch_cache_key = (prof[0], tuple(groups))
1173                                 xcache = arch_xmatch_caches.get(xmatch_cache_key)
1174                                 if xcache is None:
1175                                         portdb.melt()
1176                                         portdb.freeze()
1177                                         xcache = portdb.xcache
1178                                         arch_xmatch_caches[xmatch_cache_key] = xcache
1179
1180                                 trees["/"]["porttree"].settings = dep_settings
1181                                 portdb.mysettings = dep_settings
1182                                 portdb.xcache = xcache
1183                                 # for package.use.mask support inside dep_check
1184                                 dep_settings.setcpv("/".join((catdir, y)))
1185                                 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1186                                 # just in case, prevent config.reset() from nuking these.
1187                                 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1188
1189                                 for myprovide in myaux["PROVIDE"].split():
1190                                         prov_cp = portage.dep_getkey(myprovide)
1191                                         if prov_cp not in dep_settings.getvirtuals():
1192                                                 stats["virtual.unavailable"]+=1
1193                                                 fails["virtual.unavailable"].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+prov_cp)
1194
1195                                 if not baddepsyntax:
1196                                         ismasked = os.path.join(catdir, y) not in \
1197                                                 portdb.xmatch("list-visible", x)
1198                                         if ismasked:
1199                                                 if "--ignore-masked" in myoptions:
1200                                                         continue
1201                                                 #we are testing deps for a masked package; give it some lee-way
1202                                                 suffix="masked"
1203                                                 matchmode="match-all"
1204                                         else:
1205                                                 suffix=""
1206                                                 matchmode="match-visible"
1207
1208                                         if prof[1] == "dev":
1209                                                 suffix=suffix+"indev"
1210
1211                                         for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1212
1213                                                 mykey=mytype+".bad"+suffix
1214                                                 myvalue = myaux[mytype]
1215                                                 if not myvalue:
1216                                                         continue
1217                                                 try:
1218                                                         mydep = portage.dep_check(myvalue, portdb,
1219                                                                 dep_settings, use="all", mode=matchmode,
1220                                                                 trees=trees)
1221                                                 except KeyError, e:
1222                                                         stats[mykey]=stats[mykey]+1
1223                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(e[0]))
1224                                                         continue
1225
1226                                                 if mydep[0]==1:
1227                                                         if mydep[1]!=[]:
1228                                                                 #we have some unsolvable deps
1229                                                                 #remove ! deps, which always show up as unsatisfiable
1230                                                                 d=0
1231                                                                 while d<len(mydep[1]):
1232                                                                         if mydep[1][d][0]=="!":
1233                                                                                 del mydep[1][d]
1234                                                                         else:
1235                                                                                 d += 1
1236                                                                 #if we emptied out our list, continue:
1237                                                                 if not mydep[1]:
1238                                                                         continue
1239                                                                 stats[mykey]=stats[mykey]+1
1240                                                                 fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1241                                                 else:
1242                                                         stats[mykey]=stats[mykey]+1
1243                                                         fails[mykey].append(x+"/"+y+".ebuild: "+keyword+"("+prof[0]+") "+repr(mydep[1]))
1244
1245                 # this check needs work, it won't catch (\ndie)
1246                 if not os.system("egrep '^[^#]*\([^)]*\<die\>' "+checkdir+"/"+y+".ebuild >/dev/null 2>&1"):
1247                         stats["ebuild.nesteddie"]=stats["ebuild.nesteddie"]+1
1248                         fails["ebuild.nesteddie"].append(x+"/"+y+".ebuild")
1249                 # uselist checks - global
1250                 myuse = []
1251                 default_use = []
1252                 for myflag in myaux["IUSE"].split():
1253                         if myflag.startswith("+"):
1254                                 default_use.append(myflag)
1255                                 myflag = myflag[1:]
1256                         myuse.append(myflag)
1257                 for mypos in range(len(myuse)-1,-1,-1):
1258                         if myuse[mypos] and (myuse[mypos] in uselist):
1259                                 del myuse[mypos]
1260                 # uselist checks - local
1261                 mykey = portage.dep_getkey(catpkg)
1262                 if luselist.has_key(mykey):
1263                         for mypos in range(len(myuse)-1,-1,-1):
1264                                 if myuse[mypos] and (myuse[mypos] in luselist[mykey]):
1265                                         del myuse[mypos]
1266                 if default_use and myaux["EAPI"] == "0":
1267                         myuse += default_use
1268                 for mypos in range(len(myuse)):
1269                         stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1270                         fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1271
1272                 # license checks
1273                 if not badlicsyntax:
1274                         myuse = myaux["LICENSE"]
1275                         # Parse the LICENSE variable, remove USE conditions and
1276                         # flatten it.
1277                         myuse=portage_dep.use_reduce(portage_dep.paren_reduce(myuse), matchall=1)
1278                         myuse=portage.flatten(myuse)
1279                         # Check each entry to ensure that it exists in PORTDIR's
1280                         # license directory.
1281                         for mypos in range(0,len(myuse)):
1282                                 # Need to check for "||" manually as no portage
1283                                 # function will remove it without removing values.
1284                                 if myuse[mypos] not in liclist and myuse[mypos] != "||":
1285                                         stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1286                                         fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1287
1288                 #keyword checks
1289                 myuse = myaux["KEYWORDS"].split()
1290                 for mykey in myuse:
1291                         myskey=mykey[:]
1292                         if myskey[0]=="-":
1293                                 myskey=myskey[1:]
1294                         if myskey[0]=="~":
1295                                 myskey=myskey[1:]
1296                         if mykey!="-*":
1297                                 if myskey not in kwlist:
1298                                         stats["KEYWORDS.invalid"] += 1
1299                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1300                                 elif not profiles.has_key(myskey):
1301                                         stats["KEYWORDS.invalid"] += 1
1302                                         fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1303
1304                 #syntax checks
1305                 myear = time.gmtime(os.stat(checkdir+"/"+y+".ebuild")[ST_MTIME])[0]
1306                 gentoo_copyright = re.compile(r'^# Copyright ((1999|200\d)-)?' + str(myear) + r' Gentoo Foundation')
1307                 gentoo_license = re.compile(r'^# Distributed under the terms of the GNU General Public License v2$')
1308                 cvs_header = re.compile(r'^#\s*\$Header.*\$$')
1309                 ignore_line = re.compile(r'(^$)|(^(\t)*#)')
1310                 leading_spaces = re.compile(r'^[\S\t]')
1311                 trailing_whitespace = re.compile(r'.*([\S]$)')
1312                 readonly_assignment = re.compile(r'^\s*(export\s+)?(A|CATEGORY|P|PV|PN|PR|PVR|PF|D|WORKDIR|FILESDIR|FEATURES|USE)=')
1313                 line_continuation = re.compile(r'([^#]*\S)(\s+|\t)\\$')
1314                 linenum=0
1315                 previous_line = None
1316                 for line in input(checkdir+"/"+y+".ebuild"):
1317                         linenum += 1
1318                         # Gentoo copyright check
1319                         if linenum == 1:
1320                                 match = gentoo_copyright.match(line)
1321                                 if not match:
1322                                         myerrormsg = "Copyright header Error. Possibly date related."
1323                                         stats["ebuild.badheader"] +=1
1324                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1325                         # Gentoo license check
1326                         elif linenum == 2:
1327                                 match = gentoo_license.match(line)
1328                                 if not match:
1329                                         myerrormsg = "Gentoo License Error."
1330                                         stats["ebuild.badheader"] +=1
1331                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1332                         # CVS Header check
1333                         elif linenum == 3:
1334                                 match = cvs_header.match(line)
1335                                 if not match:
1336                                         myerrormsg = "CVS Header Error."
1337                                         stats["ebuild.badheader"] +=1
1338                                         fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1339                         else:
1340                                 match = ignore_line.match(line)
1341                                 if not match:
1342                                         # Excluded Blank lines and full line comments. Good!
1343                                         # Leading Spaces Check
1344                                         match = leading_spaces.match(line)
1345                                         if not match:
1346                                                 #Line has got leading spaces. Bad!
1347                                                 myerrormsg = "Leading Space Syntax Error. Line %d" % linenum
1348                                                 stats["ebuild.minorsyn"] +=1
1349                                                 fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1350                                         # Trailing whitespace check
1351                                         match = trailing_whitespace.match(line)
1352                                         if not match:
1353                                                 #Line has got trailing whitespace. Bad!
1354                                                 myerrormsg = "Trailing whitespace Syntax Error. Line %d" % linenum
1355                                                 stats["ebuild.minorsyn"] +=1
1356                                                 fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1357                                         # Readonly variable assignment check
1358                                         match = readonly_assignment.match(line)
1359                                         # The regex can give a false positive for continued lines,
1360                                         # so we check the previous line to see if it was continued.
1361                                         if match and (not previous_line or not line_continuation.match(previous_line)):
1362                                                 # invalid assignment, very bad!
1363                                                 myerrormsg = "Readonly variable assignment to %s on line %d" % (match.group(2), linenum)
1364                                                 stats["variable.readonly"] += 1
1365                                                 fails["variable.readonly"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
1366                         previous_line = line
1367                 del previous_line
1368
1369         # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1370         # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1371         #if not portage.portdb.xmatch("bestmatch-visible",x):
1372         #    stats["ebuild.nostable"]+=1
1373         #    fails["ebuild.nostable"].append(x)
1374         if allmasked and repolevel == 3:
1375                 stats["ebuild.allmasked"]+=1
1376                 fails["ebuild.allmasked"].append(x)
1377
1378 #Pickle and save results for instant reuse in last and lfull
1379 if os.access(portage_const.CACHE_PATH, os.W_OK):
1380         for myobj, fname in (stats, "repo.stats"), (fails, "repo.fails"):
1381                 fpath = os.path.join(portage_const.CACHE_PATH, fname)
1382                 savef = open(fpath, 'w')
1383                 pickle.dump(myobj, savef)
1384                 savef.close()
1385                 portage.apply_secpass_permissions(fpath, gid=portage.portage_gid,
1386                         mode=0664)
1387 if quiet < 2:
1388         print
1389 #dofail will be set to 1 if we have failed in at least one non-warning category
1390 dofail=0
1391 #dowarn will be set to 1 if we tripped any warnings
1392 dowarn=0
1393 #dofull will be set if we should print a "repoman full" informational message
1394 dofull=0
1395 for x in qacats:
1396         if not isCvs and x.find("notadded") != -1:
1397                 stats[x] = 0
1398         if stats[x]:
1399                 dowarn=1
1400                 if x not in qawarnings:
1401                         dofail=1
1402         else:
1403                 continue
1404         print "  "+x.ljust(30),
1405         if stats[x]==0:
1406                 print green(`stats[x]`)
1407                 continue
1408         elif x in qawarnings:
1409                 print yellow(`stats[x]`)
1410         else:
1411                 print red(`stats[x]`)
1412         if mymode!="full":
1413                 if stats[x]<12:
1414                         for y in fails[x]:
1415                                 print "   "+y
1416                 else:
1417                         dofull=1
1418         else:
1419                 for y in fails[x]:
1420                         print "   "+y
1421
1422 def grouplist(mylist,seperator="/"):
1423         """(list,seperator="/") -- Takes a list of elements; groups them into
1424         same initial element categories. Returns a dict of {base:[sublist]}
1425         From: ["blah/foo","spork/spatula","blah/weee/splat"]
1426         To:   {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
1427         mygroups={}
1428         for x in mylist:
1429                 xs=x.split(seperator)
1430                 if xs[0]==".":
1431                         xs=xs[1:]
1432                 if xs[0] not in mygroups.keys():
1433                         mygroups[xs[0]]=[seperator.join(xs[1:])]
1434                 else:
1435                         mygroups[xs[0]]+=[seperator.join(xs[1:])]
1436         return mygroups
1437
1438 if mymode!="commit":
1439         if dofull:
1440                 print bold("Note: type \"repoman full\" for a complete listing.")
1441                 if quiet < 1:
1442                         print
1443         if dowarn and not dofail:
1444                 if quiet < 2:
1445                         print green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n              I'll take it this time, but I'm not happy.\""
1446                 else:
1447                         print green("RepoMan sez:"),"\"OK for now, but I'll be back ...\""
1448         elif not dofail:
1449                 print green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\""
1450         if quiet < 1:
1451                 print
1452 else:
1453         if dofail:
1454                 print turquoise("Please fix these important QA issues first.")
1455                 print green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n"
1456                 sys.exit(1)
1457
1458         if "--pretend" in myoptions:
1459                 print green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n"
1460
1461         if fails["digest.missing"]:
1462                 print green("Creating missing digests...")
1463         for x in fails["digest.missing"]:
1464                 xs=x.split("/")
1465                 del xs[-2]
1466                 myeb="/".join(xs[:-1])+"/"+xs[-1][7:]
1467                 if "--pretend" in myoptions:
1468                         print "(ebuild "+portdir+"/"+myeb+".ebuild digest)"
1469                 else:
1470                         retval=os.system("ebuild "+portdir+"/"+myeb+".ebuild digest")
1471                         if retval:
1472                                 print "!!! Exiting on ebuild digest (shell) error code:",retval
1473                                 sys.exit(retval)
1474
1475         mycvstree=cvstree.getentries("./",recursive=1)
1476         if isCvs and not mycvstree:
1477                 print "!!! It seems we don't have a cvs tree?"
1478                 sys.exit(3)
1479
1480         myunadded=cvstree.findunadded(mycvstree,recursive=1,basedir="./")
1481         myautoadd=[]
1482         if myunadded:
1483                 for x in range(len(myunadded)-1,-1,-1):
1484                         xs=myunadded[x].split("/")
1485                         if xs[-1]=="files":
1486                                 print "!!! files dir is not added! Please correct this."
1487                                 sys.exit(-1)
1488                         elif xs[-1]=="Manifest":
1489                                 # It's a manifest... auto add
1490                                 myautoadd+=[myunadded[x]]
1491                                 del myunadded[x]
1492                         elif len(xs[-1])>=7:
1493                                 if xs[-1][:7]=="digest-":
1494                                         del xs[-2]
1495                                         myeb="/".join(xs[:-1]+[xs[-1][7:]])+".ebuild"
1496                                         if os.path.exists(myeb):
1497                                                 # Ebuild exists for digest... So autoadd it.
1498                                                 myautoadd+=[myunadded[x]]
1499                                                 del myunadded[x]
1500
1501         if myautoadd:
1502                 print ">>> Auto-Adding missing digests..."
1503                 if "--pretend" in myoptions:
1504                         print "(/usr/bin/cvs add "+" ".join(myautoadd)+")"
1505                         retval=0
1506                 else:
1507                         retval=os.system("/usr/bin/cvs add "+" ".join(myautoadd))
1508                 if retval:
1509                         print "!!! Exiting on cvs (shell) error code:",retval
1510                         sys.exit(retval)
1511
1512         if myunadded:
1513                 print red("!!! The following files are in your cvs tree but are not added to the master")
1514                 print red("!!! tree. Please remove them from the cvs tree or add them to the master tree.")
1515                 for x in myunadded:
1516                         print "   ",x
1517                 print
1518                 print
1519                 sys.exit(1)
1520
1521         retval=["",""]
1522         if isCvs:
1523                 print "Performing a "+green("cvs -n up")+" with a little magic grep to check for updates."
1524                 retval=getstatusoutput("/usr/bin/cvs -n up 2>&1 | egrep '^[^\?] .*' | egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'")
1525
1526         mylines=retval[1].split("\n")
1527         myupdates=[]
1528         for x in mylines:
1529                 if not x:
1530                         continue
1531                 if x[0] not in "UPMAR": # Updates,Patches,Modified,Added,Removed
1532                         print red("!!! Please fix the following issues reported from cvs: ")+green("(U,P,M,A,R are ok)")
1533                         print red("!!! Note: This is a pretend/no-modify pass...")
1534                         print retval[1]
1535                         print
1536                         sys.exit(1)
1537                 elif x[0] in ["U","P"]:
1538                         myupdates+=[x[2:]]
1539
1540         if myupdates:
1541                 print green("Fetching trivial updates...")
1542                 if "--pretend" in myoptions:
1543                         print "(/usr/bin/cvs up "+" ".join(myupdates)+")"
1544                         retval=0
1545                 else:
1546                         retval=os.system("/usr/bin/cvs up "+" ".join(myupdates))
1547                 if retval!=0:
1548                         print "!!! cvs exited with an error. Terminating."
1549                         sys.exit(retval)
1550
1551         if isCvs:
1552                 mycvstree=cvstree.getentries("./",recursive=1)
1553                 mychanged=cvstree.findchanged(mycvstree,recursive=1,basedir="./")
1554                 mynew=cvstree.findnew(mycvstree,recursive=1,basedir="./")
1555                 myremoved=cvstree.findremoved(mycvstree,recursive=1,basedir="./")
1556                 if not (mychanged or mynew or myremoved):
1557                         print
1558                         print green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"\n"
1559                         print
1560                         print "(Didn't find any changed files...)"
1561                         print
1562                         sys.exit(0)
1563
1564         # Manifests need to be regenerated after all other commits, so don't commit
1565         # them now even if they have changed.
1566         mymanifests = [f for f in mychanged if "Manifest" == os.path.basename(f)]
1567         mychanged = [f for f in mychanged if "Manifest" != os.path.basename(f)]
1568         myupdates=mychanged+mynew
1569         myheaders=[]
1570         mydirty=[]
1571         headerstring="'\$(Header|Id)"
1572         headerstring+=".*\$'"
1573         for myfile in myupdates:
1574                 myout=getstatusoutput("egrep -q "+headerstring+" "+myfile)
1575                 if myout[0]==0:
1576                         myheaders.append(myfile)
1577
1578         print "*",green(str(len(myupdates))),"files being committed...",green(str(len(myheaders))),"have headers that will change."
1579         print "*","Files with headers will cause the manifests to be made and recommited."
1580         if quiet == 0:
1581                 print "myupdates:",myupdates
1582                 print "myheaders:",myheaders
1583                 print
1584
1585         if commitmessagefile:
1586                 try:
1587                         f = open(commitmessagefile)
1588                         commitmessage = f.read()
1589                         f.close()
1590                         del f
1591                 except (IOError, OSError), e:
1592                         if e.errno == errno.ENOENT:
1593                                 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % commitmessagefile)
1594                         else:
1595                                 raise
1596                 # We've read the content so the file is no longer needed.
1597                 commitmessagefile = None
1598         if not commitmessage:
1599                 print "Please enter a CVS commit message at the prompt:"
1600                 while not commitmessage:
1601                         try:
1602                                 commitmessage=raw_input(green("> "))
1603                         except KeyboardInterrupt:
1604                                 exithandler()
1605         try:
1606                 commitmessage+="\n(Portage version: "+str(portage.VERSION)+")"
1607         except AttributeError:
1608                 print "Failed to insert portage version in message!"
1609                 commitmessage+="\n(Portage version: Unknown)"
1610
1611         if not manifest1_compat:
1612                 myfiles = myupdates + myremoved + mymanifests
1613                 filesdirs = set()
1614                 if repolevel == 3:
1615                         filesdirs.add(os.path.join(".", "files"))
1616                 elif repolevel in (1, 2):
1617                         for x in myfiles:
1618                                 xs = x.split(os.path.sep)
1619                                 if len(xs) < 4-repolevel:
1620                                         continue
1621                                 xs = xs[0:4-repolevel]
1622                                 xs.append("files")
1623                                 filesdirs.add(os.path.join(*xs))
1624                 else:
1625                         raise AssertionError("repolevel=%s" % str(repolevel))
1626                 digest_files = []
1627                 for x in filesdirs:
1628                         dir_path = os.path.join(startdir, x)
1629                         if not os.path.isdir(dir_path):
1630                                 continue
1631                         for y in os.listdir(dir_path):
1632                                 if y.startswith("digest-"):
1633                                         digest_files.append(os.path.join(x, y))
1634                 if digest_files:
1635                         digest_files.sort()
1636                         if "--pretend" in myoptions:
1637                                 print "(rm %s)" % " ".join(digest_files)
1638                                 print "(/usr/bin/cvs remove %s)" % " ".join(digest_files)
1639                         else:
1640                                 for x in digest_files:
1641                                         os.unlink(os.path.join(startdir, x))
1642                                 retval = spawn(["/usr/bin/cvs", "remove"] + digest_files,
1643                                         env=os.environ)
1644                                 if retval != os.EX_OK:
1645                                         print "!!! Exiting on cvs (shell) error code:",retval
1646                                         sys.exit(retval)
1647                         myremoved.extend(digest_files)
1648                         myremoved = list(set(myremoved))
1649                         myremoved.sort()
1650
1651         if myupdates or myremoved:
1652                 myfiles = myupdates + myremoved
1653                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1654                 mymsg = os.fdopen(fd, "w")
1655                 mymsg.write(commitmessage)
1656                 mymsg.close()
1657
1658                 print
1659                 print green("Using commit message:")
1660                 print green("------------------------------------------------------------------------------")
1661                 print commitmessage
1662                 print green("------------------------------------------------------------------------------")
1663                 print
1664
1665                 retval = None
1666                 if "--pretend" in myoptions:
1667                         print "(/usr/bin/cvs -q commit -F %s %s)" % \
1668                                 (commitmessagefile, " ".join(myfiles))
1669                 else:
1670                         retval = spawn(["/usr/bin/cvs", "-q", "commit",
1671                                 "-F", commitmessagefile] + myfiles,
1672                                 env=os.environ)
1673                 try:
1674                         os.unlink(commitmessagefile)
1675                 except OSError:
1676                         pass
1677                 if retval:
1678                         print "!!! Exiting on cvs (shell) error code:",retval
1679                         sys.exit(retval)
1680
1681         # Setup the GPG commands
1682         def gpgsign(filename):
1683                 if "PORTAGE_GPG_KEY" not in repoman_settings:
1684                         raise portage_exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
1685                 if "PORTAGE_GPG_DIR" not in repoman_settings:
1686                         if os.environ.has_key("HOME"):
1687                                 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(os.environ["HOME"], ".gnupg")
1688                                 if quiet < 1:
1689                                         print "Automatically setting PORTAGE_GPG_DIR to",repoman_settings["PORTAGE_GPG_DIR"]
1690                         else:
1691                                 raise portage_exception.MissingParameter("PORTAGE_GPG_DIR is unset!")
1692                 gpg_dir = repoman_settings["PORTAGE_GPG_DIR"]
1693                 if gpg_dir.startswith("~") and "HOME" in os.environ:
1694                         repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(
1695                                 os.environ["HOME"], gpg_dir[1:].lstrip(os.path.sep))
1696                 if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
1697                         raise portage_exception.InvalidLocation(
1698                                 "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
1699                                 repoman_settings["PORTAGE_GPG_DIR"])
1700                 gpgcmd = "gpg --sign --clearsign --yes "
1701                 gpgcmd+= "--default-key "+repoman_settings["PORTAGE_GPG_KEY"]
1702                 if repoman_settings.has_key("PORTAGE_GPG_DIR"):
1703                         gpgcmd += " --homedir "+repoman_settings["PORTAGE_GPG_DIR"]
1704                 if "--pretend" in myoptions:
1705                         print "("+gpgcmd+" "+filename+")"
1706                 else:
1707                         rValue = os.system(gpgcmd+" "+filename)
1708                         if rValue == os.EX_OK:
1709                                 os.rename(filename+".asc", filename)
1710                         else:
1711                                 raise portage_exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
1712
1713         manifest_commit_required = True
1714         if myheaders or myupdates or myremoved or mynew:
1715                 myfiles=myheaders+myupdates+myremoved+mynew
1716                 for x in range(len(myfiles)-1, -1, -1):
1717                         if myfiles[x].count("/") < 4-repolevel:
1718                                 del myfiles[x]
1719                 mydone=[]
1720                 if repolevel==3:   # In a package dir
1721                         repoman_settings["O"] = startdir
1722                         portage.digestgen([], repoman_settings, manifestonly=1,
1723                                 myportdb=portdb)
1724                 elif repolevel==2: # In a category dir
1725                         for x in myfiles:
1726                                 xs=x.split("/")
1727                                 if len(xs) < 4-repolevel:
1728                                         continue
1729                                 if xs[0]==".":
1730                                         xs=xs[1:]
1731                                 if xs[0] in mydone:
1732                                         continue
1733                                 mydone.append(xs[0])
1734                                 repoman_settings["O"] = os.path.join(startdir, xs[0])
1735                                 if not os.path.isdir(repoman_settings["O"]):
1736                                         continue
1737                                 portage.digestgen([], repoman_settings, manifestonly=1,
1738                                         myportdb=portdb)
1739                 elif repolevel==1: # repo-cvsroot
1740                         print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
1741                         for x in myfiles:
1742                                 xs=x.split("/")
1743                                 if len(xs) < 4-repolevel:
1744                                         continue
1745                                 if xs[0]==".":
1746                                         xs=xs[1:]
1747                                 if "/".join(xs[:2]) in mydone:
1748                                         continue
1749                                 mydone.append("/".join(xs[:2]))
1750                                 repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1])
1751                                 if not os.path.isdir(repoman_settings["O"]):
1752                                         continue
1753                                 portage.digestgen([], repoman_settings, manifestonly=1,
1754                                         myportdb=portdb)
1755                 else:
1756                         print red("I'm confused... I don't know where I am!")
1757                         sys.exit(1)
1758
1759                 # Force an unsigned commit when more than one Manifest needs to be signed.
1760                 if repolevel < 3 and "sign" in repoman_settings.features:
1761                         if "--pretend" in myoptions:
1762                                 print "(/usr/bin/cvs -q commit -F commitmessagefile)"
1763                         else:
1764                                 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1765                                 mymsg = os.fdopen(fd, "w")
1766                                 mymsg.write(commitmessage)
1767                                 mymsg.write("\n (Unsigned Manifest commit)")
1768                                 mymsg.close()
1769                                 retval=os.system("/usr/bin/cvs -q commit -F "+commitmessagefile)
1770                                 try:
1771                                         os.unlink(commitmessagefile)
1772                                 except OSError:
1773                                         pass
1774                                 if retval:
1775                                         print "!!! Exiting on cvs (shell) error code:",retval
1776                                         sys.exit(retval)
1777                         manifest_commit_required = False
1778
1779         signed = False
1780         if "sign" in repoman_settings.features:
1781                 signed = True
1782                 myfiles = myupdates + myremoved + mymanifests
1783                 try:
1784                         if repolevel==3:   # In a package dir
1785                                 repoman_settings["O"] = "."
1786                                 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1787                         elif repolevel==2: # In a category dir
1788                                 mydone=[]
1789                                 for x in myfiles:
1790                                         xs=x.split("/")
1791                                         if len(xs) < 4-repolevel:
1792                                                 continue
1793                                         if xs[0]==".":
1794                                                 xs=xs[1:]
1795                                         if xs[0] in mydone:
1796                                                 continue
1797                                         mydone.append(xs[0])
1798                                         repoman_settings["O"] = os.path.join(".", xs[0])
1799                                         if not os.path.isdir(repoman_settings["O"]):
1800                                                 continue
1801                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1802                         elif repolevel==1: # repo-cvsroot
1803                                 print green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n"
1804                                 mydone=[]
1805                                 for x in myfiles:
1806                                         xs=x.split("/")
1807                                         if len(xs) < 4-repolevel:
1808                                                 continue
1809                                         if xs[0]==".":
1810                                                 xs=xs[1:]
1811                                         if "/".join(xs[:2]) in mydone:
1812                                                 continue
1813                                         mydone.append("/".join(xs[:2]))
1814                                         repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
1815                                         if not os.path.isdir(repoman_settings["O"]):
1816                                                 continue
1817                                         gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
1818                 except portage_exception.PortageException, e:
1819                         portage.writemsg("!!! %s\n" % str(e))
1820                         portage.writemsg("!!! Disabled FEATURES='sign'\n")
1821                         signed = False
1822
1823         if manifest_commit_required or signed:
1824                 if "--pretend" in myoptions:
1825                         print "(/usr/bin/cvs -q commit -F commitmessagefile)"
1826                 else:
1827                         fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
1828                         mymsg = os.fdopen(fd, "w")
1829                         mymsg.write(commitmessage)
1830                         if signed:
1831                                 mymsg.write("\n (Signed Manifest commit)")
1832                         else:
1833                                 mymsg.write("\n (Unsigned Manifest commit)")
1834                         mymsg.close()
1835                         retval=os.system("/usr/bin/cvs -q commit -F "+commitmessagefile)
1836                         try:
1837                                 os.unlink(commitmessagefile)
1838                         except OSError:
1839                                 pass
1840                         if retval:
1841                                 print "!!! Exiting on cvs (shell) error code:",retval
1842                                 sys.exit(retval)
1843
1844         print
1845         if isCvs:
1846                 print "CVS commit complete."
1847         else:
1848                 print "repoman was too scared by not seeing any familiar cvs file that he forgot to commit anything"
1849         print green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n"
1850 sys.exit(0)
1851