catalyst

   1 #!/usr/bin/python -OO
   2
   3 # Maintained in full by:
   4 # Catalyst Team <catalyst@gentoo.org>
   5 # Release Engineering Team <releng@gentoo.org>
   6 # Andrew Gaffney <agaffney@gentoo.org>
   7 # Chris Gianelloni <wolf31o2@wolf31o2.org>
   8 # $Id$
   9
  10 import os, sys, imp, string, getopt
  11 import pdb
  12 import os.path
  13
  14 __selfpath__ = os.path.abspath(os.path.dirname(__file__))
  15
  16 sys.path.append(__selfpath__ + "/modules")
  17
  18 import catalyst.config
  19 import catalyst.util
  20
  21 __maintainer__="Catalyst <catalyst@gentoo.org>"
  22 __version__="2.0.12.2"
  23
  24 conf_values={}
  25
  26 def usage():
  27         print """Usage catalyst [options] [-C variable=value...] [ -s identifier]
  28  -a --clear-autoresume  clear autoresume flags
  29  -c --config            use specified configuration file
  30  -C --cli               catalyst commandline (MUST BE LAST OPTION)
  31  -d --debug             enable debugging
  32  -f --file              read specfile
  33  -F --fetchonly         fetch files only
  34  -h --help              print this help message
  35  -p --purge             clear tmp dirs,package cache, autoresume flags
  36  -P --purgeonly         clear tmp dirs,package cache, autoresume flags and exit
  37  -T --purgetmponly      clear tmp dirs and autoresume flags and exit
  38  -s --snapshot          generate a release snapshot
  39  -V --version           display version information
  40  -v --verbose           verbose output
  41
  42 Usage examples:
  43
  44 Using the commandline option (-C, --cli) to build a Portage snapshot:
  45 catalyst -C target=snapshot version_stamp=my_date
  46
  47 Using the snapshot option (-s, --snapshot) to build a release snapshot:
  48 catalyst -s 20071121"
  49
  50 Using the specfile option (-f, --file) to build a stage target:
  51 catalyst -f stage1-specfile.spec
  52 """
  53
  54
  55 def version():
  56         print "Catalyst, version "+__version__
  57         print "Copyright 2003-2008 Gentoo Foundation"
  58         print "Copyright 2008-2012 various authors"
  59         print "Distributed under the GNU General Public License version 2.1\n"
  60
  61 def parse_config(myconfig):
  62         # search a couple of different areas for the main config file
  63         myconf={}
  64         config_file=""
  65
  66         confdefaults={
  67                 "storedir": "/var/tmp/catalyst",
  68                 "sharedir": "/usr/lib/catalyst",
  69                 "distdir": "/usr/portage/distfiles",
  70                 "repo_name": "portage",
  71                 "portdir": "/usr/portage",
  72                 "packagedir": "/usr/portage/packages",
  73                 "port_tmpdir": "/var/tmp/portage",
  74                 "options": "",
  75                 "snapshot_name": "portage-",
  76                 "snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
  77                 "hash_function": "crc32",
  78                 }
  79
  80         # first, try the one passed (presumably from the cmdline)
  81         if myconfig:
  82                 if os.path.exists(myconfig):
  83                         print "Using command line specified Catalyst configuration file, "+\
  84                                 myconfig
  85                         config_file=myconfig
  86
  87                 else:
  88                         print "!!! catalyst: Could not use specified configuration file "+\
  89                                 myconfig
  90                         sys.exit(1)
  91
  92         # next, try the default location
  93         elif os.path.exists("/etc/catalyst/catalyst.conf"):
  94                 print "Using default Catalyst configuration file," + \
  95                         " /etc/catalyst/catalyst.conf"
  96                 config_file="/etc/catalyst/catalyst.conf"
  97
  98         # can't find a config file (we are screwed), so bail out
  99         else:
 100                 print "!!! catalyst: Could not find a suitable configuration file"
 101                 sys.exit(1)
 102
 103         # now, try and parse the config file "config_file"
 104         try:
 105 #               execfile(config_file, myconf, myconf)
 106                 myconfig = catalyst.config.ConfigParser(config_file)
 107                 myconf.update(myconfig.get_values())
 108
 109         except:
 110                 print "!!! catalyst: Unable to parse configuration file, "+myconfig
 111                 sys.exit(1)
 112
 113         # now, load up the values into conf_values so that we can use them
 114         for x in confdefaults.keys():
 115                 if x in myconf:
 116                         print "Setting",x,"to config file value \""+myconf[x]+"\""
 117                         conf_values[x]=myconf[x]
 118                 else:
 119                         print "Setting",x,"to default value \""+confdefaults[x]+"\""
 120                         conf_values[x]=confdefaults[x]
 121
 122         # parse out the rest of the options from the config file
 123         if "autoresume" in string.split(conf_values["options"]):
 124                 print "Autoresuming support enabled."
 125                 conf_values["AUTORESUME"]="1"
 126
 127         if "bindist" in string.split(conf_values["options"]):
 128                 print "Binary redistribution enabled"
 129                 conf_values["BINDIST"]="1"
 130         else:
 131                 print "Bindist is not enabled in catalyst.conf"
 132                 print "Binary redistribution of generated stages/isos may be prohibited by law."
 133                 print "Please see the use description for bindist on any package you are including."
 134
 135         if "ccache" in string.split(conf_values["options"]):
 136                 print "Compiler cache support enabled."
 137                 conf_values["CCACHE"]="1"
 138
 139         if "clear-autoresume" in string.split(conf_values["options"]):
 140                 print "Cleaning autoresume flags support enabled."
 141                 conf_values["CLEAR_AUTORESUME"]="1"
 142
 143 #       if "compress" in string.split(conf_values["options"]):
 144 #               print "Compression enabled."
 145 #               conf_values["COMPRESS"]="1"
 146
 147         if "distcc" in string.split(conf_values["options"]):
 148                 print "Distcc support enabled."
 149                 conf_values["DISTCC"]="1"
 150
 151         if "icecream" in string.split(conf_values["options"]):
 152                 print "Icecream compiler cluster support enabled."
 153                 conf_values["ICECREAM"]="1"
 154
 155         if "kerncache" in string.split(conf_values["options"]):
 156                 print "Kernel cache support enabled."
 157                 conf_values["KERNCACHE"]="1"
 158
 159         if "pkgcache" in string.split(conf_values["options"]):
 160                 print "Package cache support enabled."
 161                 conf_values["PKGCACHE"]="1"
 162
 163         if "purge" in string.split(conf_values["options"]):
 164                 print "Purge support enabled."
 165                 conf_values["PURGE"]="1"
 166
 167         if "seedcache" in string.split(conf_values["options"]):
 168                 print "Seed cache support enabled."
 169                 conf_values["SEEDCACHE"]="1"
 170
 171         if "snapcache" in string.split(conf_values["options"]):
 172                 print "Snapshot cache support enabled."
 173                 conf_values["SNAPCACHE"]="1"
 174
 175 #       if "tarball" in string.split(conf_values["options"]):
 176 #               print "Tarball creation enabled."
 177 #               conf_values["TARBALL"]="1"
 178
 179         if "digests" in myconf:
 180                 conf_values["digests"]=myconf["digests"]
 181         if "contents" in myconf:
 182                 conf_values["contents"]=myconf["contents"]
 183
 184         if "envscript" in myconf:
 185                 print "Envscript support enabled."
 186                 conf_values["ENVSCRIPT"]=myconf["envscript"]
 187
 188         if "var_tmpfs_portage" in myconf:
 189                 conf_values["var_tmpfs_portage"]=myconf["var_tmpfs_portage"];
 190
 191         if "port_logdir" in myconf:
 192                 conf_values["port_logdir"]=myconf["port_logdir"];
 193
 194 def import_modules():
 195         # import catalyst's own modules
 196         # (i.e. catalyst_support and the arch modules)
 197         targetmap={}
 198
 199         try:
 200                 for x in required_build_targets:
 201                         try:
 202                                 fh=open(conf_values["sharedir"]+"/modules/"+x+".py")
 203                                 module=imp.load_module(x,fh,"modules/"+x+".py",
 204                                         (".py","r",imp.PY_SOURCE))
 205                                 fh.close()
 206
 207                         except IOError:
 208                                 raise CatalystError,"Can't find "+x+".py plugin in "+\
 209                                         conf_values["sharedir"]+"/modules/"
 210
 211                 for x in valid_build_targets:
 212                         try:
 213                                 fh=open(conf_values["sharedir"]+"/modules/"+x+".py")
 214                                 module=imp.load_module(x,fh,"modules/"+x+".py",
 215                                         (".py","r",imp.PY_SOURCE))
 216                                 module.register(targetmap)
 217                                 fh.close()
 218
 219                         except IOError:
 220                                 raise CatalystError,"Can't find "+x+".py plugin in "+\
 221                                         conf_values["sharedir"]+"/modules/"
 222
 223         except ImportError:
 224                 print "!!! catalyst: Python modules not found in "+\
 225                         conf_values["sharedir"]+"/modules; exiting."
 226                 sys.exit(1)
 227
 228         return targetmap
 229
 230 def build_target(addlargs, targetmap):
 231         try:
 232                 if addlargs["target"] not in targetmap:
 233                         raise CatalystError, \
 234                                 "Target \"%s\" not available." % addlargs["target"]
 235
 236                 mytarget=targetmap[addlargs["target"]](conf_values, addlargs)
 237
 238                 mytarget.run()
 239
 240         except:
 241                 catalyst.util.print_traceback()
 242                 print "!!! catalyst: Error encountered during run of target " + \
 243                         addlargs["target"]
 244                 sys.exit(1)
 245
 246 if __name__ == "__main__":
 247         targetmap={}
 248
 249         version()
 250         if os.getuid() != 0:
 251                 # catalyst cannot be run as a normal user due to chroots, mounts, etc
 252                 print "!!! catalyst: This script requires root privileges to operate"
 253                 sys.exit(2)
 254
 255         # we need some options in order to work correctly
 256         if len(sys.argv) < 2:
 257                 usage()
 258                 sys.exit(2)
 259
 260         # parse out the command line arguments
 261         try:
 262                 opts,args = getopt.getopt(sys.argv[1:], "apPThvdc:C:f:FVs:",
 263                         ["purge", "purgeonly", "purgetmponly", "help", "version", "debug",
 264                         "clear-autoresume", "config=", "cli=", "file=", "fetch",
 265                         "verbose","snapshot="
 266                         ]
 267                 )
 268
 269         except getopt.GetoptError:
 270                 usage()
 271                 sys.exit(2)
 272
 273         # defaults for commandline opts
 274         debug=False
 275         verbose=False
 276         fetch=False
 277         myconfig=""
 278         myspecfile=""
 279         mycmdline=[]
 280         myopts=[]
 281
 282         # check preconditions
 283         if len(opts) == 0:
 284                 print "!!! catalyst: please specify one of either -f or -C\n"
 285                 usage()
 286                 sys.exit(2)
 287
 288         run = False
 289         for o, a in opts:
 290                 if o in ("-h", "--help"):
 291                         usage()
 292                         sys.exit(1)
 293
 294                 if o in ("-V", "--version"):
 295                         print "Catalyst version "+__version__
 296                         sys.exit(1)
 297
 298                 if o in ("-d", "--debug"):
 299                         conf_values["DEBUG"]="1"
 300                         conf_values["VERBOSE"]="1"
 301
 302                 if o in ("-c", "--config"):
 303                         myconfig=a
 304
 305                 if o in ("-C", "--cli"):
 306                         run = True
 307                         x=sys.argv.index(o)+1
 308                         while x < len(sys.argv):
 309                                 mycmdline.append(sys.argv[x])
 310                                 x=x+1
 311
 312                 if o in ("-f", "--file"):
 313                         run = True
 314                         myspecfile=a
 315
 316                 if o in ("-F", "--fetchonly"):
 317                         conf_values["FETCH"]="1"
 318
 319                 if o in ("-v", "--verbose"):
 320                         conf_values["VERBOSE"]="1"
 321
 322                 if o in ("-s", "--snapshot"):
 323                         if len(sys.argv) < 3:
 324                                 print "!!! catalyst: missing snapshot identifier\n"
 325                                 usage()
 326                                 sys.exit(2)
 327                         else:
 328                                 run = True
 329                                 mycmdline.append("target=snapshot")
 330                                 mycmdline.append("version_stamp="+a)
 331
 332                 if o in ("-p", "--purge"):
 333                         conf_values["PURGE"] = "1"
 334
 335                 if o in ("-P", "--purgeonly"):
 336                         conf_values["PURGEONLY"] = "1"
 337
 338                 if o in ("-T", "--purgetmponly"):
 339                         conf_values["PURGETMPONLY"] = "1"
 340
 341                 if o in ("-a", "--clear-autoresume"):
 342                         conf_values["CLEAR_AUTORESUME"] = "1"
 343
 344         if not run:
 345                 print "!!! catalyst: please specify one of either -f or -C\n"
 346                 usage()
 347                 sys.exit(2)
 348
 349         # import configuration file and import our main module using those settings
 350         parse_config(myconfig)
 351         sys.path.append(conf_values["sharedir"]+"/modules")
 352         from catalyst_support import *
 353
 354         # Start checking that digests are valid now that the hash_map was imported
 355         # from catalyst_support
 356         if "digests" in conf_values:
 357                 for i in conf_values["digests"].split():
 358                         if i not in hash_map:
 359                                 print
 360                                 print i+" is not a valid digest entry"
 361                                 print "Valid digest entries:"
 362                                 print hash_map.keys()
 363                                 print
 364                                 print "Catalyst aborting...."
 365                                 sys.exit(2)
 366                         if find_binary(hash_map[i][1]) == None:
 367                                 print
 368                                 print "digest="+i
 369                                 print "\tThe "+hash_map[i][1]+\
 370                                         " binary was not found. It needs to be in your system path"
 371                                 print
 372                                 print "Catalyst aborting...."
 373                                 sys.exit(2)
 374         if "hash_function" in conf_values:
 375                 if conf_values["hash_function"] not in hash_map:
 376                         print
 377                         print conf_values["hash_function"]+\
 378                                 " is not a valid hash_function entry"
 379                         print "Valid hash_function entries:"
 380                         print hash_map.keys()
 381                         print
 382                         print "Catalyst aborting...."
 383                         sys.exit(2)
 384                 if find_binary(hash_map[conf_values["hash_function"]][1]) == None:
 385                         print
 386                         print "hash_function="+conf_values["hash_function"]
 387                         print "\tThe "+hash_map[conf_values["hash_function"]][1]+\
 388                                 " binary was not found. It needs to be in your system path"
 389                         print
 390                         print "Catalyst aborting...."
 391                         sys.exit(2)
 392
 393         # import the rest of the catalyst modules
 394         targetmap=import_modules()
 395
 396         addlargs={}
 397
 398         if myspecfile:
 399                 spec = catalyst.config.SpecParser(myspecfile)
 400                 addlargs.update(spec.get_values())
 401
 402         if mycmdline:
 403                 try:
 404                         cmdline = catalyst.config.ConfigParser()
 405                         cmdline.parse_lines(mycmdline)
 406                         addlargs.update(cmdline.get_values())
 407                 except CatalystError:
 408                         print "!!! catalyst: Could not parse commandline, exiting."
 409                         sys.exit(1)
 410
 411         if "target" not in addlargs:
 412                 raise CatalystError, "Required value \"target\" not specified."
 413
 414         # everything is setup, so the build is a go
 415         try:
 416                 build_target(addlargs, targetmap)
 417
 418         except CatalystError:
 419                 print
 420                 print "Catalyst aborting...."
 421                 sys.exit(2)
 422         except KeyboardInterrupt:
 423                 print "\nCatalyst build aborted due to user interrupt ( Ctrl-C )"
 424                 print
 425                 print "Catalyst aborting...."
 426                 sys.exit(2)
 427         except LockInUse:
 428                 print "Catalyst aborting...."
 429                 sys.exit(2)
 430         except:
 431                 print "Catalyst aborting...."
 432                 raise
 433                 sys.exit(2)
 434
 435         #except CatalystError:
 436         #       print
 437         #       print "Catalyst aborting...."
 438         #       sys.exit(2)
 439         #except KeyError:
 440         #       print "\nproblem with command line or spec file ( Key Error )"
 441         #       print "Key: "+str(sys.exc_value)+" was not found"
 442         #       print "Catalyst aborting...."
 443         #       sys.exit(2)
 444         #except UnboundLocalError:
 445         #       print
 446         #       print "UnboundLocalError: "+str(sys.exc_value)+" was not found"
 447         #       raise
 448         #       print
 449         #       print "Catalyst aborting...."
 450         #       sys.exit(2)