catalyst/main.py

   1
   2 # Maintained in full by:
   3 # Catalyst Team <catalyst@gentoo.org>
   4 # Release Engineering Team <releng@gentoo.org>
   5 # Andrew Gaffney <agaffney@gentoo.org>
   6 # Chris Gianelloni <wolf31o2@wolf31o2.org>
   7 # $Id$
   8
   9 import os
  10 import sys
  11 import imp
  12 import string
  13 import getopt
  14 import pdb
  15 import os.path
  16
  17 __selfpath__ = os.path.abspath(os.path.dirname(__file__))
  18
  19 sys.path.append(__selfpath__ + "/modules")
  20
  21 import catalyst.config
  22 import catalyst.util
  23 from catalyst.support import (required_build_targets,
  24         valid_build_targets, CatalystError, hash_map, find_binary, LockInUse)
  25
  26 __maintainer__="Catalyst <catalyst@gentoo.org>"
  27 __version__="2.0.15"
  28
  29 conf_values={}
  30
  31 def usage():
  32         print """Usage catalyst [options] [-C variable=value...] [ -s identifier]
  33  -a --clear-autoresume  clear autoresume flags
  34  -c --config            use specified configuration file
  35  -C --cli               catalyst commandline (MUST BE LAST OPTION)
  36  -d --debug             enable debugging
  37  -f --file              read specfile
  38  -F --fetchonly         fetch files only
  39  -h --help              print this help message
  40  -p --purge             clear tmp dirs,package cache, autoresume flags
  41  -P --purgeonly         clear tmp dirs,package cache, autoresume flags and exit
  42  -T --purgetmponly      clear tmp dirs and autoresume flags and exit
  43  -s --snapshot          generate a release snapshot
  44  -V --version           display version information
  45  -v --verbose           verbose output
  46
  47 Usage examples:
  48
  49 Using the commandline option (-C, --cli) to build a Portage snapshot:
  50 catalyst -C target=snapshot version_stamp=my_date
  51
  52 Using the snapshot option (-s, --snapshot) to build a release snapshot:
  53 catalyst -s 20071121"
  54
  55 Using the specfile option (-f, --file) to build a stage target:
  56 catalyst -f stage1-specfile.spec
  57 """
  58
  59
  60 def version():
  61         print "Catalyst, version "+__version__
  62         print "Copyright 2003-2008 Gentoo Foundation"
  63         print "Copyright 2008-2012 various authors"
  64         print "Distributed under the GNU General Public License version 2.1\n"
  65
  66 def parse_config(myconfig):
  67         # search a couple of different areas for the main config file
  68         myconf={}
  69         config_file=""
  70
  71         confdefaults = {
  72                 "distdir": "/usr/portage/distfiles",
  73                 "hash_function": "crc32",
  74                 "icecream": "/var/cache/icecream",
  75                 "local_overlay": "/usr/local/portage",
  76                 "options": "",
  77                 "packagedir": "/usr/portage/packages",
  78                 "portdir": "/usr/portage",
  79                 "repo_name": "portage",
  80                 "sharedir": "/usr/share/catalyst",
  81                 "snapshot_name": "portage-",
  82                 "snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
  83                 "storedir": "/var/tmp/catalyst",
  84                 }
  85
  86         # first, try the one passed (presumably from the cmdline)
  87         if myconfig:
  88                 if os.path.exists(myconfig):
  89                         print "Using command line specified Catalyst configuration file, "+myconfig
  90                         config_file=myconfig
  91
  92                 else:
  93                         print "!!! catalyst: Could not use specified configuration file "+\
  94                                 myconfig
  95                         sys.exit(1)
  96
  97         # next, try the default location
  98         elif os.path.exists("/etc/catalyst/catalyst.conf"):
  99                 print "Using default Catalyst configuration file, /etc/catalyst/catalyst.conf"
 100                 config_file="/etc/catalyst/catalyst.conf"
 101
 102         # can't find a config file (we are screwed), so bail out
 103         else:
 104                 print "!!! catalyst: Could not find a suitable configuration file"
 105                 sys.exit(1)
 106
 107         # now, try and parse the config file "config_file"
 108         try:
 109 #               execfile(config_file, myconf, myconf)
 110                 myconfig = catalyst.config.ConfigParser(config_file)
 111                 myconf.update(myconfig.get_values())
 112
 113         except:
 114                 print "!!! catalyst: Unable to parse configuration file, "+myconfig
 115                 sys.exit(1)
 116
 117         # now, load up the values into conf_values so that we can use them
 118         for x in confdefaults.keys():
 119                 if x in myconf:
 120                         print "Setting",x,"to config file value \""+myconf[x]+"\""
 121                         conf_values[x]=myconf[x]
 122                 else:
 123                         print "Setting",x,"to default value \""+confdefaults[x]+"\""
 124                         conf_values[x]=confdefaults[x]
 125
 126         # add our python base directory to use for loading target arch's
 127         conf_values["PythonDir"] = __selfpath__
 128
 129         # parse out the rest of the options from the config file
 130         if "autoresume" in string.split(conf_values["options"]):
 131                 print "Autoresuming support enabled."
 132                 conf_values["AUTORESUME"]="1"
 133
 134         if "bindist" in string.split(conf_values["options"]):
 135                 print "Binary redistribution enabled"
 136                 conf_values["BINDIST"]="1"
 137         else:
 138                 print "Bindist is not enabled in catalyst.conf"
 139                 print "Binary redistribution of generated stages/isos may be prohibited by law."
 140                 print "Please see the use description for bindist on any package you are including."
 141
 142         if "ccache" in string.split(conf_values["options"]):
 143                 print "Compiler cache support enabled."
 144                 conf_values["CCACHE"]="1"
 145
 146         if "clear-autoresume" in string.split(conf_values["options"]):
 147                 print "Cleaning autoresume flags support enabled."
 148                 conf_values["CLEAR_AUTORESUME"]="1"
 149
 150         if "distcc" in string.split(conf_values["options"]):
 151                 print "Distcc support enabled."
 152                 conf_values["DISTCC"]="1"
 153
 154         if "icecream" in string.split(conf_values["options"]):
 155                 print "Icecream compiler cluster support enabled."
 156                 conf_values["ICECREAM"]="1"
 157
 158         if "kerncache" in string.split(conf_values["options"]):
 159                 print "Kernel cache support enabled."
 160                 conf_values["KERNCACHE"]="1"
 161
 162         if "pkgcache" in string.split(conf_values["options"]):
 163                 print "Package cache support enabled."
 164                 conf_values["PKGCACHE"]="1"
 165
 166         if "preserve_libs" in string.split(conf_values["options"]):
 167                 print "Preserving libs during unmerge."
 168                 conf_values["PRESERVE_LIBS"]="1"
 169
 170         if "purge" in string.split(conf_values["options"]):
 171                 print "Purge support enabled."
 172                 conf_values["PURGE"]="1"
 173
 174         if "seedcache" in string.split(conf_values["options"]):
 175                 print "Seed cache support enabled."
 176                 conf_values["SEEDCACHE"]="1"
 177
 178         if "snapcache" in string.split(conf_values["options"]):
 179                 print "Snapshot cache support enabled."
 180                 conf_values["SNAPCACHE"]="1"
 181
 182         if "digests" in myconf:
 183                 conf_values["digests"]=myconf["digests"]
 184         if "contents" in myconf:
 185                 conf_values["contents"]=myconf["contents"]
 186
 187         if "envscript" in myconf:
 188                 print "Envscript support enabled."
 189                 conf_values["ENVSCRIPT"]=myconf["envscript"]
 190
 191         if "var_tmpfs_portage" in myconf:
 192                 conf_values["var_tmpfs_portage"]=myconf["var_tmpfs_portage"];
 193
 194         if "port_logdir" in myconf:
 195                 conf_values["port_logdir"]=myconf["port_logdir"];
 196
 197 def import_modules():
 198         # import catalyst's own modules
 199         # (i.e. stage and the arch modules)
 200         targetmap={}
 201
 202         try:
 203                 module_dir = __selfpath__ + "/targets/"
 204                 for x in required_build_targets:
 205                         try:
 206                                 fh=open(module_dir + x + ".py")
 207                                 module=imp.load_module(x, fh,"targets/" + x + ".py",
 208                                         (".py", "r", imp.PY_SOURCE))
 209                                 fh.close()
 210
 211                         except IOError:
 212                                 raise CatalystError, "Can't find " + x + ".py plugin in " + \
 213                                         module_dir
 214                 for x in valid_build_targets:
 215                         try:
 216                                 fh=open(module_dir + x + ".py")
 217                                 module=imp.load_module(x, fh, "targets/" + x + ".py",
 218                                         (".py", "r", imp.PY_SOURCE))
 219                                 module.register(targetmap)
 220                                 fh.close()
 221
 222                         except IOError:
 223                                 raise CatalystError,"Can't find " + x + ".py plugin in " + \
 224                                         module_dir
 225
 226         except ImportError:
 227                 print "!!! catalyst: Python modules not found in "+\
 228                         module_dir + "; exiting."
 229                 sys.exit(1)
 230
 231         return targetmap
 232
 233 def build_target(addlargs, targetmap):
 234         try:
 235                 if addlargs["target"] not in targetmap:
 236                         raise CatalystError,"Target \""+addlargs["target"]+"\" not available."
 237
 238                 mytarget=targetmap[addlargs["target"]](conf_values, addlargs)
 239
 240                 mytarget.run()
 241
 242         except:
 243                 catalyst.util.print_traceback()
 244                 print "!!! catalyst: Error encountered during run of target " + addlargs["target"]
 245                 sys.exit(1)
 246
 247 def main():
 248         targetmap={}
 249
 250         version()
 251         if os.getuid() != 0:
 252                 # catalyst cannot be run as a normal user due to chroots, mounts, etc
 253                 print "!!! catalyst: This script requires root privileges to operate"
 254                 sys.exit(2)
 255
 256         # we need some options in order to work correctly
 257         if len(sys.argv) < 2:
 258                 usage()
 259                 sys.exit(2)
 260
 261         # parse out the command line arguments
 262         try:
 263                 opts,args = getopt.getopt(sys.argv[1:], "apPThvdc:C:f:FVs:", ["purge", "purgeonly", "purgetmponly", "help", "version", "debug",\
 264                         "clear-autoresume", "config=", "cli=", "file=", "fetch", "verbose","snapshot="])
 265
 266         except getopt.GetoptError:
 267                 usage()
 268                 sys.exit(2)
 269
 270         # defaults for commandline opts
 271         debug=False
 272         verbose=False
 273         fetch=False
 274         myconfig=""
 275         myspecfile=""
 276         mycmdline=[]
 277         myopts=[]
 278
 279         # check preconditions
 280         if len(opts) == 0:
 281                 print "!!! catalyst: please specify one of either -f or -C\n"
 282                 usage()
 283                 sys.exit(2)
 284
 285         run = False
 286         for o, a in opts:
 287                 if o in ("-h", "--help"):
 288                         usage()
 289                         sys.exit(1)
 290
 291                 if o in ("-V", "--version"):
 292                         print "Catalyst version "+__version__
 293                         sys.exit(1)
 294
 295                 if o in ("-d", "--debug"):
 296                         conf_values["DEBUG"]="1"
 297                         conf_values["VERBOSE"]="1"
 298
 299                 if o in ("-c", "--config"):
 300                         myconfig=a
 301
 302                 if o in ("-C", "--cli"):
 303                         run = True
 304                         x=sys.argv.index(o)+1
 305                         while x < len(sys.argv):
 306                                 mycmdline.append(sys.argv[x])
 307                                 x=x+1
 308
 309                 if o in ("-f", "--file"):
 310                         run = True
 311                         myspecfile=a
 312
 313                 if o in ("-F", "--fetchonly"):
 314                         conf_values["FETCH"]="1"
 315
 316                 if o in ("-v", "--verbose"):
 317                         conf_values["VERBOSE"]="1"
 318
 319                 if o in ("-s", "--snapshot"):
 320                         if len(sys.argv) < 3:
 321                                 print "!!! catalyst: missing snapshot identifier\n"
 322                                 usage()
 323                                 sys.exit(2)
 324                         else:
 325                                 run = True
 326                                 mycmdline.append("target=snapshot")
 327                                 mycmdline.append("version_stamp="+a)
 328
 329                 if o in ("-p", "--purge"):
 330                         conf_values["PURGE"] = "1"
 331
 332                 if o in ("-P", "--purgeonly"):
 333                         conf_values["PURGEONLY"] = "1"
 334
 335                 if o in ("-T", "--purgetmponly"):
 336                         conf_values["PURGETMPONLY"] = "1"
 337
 338                 if o in ("-a", "--clear-autoresume"):
 339                         conf_values["CLEAR_AUTORESUME"] = "1"
 340
 341         if not run:
 342                 print "!!! catalyst: please specify one of either -f or -C\n"
 343                 usage()
 344                 sys.exit(2)
 345
 346         # import configuration file and import our main module using those settings
 347         parse_config(myconfig)
 348
 349         # Start checking that digests are valid now that the hash_map was imported
 350         # from catalyst.support
 351         if "digests" in conf_values:
 352                 for i in conf_values["digests"].split():
 353                         if i not in hash_map:
 354                                 print
 355                                 print i+" is not a valid digest entry"
 356                                 print "Valid digest entries:"
 357                                 print hash_map.keys()
 358                                 print
 359                                 print "Catalyst aborting...."
 360                                 sys.exit(2)
 361                         if find_binary(hash_map[i][1]) == None:
 362                                 print
 363                                 print "digest="+i
 364                                 print "\tThe "+hash_map[i][1]+\
 365                                         " binary was not found. It needs to be in your system path"
 366                                 print
 367                                 print "Catalyst aborting...."
 368                                 sys.exit(2)
 369         if "hash_function" in conf_values:
 370                 if conf_values["hash_function"] not in hash_map:
 371                         print
 372                         print conf_values["hash_function"]+\
 373                                 " is not a valid hash_function entry"
 374                         print "Valid hash_function entries:"
 375                         print hash_map.keys()
 376                         print
 377                         print "Catalyst aborting...."
 378                         sys.exit(2)
 379                 if find_binary(hash_map[conf_values["hash_function"]][1]) == None:
 380                         print
 381                         print "hash_function="+conf_values["hash_function"]
 382                         print "\tThe "+hash_map[conf_values["hash_function"]][1]+\
 383                                 " binary was not found. It needs to be in your system path"
 384                         print
 385                         print "Catalyst aborting...."
 386                         sys.exit(2)
 387
 388         # import the rest of the catalyst modules
 389         targetmap=import_modules()
 390
 391         addlargs={}
 392
 393         if myspecfile:
 394                 spec = catalyst.config.SpecParser(myspecfile)
 395                 addlargs.update(spec.get_values())
 396
 397         if mycmdline:
 398                 try:
 399                         cmdline = catalyst.config.ConfigParser()
 400                         cmdline.parse_lines(mycmdline)
 401                         addlargs.update(cmdline.get_values())
 402                 except CatalystError:
 403                         print "!!! catalyst: Could not parse commandline, exiting."
 404                         sys.exit(1)
 405
 406         if "target" not in addlargs:
 407                 raise CatalystError, "Required value \"target\" not specified."
 408
 409         # everything is setup, so the build is a go
 410         try:
 411                 build_target(addlargs, targetmap)
 412
 413         except CatalystError:
 414                 print
 415                 print "Catalyst aborting...."
 416                 sys.exit(2)
 417         except KeyboardInterrupt:
 418                 print "\nCatalyst build aborted due to user interrupt ( Ctrl-C )"
 419                 print
 420                 print "Catalyst aborting...."
 421                 sys.exit(2)
 422         except LockInUse:
 423                 print "Catalyst aborting...."
 424                 sys.exit(2)
 425         except:
 426                 print "Catalyst aborting...."
 427                 raise
 428                 sys.exit(2)