modules/generic_stage_target.py

   1 # $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.141 2006/10/02 20:41:54 wolf31o2 Exp $
   2
   3 """
   4 This class does all of the chroot setup, copying of files, etc. It is
   5 the driver class for pretty much everything that Catalyst does.
   6 """
   7
   8 import os,string,imp,types,shutil
   9 from catalyst_support import *
  10 from generic_target import *
  11 from stat import *
  12 import catalyst_lock
  13 class generic_stage_target(generic_target):
  14
  15         def __init__(self,myspec,addlargs):
  16                 self.required_values.extend(["version_stamp","target","subarch","rel_type",\
  17                         "profile","snapshot","source_subpath"])
  18
  19                 self.valid_values.extend(["version_stamp","target","subarch","rel_type","profile",\
  20                         "snapshot","source_subpath","portage_confdir","cflags","cxxflags",\
  21                         "ldflags","chost","hostuse","portage_overlay","distcc_hosts","makeopts",\
  22                         "pkgcache_path","kerncache_path"])
  23
  24                 self.set_valid_build_kernel_vars(addlargs)
  25                 generic_target.__init__(self,myspec,addlargs)
  26                 # map the mainarch we are running under to the mainarches we support for
  27                 # building stages and LiveCDs. (for example, on amd64, we can build
  28                 # stages for x86 or amd64.
  29                 targetmap={
  30                                 "x86" : ["x86"],
  31                                 "amd64" : ["x86","amd64"],
  32                                 "sparc64" : ["sparc","sparc64"],
  33                                 "ia64" : ["ia64"],
  34                                 "alpha" : ["alpha"],
  35                                 "sparc" : ["sparc"],
  36                                 "sh" : ["sh"],
  37                                 "s390" : ["s390"],
  38                                 "ppc" : ["ppc"],
  39                                 "ppc64" : ["ppc","ppc64"],
  40                                 "hppa" : ["hppa"],
  41                                 "mips" : ["mips"],
  42                                 "arm" : ["arm"]
  43                 }
  44
  45                 machinemap={
  46                                 "i386" : "x86",
  47                                 "i486" : "x86",
  48                                 "i586" : "x86",
  49                                 "i686" : "x86",
  50                                 "x86_64" : "amd64",
  51                                 "sparc64" : "sparc64",
  52                                 "ia64" : "ia64",
  53                                 "alpha" : "alpha",
  54                                 "sparc" : "sparc",
  55                                 "sh2" : "sh",
  56                                 "sh3" : "sh",
  57                                 "sh4" : "sh",
  58                                 "sh2eb" : "sh",
  59                                 "sh3eb" : "sh",
  60                                 "sh4eb" : "sh",
  61                                 "s390" : "s390",
  62                                 "ppc" : "ppc",
  63                                 "ppc64" : "ppc64",
  64                                 "parisc" : "hppa",
  65                                 "parisc64" : "hppa",
  66                                 "mips" : "mips",
  67                                 "mips64" : "mips",
  68                                 "arm" : "arm",
  69                                 "armv4l" : "arm",
  70                                 "armeb" : "arm",
  71                                 "armv5b" : "arm"
  72                 }
  73
  74                 mymachine=os.uname()[4]
  75                 if not machinemap.has_key(mymachine):
  76                         raise CatalystError, "Unknown machine type "+mymachine
  77
  78                 self.settings["hostarch"]=machinemap[mymachine]
  79                 self.archmap={}
  80                 self.subarchmap={}
  81
  82                 for x in targetmap[self.settings["hostarch"]]:
  83                         try:
  84                                 fh=open(self.settings["sharedir"]+"/arch/"+x+".py")
  85                                 # this next line loads the plugin as a module and assigns it to archmap[x]
  86                                 self.archmap[x]=imp.load_module(x,fh,"arch/"+x+".py",(".py","r",imp.PY_SOURCE))
  87                                 # this next line registers all the subarches supported in the plugin
  88                                 self.archmap[x].register(self.subarchmap)
  89                                 fh.close()
  90
  91                         except IOError:
  92                                 msg("Can't find "+x+".py plugin in "+self.settings["sharedir"]+"/arch/")
  93                 # call arch constructor, pass our settings
  94                 try:
  95                         self.arch=self.subarchmap[self.settings["subarch"]](self.settings)
  96                 except:
  97                         print "Invalid subarch: "+self.settings["subarch"]
  98                         print "Choose one of the following:",
  99                         for x in self.subarchmap:
 100                                 print x,
 101                         print
 102                         sys.exit(2)
 103
 104                 print "Using target:",self.settings["target"]
 105                 # self.settings["mainarch"] should now be set by our arch constructor,
 106                 # so we print a nice informational message:
 107                 if self.settings["mainarch"]==self.settings["hostarch"]:
 108                         print "Building natively for",self.settings["hostarch"]
 109
 110                 else:
 111                         print "Building on",self.settings["hostarch"],"for alternate machine type",\
 112                                 self.settings["mainarch"]
 113                 # This should be first to be set as other set_ options depend on this
 114                 self.set_spec_prefix()
 115
 116                 # define all of our core variables
 117                 self.set_target_profile()
 118                 self.set_target_subpath()
 119                 self.set_source_subpath()
 120
 121                 # set paths
 122                 self.set_snapshot_path()
 123                 self.set_root_path()
 124                 self.set_source_path()
 125                 self.set_snapcache_path()
 126                 self.set_chroot_path()
 127                 self.set_autoresume_path()
 128                 self.set_dest_path()
 129                 self.set_stage_path()
 130                 self.set_target_path()
 131
 132                 self.set_controller_file()
 133                 self.set_action_sequence()
 134                 self.set_use()
 135                 self.set_cleanables()
 136                 self.set_iso_volume_id()
 137                 self.set_build_kernel_vars()
 138                 self.set_fsscript()
 139                 self.set_archscript()
 140                 self.set_runscript()
 141                 self.set_install_mask()
 142                 self.set_rcadd()
 143                 self.set_rcdel()
 144                 self.set_cdtar()
 145                 self.set_fstype()
 146                 self.set_fsops()
 147                 self.set_iso()
 148                 self.set_packages()
 149                 self.set_rm()
 150                 self.set_linuxrc()
 151                 self.set_overlay()
 152                 self.set_portage_overlay()
 153                 self.set_root_overlay()
 154
 155                 # This next line checks to make sure that the specified variables exist
 156                 # on disk.
 157                 #pdb.set_trace()
 158                 file_locate(self.settings,["source_path","snapshot_path","distdir"],expand=0)
 159                 # If we are using portage_confdir, check that as well.
 160                 if self.settings.has_key("portage_confdir"):
 161                         file_locate(self.settings,["portage_confdir"],expand=0)
 162
 163                 # setup our mount points
 164                 if self.settings.has_key("SNAPCACHE"):
 165                         self.mounts=[ "/proc","/dev","/usr/portage","/usr/portage/distfiles" ]
 166                         self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
 167                                 "/usr/portage":self.settings["snapshot_cache_path"]+"/portage",\
 168                                 "/usr/portage/distfiles":self.settings["distdir"]}
 169                 else:
 170                         self.mounts=[ "/proc","/dev","/usr/portage/distfiles" ]
 171                         self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
 172                                 "/usr/portage/distfiles":self.settings["distdir"]}
 173                 if os.uname()[0] == "Linux":
 174                         self.mounts.append("/dev/pts")
 175
 176                 self.set_mounts()
 177
 178                 # Configure any user specified options (either in catalyst.conf or on
 179                 # the command line).
 180                 if self.settings.has_key("PKGCACHE"):
 181                         self.set_pkgcache_path()
 182                         print "Location of the package cache is " + self.settings["pkgcache_path"]
 183                         self.mounts.append("/usr/portage/packages")
 184                         self.mountmap["/usr/portage/packages"]=self.settings["pkgcache_path"]
 185
 186                 if self.settings.has_key("KERNCACHE"):
 187                         self.set_kerncache_path()
 188                         print "Location of the kerncache is " + self.settings["kerncache_path"]
 189                         self.mounts.append("/tmp/kerncache")
 190                         self.mountmap["/tmp/kerncache"]=self.settings["kerncache_path"]
 191
 192                 if self.settings.has_key("CCACHE"):
 193                         if os.environ.has_key("CCACHE_DIR"):
 194                                 ccdir=os.environ["CCACHE_DIR"]
 195                                 del os.environ["CCACHE_DIR"]
 196                         else:
 197                                 ccdir="/root/.ccache"
 198                         if not os.path.isdir(ccdir):
 199                                         raise CatalystError,\
 200                                                 "Compiler cache support can't be enabled (can't find "+ccdir+")"
 201                         self.mounts.append("/var/tmp/ccache")
 202                         self.mountmap["/var/tmp/ccache"]=ccdir
 203                         # for the chroot:
 204                         self.env["CCACHE_DIR"]="/var/tmp/ccache"
 205
 206         def override_chost(self):
 207                 if self.makeconf.has_key("CHOST"):
 208                         self.settings["CHOST"]=self.makeconf["CHOST"]
 209
 210         def override_cflags(self):
 211                 if self.makeconf.has_key("CFLAGS"):
 212                         self.settings["CFLAGS"]=self.makeconf["CFLAGS"]
 213
 214         def override_cxxflags(self):
 215                 if self.makeconf.has_key("CXXFLAGS"):
 216                         self.settings["CXXFLAGS"]=self.makeconf["CXXFLAGS"]
 217
 218         def override_ldflags(self):
 219                 if self.makeconf.has_key("LDFLAGS"):
 220                         self.settings["LDFLAGS"]=self.makeconf["LDFLAGS"]
 221
 222         def set_install_mask(self):
 223                 if self.settings.has_key("install_mask"):
 224                         if type(self.settings["install_mask"]) != types.StringType:
 225                                 self.settings["install_mask"]=string.join(self.settings["install_mask"])
 226
 227         def set_spec_prefix(self):
 228                 self.settings["spec_prefix"]=self.settings["target"]
 229
 230         def set_target_profile(self):
 231                 self.settings["target_profile"]=self.settings["profile"]
 232
 233         def set_target_subpath(self):
 234                 self.settings["target_subpath"]=self.settings["rel_type"]+"/"+self.settings["target"]+\
 235                         "-"+self.settings["subarch"]+"-"+self.settings["version_stamp"]
 236
 237         def set_source_subpath(self):
 238                 if type(self.settings["source_subpath"]) != types.StringType:
 239                         raise CatalystError, "source_subpath should have been a string. Perhaps you have something wrong in your spec file?"
 240
 241         def set_pkgcache_path(self):
 242                 if self.settings.has_key("pkgcache_path"):
 243                         if type(self.settings["pkgcache_path"]) != types.StringType:
 244                                 self.settings["pkgcache_path"]=normpath(string.join(self.settings["pkgcache_path"]))
 245                 else:
 246                         self.settings["pkgcache_path"]=normpath(self.settings["storedir"]+"/packages/"+\
 247                                 self.settings["target_subpath"]+"/")
 248
 249         def set_kerncache_path(self):
 250                 if self.settings.has_key("kerncache_path"):
 251                         if type(self.settings["kerncache_path"]) != types.StringType:
 252                                 self.settings["kerncache_path"]=normpath(string.join(self.settings["kerncache_path"]))
 253                 else:
 254                         self.settings["kerncache_path"]=normpath(self.settings["storedir"]+"/kerncache/"+\
 255                                 self.settings["target_subpath"]+"/")
 256
 257         def set_target_path(self):
 258                 self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\
 259                         self.settings["target_subpath"]+".tar.bz2")
 260                 if self.settings.has_key("AUTORESUME") \
 261                         and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"):
 262                         print "Resume point detected, skipping target path setup operation..."
 263                 else:
 264                         # first clean up any existing target stuff
 265                         if os.path.isfile(self.settings["target_path"]):
 266                                 cmd("rm -f "+self.settings["target_path"], \
 267                                         "Could not remove existing file: "+self.settings["target_path"],env=self.env)
 268                                 touch(self.settings["autoresume_path"]+"setup_target_path")
 269
 270                         if not os.path.exists(self.settings["storedir"]+"/builds/"):
 271                                 os.makedirs(self.settings["storedir"]+"/builds/")
 272
 273         def set_archscript(self):
 274             if self.settings.has_key(self.settings["spec_prefix"]+"/archscript"):
 275                 print "\nWarning!!!  "
 276                 print "\t"+self.settings["spec_prefix"]+"/archscript" + " is deprecated and no longer used.\n"
 277         def set_runscript(self):
 278             if self.settings.has_key(self.settings["spec_prefix"]+"/runscript"):
 279                 print "\nWarning!!!  "
 280                 print "\t"+self.settings["spec_prefix"]+"/runscript" + " is deprecated and no longer used.\n"
 281
 282         def set_fsscript(self):
 283                 if self.settings.has_key(self.settings["spec_prefix"]+"/fsscript"):
 284                         self.settings["fsscript"]=self.settings[self.settings["spec_prefix"]+"/fsscript"]
 285                         del self.settings[self.settings["spec_prefix"]+"/fsscript"]
 286
 287         def set_rcadd(self):
 288                 if self.settings.has_key(self.settings["spec_prefix"]+"/rcadd"):
 289                         self.settings["rcadd"]=self.settings[self.settings["spec_prefix"]+"/rcadd"]
 290                         del self.settings[self.settings["spec_prefix"]+"/rcadd"]
 291
 292         def set_rcdel(self):
 293                 if self.settings.has_key(self.settings["spec_prefix"]+"/rcdel"):
 294                         self.settings["rcdel"]=self.settings[self.settings["spec_prefix"]+"/rcdel"]
 295                         del self.settings[self.settings["spec_prefix"]+"/rcdel"]
 296
 297         def set_cdtar(self):
 298                 if self.settings.has_key(self.settings["spec_prefix"]+"/cdtar"):
 299                         self.settings["cdtar"]=normpath(self.settings[self.settings["spec_prefix"]+"/cdtar"])
 300                         del self.settings[self.settings["spec_prefix"]+"/cdtar"]
 301
 302         def set_iso(self):
 303                 if self.settings.has_key(self.settings["spec_prefix"]+"/iso"):
 304                         self.settings["iso"]=normpath(self.settings[self.settings["spec_prefix"]+"/iso"])
 305                         del self.settings[self.settings["spec_prefix"]+"/iso"]
 306
 307         def set_fstype(self):
 308                 if self.settings.has_key(self.settings["spec_prefix"]+"/cdfstype"):
 309                         print "\nWarning!!!  "
 310                         print self.settings["spec_prefix"]+"/cdfstype" + " is deprecated and may be removed."
 311                         print "\tUse "+self.settings["spec_prefix"]+"/fstype" + " instead."
 312                         print "\tConverting to "+self.settings["spec_prefix"]+"/fstype" + " internally."
 313                         print "\tContinuing ....\n"
 314                         self.settings["fstype"]=self.settings[self.settings["spec_prefix"]+"/cdfstype"]
 315                         del self.settings[self.settings["spec_prefix"]+"/cdfstype"]
 316
 317                 if self.settings.has_key(self.settings["spec_prefix"]+"/fstype"):
 318                         self.settings["fstype"]=self.settings[self.settings["spec_prefix"]+"/fstype"]
 319                         del self.settings[self.settings["spec_prefix"]+"/fstype"]
 320
 321                 if not self.settings.has_key("fstype"):
 322                         self.settings["fstype"]="normal"
 323                         for x in self.valid_values:
 324                                 if x ==  self.settings["spec_prefix"]+"/fstype" or x == self.settings["spec_prefix"]+"/cdfstype":
 325                                         print "\n"+self.settings["spec_prefix"]+"/fstype is being set to the default of \"normal\"\n"
 326
 327         def set_fsops(self):
 328                 if self.settings.has_key("fstype"):
 329                         self.valid_values.append("fsops")
 330                         if self.settings.has_key(self.settings["spec_prefix"]+"/fsops"):
 331                                 self.settings["fsops"]=self.settings[self.settings["spec_prefix"]+"/fsops"]
 332                                 del self.settings[self.settings["spec_prefix"]+"/fsops"]
 333
 334         def set_source_path(self):
 335                 if self.settings.has_key("SEEDCACHE") and os.path.isdir(normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")):
 336                         self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")
 337                 else:
 338                         self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2")
 339
 340                         if os.path.isfile(self.settings["source_path"]):
 341                                 if os.path.exists(self.settings["source_path"]):
 342                                         self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\
 343                                                         hash_function=self.settings["hash_function"],verbose=False)
 344                 if os.path.isdir(self.settings["source_path"]):
 345                         print "Source path set to "+self.settings["source_path"]
 346                         print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf"
 347                         print "\tthe source path with then be "+normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2\n")
 348                 else:
 349                         print "Source path set to "+self.settings["source_path"]
 350
 351
 352         def set_dest_path(self):
 353                 if self.settings.has_key("root_path"):
 354                         self.settings["destpath"]=normpath(self.settings["chroot_path"]+self.settings["root_path"])
 355                 else:
 356                         self.settings["destpath"]=normpath(self.settings["chroot_path"])
 357
 358         def set_cleanables(self):
 359                 self.settings["cleanables"]=["/etc/resolv.conf","/var/tmp/*","/tmp/*","/root/*",\
 360                                                 "/usr/portage"]
 361
 362         def set_snapshot_path(self):
 363                 self.settings["snapshot_path"]=normpath(self.settings["storedir"]+"/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
 364
 365                 if os.path.exists(self.settings["snapshot_path"]):
 366                         self.settings["snapshot_path_hash"]=generate_hash(self.settings["snapshot_path"],\
 367                                         hash_function=self.settings["hash_function"],verbose=False)
 368
 369         def set_snapcache_path(self):
 370                 if self.settings.has_key("SNAPCACHE"):
 371                         self.settings["snapshot_cache_path"]=normpath(self.settings["snapshot_cache"]+"/"+self.settings["snapshot"]+"/")
 372                         self.snapcache_lock=catalyst_lock.LockDir(self.settings["snapshot_cache_path"])
 373                         print "Caching snapshot to " + self.settings["snapshot_cache_path"]
 374
 375         def set_chroot_path(self):
 376                 # Note the trailing slash is very important and things would break without it
 377                 self.settings["chroot_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["target_subpath"]+"/")
 378                 self.chroot_lock=catalyst_lock.LockDir(self.settings["chroot_path"])
 379
 380         def set_autoresume_path(self):
 381                 self.settings["autoresume_path"]=normpath(self.settings["storedir"]+"/tmp/"+\
 382                         self.settings["rel_type"]+"/"+".autoresume-"+self.settings["target"]+\
 383                         "-"+self.settings["subarch"]+"-"+self.settings["version_stamp"]+"/")
 384                 if self.settings.has_key("AUTORESUME"):
 385                         print "The autoresume path is " + self.settings["autoresume_path"]
 386                 if not os.path.exists(self.settings["autoresume_path"]):
 387                         os.makedirs(self.settings["autoresume_path"],0755)
 388
 389         def set_controller_file(self):
 390                 self.settings["controller_file"]=normpath(self.settings["sharedir"]+"/targets/"+self.settings["target"]+"/"+self.settings["target"]+"-controller.sh")
 391
 392         def set_iso_volume_id(self):
 393                 if self.settings.has_key(self.settings["spec_prefix"]+"/volid"):
 394                         self.settings["iso_volume_id"] = string.join(self.settings[self.settings["spec_prefix"]+"/volid"])
 395                         if len(self.settings["iso_volume_id"])>32:
 396                                 raise CatalystError,"ISO VOLUME ID: volid must not exceed 32 characters."
 397                 else:
 398                         self.settings["iso_volume_id"] = "catalyst " + self.settings["snapshot"]
 399
 400         def set_action_sequence(self):
 401                 #Default action sequence for run method
 402                 self.settings["action_sequence"]=["unpack","unpack_snapshot",\
 403                                 "config_profile_link","setup_confdir","portage_overlay","bind","chroot_setup",\
 404                                 "setup_environment","run_local","preclean","unbind","clean"]
 405                 if not self.settings.has_key("TARBALL"):
 406                         self.settings["action_sequence"].append("capture")
 407                 self.settings["action_sequence"].append("clear_autoresume")
 408
 409         def set_use(self):
 410                 if self.settings.has_key(self.settings["spec_prefix"]+"/use"):
 411                         self.settings["use"]=self.settings[self.settings["spec_prefix"]+"/use"]
 412                         del self.settings[self.settings["spec_prefix"]+"/use"]
 413                 if self.settings.has_key("use"):
 414                     if type(self.settings["use"])==types.StringType:
 415                         self.settings["use"]=self.settings["use"].split()
 416
 417         def set_stage_path(self):
 418                         self.settings["stage_path"]=normpath(self.settings["chroot_path"])
 419
 420         def set_mounts(self):
 421                 pass
 422
 423         def set_packages(self):
 424                 pass
 425
 426         def set_rm(self):
 427             if self.settings.has_key(self.settings["spec_prefix"]+"/rm"):
 428                 if type(self.settings[self.settings["spec_prefix"]+"/rm"])==types.StringType:
 429                     self.settings[self.settings["spec_prefix"]+"/rm"]=self.settings[self.settings["spec_prefix"]+"/rm"].split()
 430
 431         def set_linuxrc(self):
 432             if self.settings.has_key(self.settings["spec_prefix"]+"/linuxrc"):
 433                         if type(self.settings[self.settings["spec_prefix"]+"/linuxrc"])==types.StringType:
 434                                 self.settings["linuxrc"]=self.settings[self.settings["spec_prefix"]+"/linuxrc"]
 435                                 del self.settings[self.settings["spec_prefix"]+"/linuxrc"]
 436
 437         def set_portage_overlay(self):
 438                 if self.settings.has_key("portage_overlay"):
 439                         if type(self.settings["portage_overlay"])==types.StringType:
 440                                 self.settings["portage_overlay"]=self.settings["portage_overlay"].split()
 441                         print "portage_overlay directories are set to: \"" + string.join(self.settings["portage_overlay"])+"\""
 442
 443         def set_overlay(self):
 444                 if self.settings.has_key(self.settings["spec_prefix"]+"/overlay"):
 445                         if type(self.settings[self.settings["spec_prefix"]+"/overlay"])==types.StringType:
 446                                 self.settings[self.settings["spec_prefix"]+"/overlay"]=self.settings[self.settings["spec_prefix"]+"/overlay"].split()
 447
 448         def set_root_overlay(self):
 449                 if self.settings.has_key(self.settings["spec_prefix"]+"/root_overlay"):
 450                         if type(self.settings[self.settings["spec_prefix"]+"/root_overlay"])==types.StringType:
 451                                 self.settings[self.settings["spec_prefix"]+"/root_overlay"]=self.settings[self.settings["spec_prefix"]+"/root_overlay"].split()
 452
 453
 454         def set_root_path(self):
 455                 # ROOT= variable for emerges
 456                 self.settings["root_path"]="/"
 457
 458         def set_valid_build_kernel_vars(self,addlargs):
 459                 if addlargs.has_key("boot/kernel"):
 460                         if type(addlargs["boot/kernel"]) == types.StringType:
 461                                 loopy=[addlargs["boot/kernel"]]
 462                         else:
 463                                 loopy=addlargs["boot/kernel"]
 464
 465                         for x in loopy:
 466                                 self.required_values.append("boot/kernel/"+x+"/sources")
 467                                 self.required_values.append("boot/kernel/"+x+"/config")
 468                                 self.valid_values.append("boot/kernel/"+x+"/aliases")
 469                                 self.valid_values.append("boot/kernel/"+x+"/extraversion")
 470                                 self.valid_values.append("boot/kernel/"+x+"/packages")
 471                                 if addlargs.has_key("boot/kernel/"+x+"/packages"):
 472                                         if type(addlargs["boot/kernel/"+x+"/packages"]) == types.StringType:
 473                                                 addlargs["boot/kernel/"+x+"/packages"]=[addlargs["boot/kernel/"+x+"/packages"]]
 474                                 self.valid_values.append("boot/kernel/"+x+"/use")
 475                                 self.valid_values.append("boot/kernel/"+x+"/gk_kernargs")
 476                                 self.valid_values.append("boot/kernel/"+x+"/gk_action")
 477                                 self.valid_values.append("boot/kernel/"+x+"/initramfs_overlay")
 478                                 self.valid_values.append("boot/kernel/"+x+"/softlevel")
 479                                 self.valid_values.append("boot/kernel/"+x+"/console")
 480                                 self.valid_values.append("boot/kernel/"+x+"/machine_type")
 481                                 self.valid_values.append("boot/kernel/"+x+"/postconf")
 482                                 if addlargs.has_key("boot/kernel/"+x+"/postconf"):
 483                                         print "boot/kernel/"+x+"/postconf is deprecated"
 484                                         print "\tInternally moving these ebuilds to boot/kernel/"+x+"/packages"
 485                                         print "\tPlease move them to boot/kernel/"+x+"/packages in your specfile"
 486                                         if type(addlargs["boot/kernel/"+x+"/postconf"]) == types.StringType:
 487                                                 loop2=[addlargs["boot/kernel/"+x+"/postconf"]]
 488                                         else:
 489                                                 loop2=addlargs["boot/kernel/"+x+"/postconf"]
 490
 491                                         for y in loop2:
 492                                                 if not addlargs.has_key("boot/kernel/"+x+"/packages"):
 493                                                         addlargs["boot/kernel/"+x+"/packages"]=[y]
 494                                                 else:
 495                                                         addlargs["boot/kernel/"+x+"/packages"].append(y)
 496
 497         def set_build_kernel_vars(self):
 498             if self.settings.has_key(self.settings["spec_prefix"]+"/devmanager"):
 499                 self.settings["devmanager"]=self.settings[self.settings["spec_prefix"]+"/devmanager"]
 500                 del self.settings[self.settings["spec_prefix"]+"/devmanager"]
 501
 502             if self.settings.has_key(self.settings["spec_prefix"]+"/splashtype"):
 503                 self.settings["splashtype"]=self.settings[self.settings["spec_prefix"]+"/splashtype"]
 504                 del self.settings[self.settings["spec_prefix"]+"/splashtype"]
 505
 506             if self.settings.has_key(self.settings["spec_prefix"]+"/gk_mainargs"):
 507                 self.settings["gk_mainargs"]=self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
 508                 del self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
 509
 510         def kill_chroot_pids(self):
 511             print "Checking for processes running in chroot and killing them."
 512
 513             # Force environment variables to be exported so script can see them
 514             self.setup_environment()
 515
 516             if os.path.exists(self.settings["sharedir"]+"/targets/support/kill-chroot-pids.sh"):
 517                             cmd("/bin/bash "+self.settings["sharedir"]+"/targets/support/kill-chroot-pids.sh",\
 518                                                 "kill-chroot-pids script failed.",env=self.env)
 519
 520         def mount_safety_check(self):
 521                 mypath=self.settings["chroot_path"]
 522
 523                 """
 524                 check and verify that none of our paths in mypath are mounted. We don't want to clean
 525                 up with things still mounted, and this allows us to check.
 526                 returns 1 on ok, 0 on "something is still mounted" case.
 527                 """
 528                 if not os.path.exists(mypath):
 529                         return
 530
 531                 for x in self.mounts:
 532                         if not os.path.exists(mypath+x):
 533                                 continue
 534
 535                         if ismount(mypath+x):
 536                                 #something is still mounted
 537                                 try:
 538                                         print x+" is still mounted; performing auto-bind-umount...",
 539                                         # try to umount stuff ourselves
 540                                         self.unbind()
 541                                         if ismount(mypath+x):
 542                                                 raise CatalystError, "Auto-unbind failed for "+x
 543
 544                                         else:
 545                                                 print "Auto-unbind successful..."
 546
 547                                 except CatalystError:
 548                                         raise CatalystError, "Unable to auto-unbind "+x
 549
 550         def unpack(self):
 551                 unpack=True
 552
 553                 clst_unpack_hash=read_from_clst(self.settings["autoresume_path"]+"unpack")
 554
 555                 if self.settings.has_key("SEEDCACHE"):
 556                         if os.path.isdir(self.settings["source_path"]):
 557                                 # SEEDCACHE Is a directory, use Rsync
 558                                 unpack_cmd="rsync -a --delete "+self.settings["source_path"]+" "+self.settings["chroot_path"]
 559                                 display_msg="\nStarting rsync from "+self.settings["source_path"]+"\nto "+\
 560                                         self.settings["chroot_path"]+" (This may take some time) ...\n"
 561                                 error_msg="Rsync of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 562                         else:
 563                                 # SEEDCACHE is a not a directory, try untar'ing
 564                                 print "Referenced SEEDCACHE does not appear to be a directory, trying to untar..."
 565                                 display_msg="\nStarting tar extract from "+self.settings["source_path"]+"\nto "+\
 566                                         self.settings["chroot_path"]+" (This may take some time) ...\n"
 567                                 unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"]
 568                                 error_msg="Tarball extraction of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 569                 else:
 570                         # No SEEDCACHE, use tar
 571                         display_msg="\nStarting tar extract from "+self.settings["source_path"]+"\nto "+\
 572                                 self.settings["chroot_path"]+" (This may take some time) ...\n"
 573                         unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"]
 574                         error_msg="Tarball extraction of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 575
 576
 577                 if self.settings.has_key("AUTORESUME"):
 578                         # Autoresume is Valid, SeedCache is Valid
 579                         if os.path.isdir(self.settings["source_path"]) and os.path.exists(self.settings["autoresume_path"]+"unpack"):
 580                                 unpack=False
 581                                 invalid_snapshot=False
 582
 583                         # Autoresume is Valid, Tarball is Valid
 584                         elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] == clst_unpack_hash:
 585                                 unpack=False
 586                                 invalid_snapshot=True
 587
 588                         # Autoresume is InValid, SeedCache
 589                         elif os.path.isdir(self.settings["source_path"]) and not os.path.exists(self.settings["autoresume_path"]+"unpack"):
 590                                 unpack=True
 591                                 invalid_snapshot=False
 592
 593                         # Autoresume is InValid, Tarball
 594                         elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] != clst_unpack_hash:
 595                                 unpack=True
 596                                 invalid_snapshot=True
 597                 else:
 598                         # No Autoresume,SeedCache
 599                         if self.settings.has_key("SEEDCACHE"):
 600
 601                                 # Seed cache so lets run rsync and rsync can clean up extra junk
 602                                 if os.path.isdir(self.settings["source_path"]):
 603                                         unpack=True
 604                                         invalid_snapshot=False
 605
 606                                 # Tarball so we better unpack and remove anything already there
 607                                 elif os.path.isfile(self.settings["source_path"]):
 608                                         unpack=True
 609                                         invalid_snapshot=True
 610
 611                         # No Autoresume,No SeedCache
 612                         else:
 613
 614                                 # Tarball so we better unpack and remove anything already there
 615                                 if os.path.isfile(self.settings["source_path"]):
 616                                         unpack=True
 617                                         invalid_snapshot=True
 618                                 # Should never reach this if so something is very wrong
 619                                 elif os.path.isdir(self.settings["source_path"]):
 620                                         raise CatalystError,"source path is a dir but seedcache is not enabled"
 621
 622                 if unpack:
 623                         self.mount_safety_check()
 624
 625                         if invalid_snapshot:
 626                                 if self.settings.has_key("AUTORESUME"):
 627                                         print "No Valid Resume point detected, cleaning up  ..."
 628
 629                                 self.clear_autoresume()
 630                                 self.clear_chroot()
 631
 632                         if not os.path.exists(self.settings["chroot_path"]):
 633                                 os.makedirs(self.settings["chroot_path"])
 634
 635                         if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
 636                                 os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
 637
 638                         if self.settings.has_key("PKGCACHE"):
 639                                 if not os.path.exists(self.settings["pkgcache_path"]):
 640                                         os.makedirs(self.settings["pkgcache_path"],0755)
 641
 642                         if self.settings.has_key("KERNCACHE"):
 643                                 if not os.path.exists(self.settings["kerncache_path"]):
 644                                         os.makedirs(self.settings["kerncache_path"],0755)
 645
 646                         print display_msg
 647                         cmd(unpack_cmd,error_msg,env=self.env)
 648
 649                         if self.settings.has_key("source_path_hash"):
 650                                 myf=open(self.settings["autoresume_path"]+"unpack","w")
 651                                 myf.write(self.settings["source_path_hash"])
 652                                 myf.close()
 653                         else:
 654                                 touch(self.settings["autoresume_path"]+"unpack")
 655                 else:
 656                     print "Resume point detected, skipping unpack operation..."
 657
 658
 659         def unpack_snapshot(self):
 660                 unpack=True
 661                 snapshot_hash=read_from_clst(self.settings["autoresume_path"]+"unpack_portage")
 662
 663                 if self.settings.has_key("SNAPCACHE"):
 664                         snapshot_cache_hash=read_from_clst(self.settings["snapshot_cache_path"]+"catalyst-hash")
 665                         destdir=self.settings["snapshot_cache_path"]
 666                         unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+destdir
 667                         unpack_errmsg="Error unpacking snapshot"
 668                         cleanup_msg="Cleaning up invalid snapshot cache at \n\t"+self.settings["snapshot_cache_path"]+" (This can take a long time)..."
 669                         cleanup_errmsg="Error removing existing snapshot cache directory."
 670                         self.snapshot_lock_object=self.snapcache_lock
 671
 672                         if self.settings["snapshot_path_hash"] == snapshot_cache_hash:
 673                                 print "Valid snapshot cache, skipping unpack of portage tree ..."
 674                                 unpack=False
 675
 676                 else:
 677                         destdir=normpath(self.settings["chroot_path"]+"/usr/portage")
 678                         cleanup_errmsg="Error removing existing snapshot directory."
 679                         cleanup_msg="Cleaning up existing portage tree (This can take a long time) ..."
 680                         unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+self.settings["chroot_path"]+"/usr"
 681                         unpack_errmsg="Error unpacking snapshot"
 682
 683                         if self.settings.has_key("AUTORESUME") \
 684                         and os.path.exists(self.settings["chroot_path"]+"/usr/portage/") \
 685                         and os.path.exists(self.settings["autoresume_path"]+"unpack_portage") \
 686                         and self.settings["snapshot_path_hash"] == snapshot_hash:
 687                                 print "Valid Resume point detected, skipping unpack of portage tree..."
 688                                 unpack=False
 689
 690
 691
 692                 if unpack:
 693                         if self.settings.has_key("SNAPCACHE"):
 694                             self.snapshot_lock_object.write_lock()
 695                         if os.path.exists(destdir):
 696                                 print cleanup_msg
 697                                 cleanup_cmd="rm -rf "+destdir
 698                                 cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
 699                         if not os.path.exists(destdir):
 700                                 os.makedirs(destdir,0755)
 701
 702                         print "Unpacking portage tree (This can take a long time) ..."
 703                         cmd(unpack_cmd,unpack_errmsg,env=self.env)
 704
 705                         if self.settings.has_key("SNAPCACHE"):
 706                                 myf=open(self.settings["snapshot_cache_path"]+"catalyst-hash","w")
 707                                 myf.write(self.settings["snapshot_path_hash"])
 708                                 myf.close()
 709
 710                         else:
 711                                 print "Setting snapshot autoresume point"
 712                                 myf=open(self.settings["autoresume_path"]+"unpack_portage","w")
 713                                 myf.write(self.settings["snapshot_path_hash"])
 714                                 myf.close()
 715
 716                         if self.settings.has_key("SNAPCACHE"):
 717                             self.snapshot_lock_object.unlock()
 718
 719         def config_profile_link(self):
 720                 if self.settings.has_key("AUTORESUME") \
 721                         and os.path.exists(self.settings["autoresume_path"]+"config_profile_link"):
 722                         print "Resume point detected, skipping config_profile_link operation..."
 723                 else:
 724                         print "Configuring profile link..."
 725                         cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.profile",\
 726                                         "Error zapping profile link",env=self.env)
 727                         cmd("ln -sf ../usr/portage/profiles/"+self.settings["target_profile"]+\
 728                                 " "+self.settings["chroot_path"]+"/etc/make.profile","Error creating profile link",env=self.env)
 729                         touch(self.settings["autoresume_path"]+"config_profile_link")
 730
 731         def setup_confdir(self):
 732                 if self.settings.has_key("AUTORESUME") \
 733                         and os.path.exists(self.settings["autoresume_path"]+"setup_confdir"):
 734                         print "Resume point detected, skipping setup_confdir operation..."
 735                 else:
 736                         if self.settings.has_key("portage_confdir"):
 737                                 print "Configuring /etc/portage..."
 738                                 cmd("rm -rf "+self.settings["chroot_path"]+"/etc/portage","Error zapping /etc/portage",env=self.env)
 739                                 cmd("cp -R "+self.settings["portage_confdir"]+"/ "+self.settings["chroot_path"]+\
 740                                         "/etc/portage","Error copying /etc/portage",env=self.env)
 741                                 touch(self.settings["autoresume_path"]+"setup_confdir")
 742
 743         def portage_overlay(self):
 744             # copy over the portage overlays
 745             # Always copy over the overlay incase it has changed
 746             if self.settings.has_key("portage_overlay"):
 747                 for x in self.settings["portage_overlay"]:
 748                         if os.path.exists(x):
 749                                 print "Copying overlay dir " +x
 750                                 cmd("mkdir -p "+self.settings["chroot_path"]+x,"Could not make portage_overlay dir",env=self.env)
 751                                 #cmd("rsync -a --delete "+x+"/* "+self.settings["chroot_path"]+x,\
 752                                 cmd("rsync -a --delete "+x+"/ "+self.settings["chroot_path"]+x,\
 753                                                 "Could not copy portage_overlay",env=self.env)
 754                                 #cmd("cp -R "+x+"/* "+self.settings["chroot_path"]+x,"Could not copy portage_overlay",env=self.env)
 755
 756         def root_overlay(self):
 757             # copy over the root_overlay
 758             # Always copy over the overlay incase it has changed
 759                 if self.settings.has_key(self.settings["spec_prefix"]+"/root_overlay"):
 760                         for x in self.settings[self.settings["spec_prefix"]+"/root_overlay"]:
 761                                 if os.path.exists(x):
 762                                         print "Copying root_overlay: "+x
 763                                         cmd("rsync -a "+x+"/ "+\
 764                                                 self.settings["chroot_path"], self.settings["spec_prefix"]+"/root_overlay: "+x+" copy failed.",env=self.env)
 765
 766         def bind(self):
 767                 for x in self.mounts:
 768                         if not os.path.exists(self.settings["chroot_path"]+x):
 769                                 os.makedirs(self.settings["chroot_path"]+x,0755)
 770
 771                         if not os.path.exists(self.mountmap[x]):
 772                                 os.makedirs(self.mountmap[x],0755)
 773
 774                         src=self.mountmap[x]
 775                         if self.settings.has_key("SNAPCACHE") and x == "/usr/portage":
 776                                 self.snapshot_lock_object.read_lock()
 777                         if os.uname()[0] == "FreeBSD":
 778                                 if src == "/dev":
 779                                         retval=os.system("mount -t devfs none "+self.settings["chroot_path"]+x)
 780                                 else:
 781                                         retval=os.system("mount_nullfs "+src+" "+self.settings["chroot_path"]+x)
 782                         else:
 783                                 retval=os.system("mount --bind "+src+" "+self.settings["chroot_path"]+x)
 784                         if retval!=0:
 785                                 self.unbind()
 786                                 raise CatalystError,"Couldn't bind mount "+src
 787
 788
 789         def unbind(self):
 790                 ouch=0
 791                 mypath=self.settings["chroot_path"]
 792                 myrevmounts=self.mounts[:]
 793                 myrevmounts.reverse()
 794                 # unmount in reverse order for nested bind-mounts
 795                 for x in myrevmounts:
 796                         if not os.path.exists(mypath+x):
 797                                 continue
 798
 799                         if not ismount(mypath+x):
 800                                 # it's not mounted, continue
 801                                 continue
 802
 803                         retval=os.system("umount "+os.path.join(mypath,x.lstrip(os.path.sep)))
 804
 805                         if retval!=0:
 806                                 warn("First attempt to unmount: "+mypath+x+" failed.")
 807                                 warn("Killing any pids still running in the chroot")
 808
 809                                 self.kill_chroot_pids()
 810
 811                                 retval2=os.system("umount "+mypath+x)
 812                                 if retval2!=0:
 813                                     ouch=1
 814                                     warn("Couldn't umount bind mount: "+mypath+x)
 815                                     # keep trying to umount the others, to minimize damage if developer makes a mistake
 816
 817                         if self.settings.has_key("SNAPCACHE") and x == "/usr/portage":
 818                                 try:
 819                                     # Its possible the snapshot lock object isnt created yet
 820                                     # this is because mount safety check calls unbind before the target is fully initialized
 821                                     self.snapshot_lock_object.unlock()
 822                                 except:
 823                                     pass
 824                 if ouch:
 825                         """
 826                         if any bind mounts really failed, then we need to raise
 827                         this to potentially prevent an upcoming bash stage cleanup script
 828                         from wiping our bind mounts.
 829                         """
 830                         raise CatalystError,"Couldn't umount one or more bind-mounts; aborting for safety."
 831
 832         def chroot_setup(self):
 833                 self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/make.conf")
 834                 self.override_chost()
 835                 self.override_cflags()
 836                 self.override_cxxflags()
 837                 self.override_ldflags()
 838                 if self.settings.has_key("AUTORESUME") \
 839                         and os.path.exists(self.settings["autoresume_path"]+"chroot_setup"):
 840                         print "Resume point detected, skipping chroot_setup operation..."
 841                 else:
 842                         print "Setting up chroot..."
 843
 844                         #self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/make.conf")
 845
 846                         cmd("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc",\
 847                                 "Could not copy resolv.conf into place.",env=self.env)
 848
 849                     # copy over the envscript, if applicable
 850                         if self.settings.has_key("ENVSCRIPT"):
 851                                 if not os.path.exists(self.settings["ENVSCRIPT"]):
 852                                         raise CatalystError, "Can't find envscript "+self.settings["ENVSCRIPT"]
 853
 854                                 print "\nWarning!!!!"
 855                                 print "\tOverriding certain env variables may cause catastrophic failure."
 856                                 print "\tIf your build fails look here first as the possible problem."
 857                                 print "\tCatalyst assumes you know what you are doing when setting"
 858                                 print "\t\tthese variables."
 859                                 print "\tCatalyst Maintainers use VERY minimal envscripts if used at all"
 860                                 print "\tYou have been warned\n"
 861
 862                                 cmd("cp "+self.settings["ENVSCRIPT"]+" "+self.settings["chroot_path"]+"/tmp/envscript",\
 863                                         "Could not copy envscript into place.",env=self.env)
 864
 865                     # Copy over /etc/hosts from the host in case there are any
 866                         # specialties in there
 867                         if os.path.exists(self.settings["chroot_path"]+"/etc/hosts"):
 868                                 cmd("mv "+self.settings["chroot_path"]+"/etc/hosts "+self.settings["chroot_path"]+\
 869                                         "/etc/hosts.bck", "Could not backup /etc/hosts",env=self.env)
 870                                 cmd("cp /etc/hosts "+self.settings["chroot_path"]+"/etc/hosts", "Could not copy /etc/hosts",env=self.env)
 871                         #self.override_chost()
 872                         #self.override_cflags()
 873                         #self.override_cxxflags()
 874                         #self.override_ldflags()
 875                         # Modify and write out make.conf (for the chroot)
 876                         cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.conf","Could not remove "+self.settings["chroot_path"]+"/etc/make.conf",\
 877                                 env=self.env)
 878                         myf=open(self.settings["chroot_path"]+"/etc/make.conf","w")
 879                         myf.write("# These settings were set by the catalyst build script that automatically\n# built this stage.\n")
 880                         myf.write("# Please consult /etc/make.conf.example for a more detailed example.\n")
 881                         myf.write('CFLAGS="'+self.settings["CFLAGS"]+'"\n')
 882                         if self.settings.has_key("CXXFLAGS"):
 883                                 myf.write('CXXFLAGS="'+self.settings["CXXFLAGS"]+'"\n')
 884                         else:
 885                                 myf.write('CXXFLAGS="${CFLAGS}"\n')
 886
 887                         if self.settings.has_key("LDFLAGS"):
 888                                 myf.write('LDFLAGS="'+self.settings["LDFLAGS"]+'"\n')
 889                         myf.write("# This should not be changed unless you know exactly what you are doing.  You\n# should probably be using a different stage, instead.\m")
 890                         myf.write('CHOST="'+self.settings["CHOST"]+'"\n')
 891
 892                     # Figure out what our USE vars are for building
 893                         myusevars=[]
 894                         if self.settings.has_key("HOSTUSE"):
 895                                 myusevars.extend(self.settings["HOSTUSE"])
 896
 897                         if self.settings.has_key("use"):
 898                                 myusevars.extend(self.settings["use"])
 899                                 myf.write('USE="'+string.join(myusevars)+'"\n')
 900
 901                     # Setup the portage overlay
 902                         if self.settings.has_key("portage_overlay"):
 903                                 myf.write('PORTDIR_OVERLAY="'+string.join(self.settings["portage_overlay"])+'"\n')
 904
 905                         myf.close()
 906                         touch(self.settings["autoresume_path"]+"chroot_setup")
 907
 908         def fsscript(self):
 909                 if self.settings.has_key("AUTORESUME") \
 910                         and os.path.exists(self.settings["autoresume_path"]+"fsscript"):
 911                         print "Resume point detected, skipping fsscript operation..."
 912                 else:
 913                     if self.settings.has_key("fsscript"):
 914                         if os.path.exists(self.settings["controller_file"]):
 915                             cmd("/bin/bash "+self.settings["controller_file"]+" fsscript","fsscript script failed.",env=self.env)
 916                         touch(self.settings["autoresume_path"]+"fsscript")
 917
 918         def rcupdate(self):
 919                 if self.settings.has_key("AUTORESUME") \
 920                         and os.path.exists(self.settings["autoresume_path"]+"rcupdate"):
 921                         print "Resume point detected, skipping rcupdate operation..."
 922                 else:
 923                     if os.path.exists(self.settings["controller_file"]):
 924                         cmd("/bin/bash "+self.settings["controller_file"]+" rc-update","rc-update script failed.",env=self.env)
 925                         touch(self.settings["autoresume_path"]+"rcupdate")
 926
 927         def clean(self):
 928                 if self.settings.has_key("AUTORESUME") \
 929                         and os.path.exists(self.settings["autoresume_path"]+"clean"):
 930                         print "Resume point detected, skipping clean operation..."
 931                 else:
 932                     for x in self.settings["cleanables"]:
 933                             print "Cleaning chroot: "+x+"... "
 934                             cmd("rm -rf "+self.settings["destpath"]+x,"Couldn't clean "+x,env=self.env)
 935
 936                     # put /etc/hosts back into place
 937                     if os.path.exists(self.settings["chroot_path"]+"/etc/hosts.bck"):
 938                             cmd("mv -f "+self.settings["chroot_path"]+"/etc/hosts.bck "+self.settings["chroot_path"]+\
 939                                           "/etc/hosts", "Could not replace /etc/hosts",env=self.env)
 940
 941                     if os.path.exists(self.settings["controller_file"]):
 942                         cmd("/bin/bash "+self.settings["controller_file"]+" clean","clean script failed.",env=self.env)
 943                         touch(self.settings["autoresume_path"]+"clean")
 944
 945         def empty(self):
 946             if self.settings.has_key("AUTORESUME") \
 947                 and os.path.exists(self.settings["autoresume_path"]+"empty"):
 948                 print "Resume point detected, skipping empty operation..."
 949             else:
 950                 if self.settings.has_key(self.settings["spec_prefix"]+"/empty"):
 951                     if type(self.settings[self.settings["spec_prefix"]+"/empty"])==types.StringType:
 952                         self.settings[self.settings["spec_prefix"]+"/empty"]=self.settings[self.settings["spec_prefix"]+"/empty"].split()
 953                     for x in self.settings[self.settings["spec_prefix"]+"/empty"]:
 954                         myemp=self.settings["destpath"]+x
 955                         if not os.path.isdir(myemp):
 956                             print x,"not a directory or does not exist, skipping 'empty' operation."
 957                             continue
 958                         print "Emptying directory",x
 959                         # stat the dir, delete the dir, recreate the dir and set
 960                         # the proper perms and ownership
 961                         mystat=os.stat(myemp)
 962                         shutil.rmtree(myemp)
 963                         os.makedirs(myemp,0755)
 964                         os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 965                         os.chmod(myemp,mystat[ST_MODE])
 966                     touch(self.settings["autoresume_path"]+"empty")
 967
 968         def remove(self):
 969             if self.settings.has_key("AUTORESUME") \
 970                 and os.path.exists(self.settings["autoresume_path"]+"remove"):
 971                 print "Resume point detected, skipping remove operation..."
 972             else:
 973                 if self.settings.has_key(self.settings["spec_prefix"]+"/rm"):
 974                     for x in self.settings[self.settings["spec_prefix"]+"/rm"]:
 975                         # we're going to shell out for all these cleaning operations,
 976                         # so we get easy glob handling
 977                         print "livecd: removing "+x
 978                         os.system("rm -rf "+self.settings["chroot_path"]+x)
 979                 try:
 980                     if os.path.exists(self.settings["controller_file"]):
 981                             cmd("/bin/bash "+self.settings["controller_file"]+" clean",\
 982                                 "Clean  failed.",env=self.env)
 983                             touch(self.settings["autoresume_path"]+"remove")
 984                 except:
 985                     self.unbind()
 986                     raise
 987
 988
 989         def preclean(self):
 990             if self.settings.has_key("AUTORESUME") \
 991                 and os.path.exists(self.settings["autoresume_path"]+"preclean"):
 992                 print "Resume point detected, skipping preclean operation..."
 993             else:
 994                 try:
 995                         if os.path.exists(self.settings["controller_file"]):
 996                                 cmd("/bin/bash "+self.settings["controller_file"]+" preclean","preclean script failed.",env=self.env)
 997                                 touch(self.settings["autoresume_path"]+"preclean")
 998
 999                 except:
1000                         self.unbind()
1001                         raise CatalystError, "Build failed, could not execute preclean"
1002
1003         def capture(self):
1004             if self.settings.has_key("AUTORESUME") \
1005                 and os.path.exists(self.settings["autoresume_path"]+"capture"):
1006                 print "Resume point detected, skipping capture operation..."
1007             else:
1008                 """capture target in a tarball"""
1009                 mypath=self.settings["target_path"].split("/")
1010                 # remove filename from path
1011                 mypath=string.join(mypath[:-1],"/")
1012
1013                 # now make sure path exists
1014                 if not os.path.exists(mypath):
1015                         os.makedirs(mypath)
1016
1017                 print "Creating stage tarball..."
1018
1019                 cmd("tar cjpf "+self.settings["target_path"]+" -C "+self.settings["stage_path"]+\
1020                         " .","Couldn't create stage tarball",env=self.env)
1021
1022                 self.gen_digest_file(self.settings["target_path"])
1023
1024                 touch(self.settings["autoresume_path"]+"capture")
1025
1026         def run_local(self):
1027             if self.settings.has_key("AUTORESUME") \
1028                 and os.path.exists(self.settings["autoresume_path"]+"run_local"):
1029                 print "Resume point detected, skipping run_local operation..."
1030             else:
1031                 try:
1032                         if os.path.exists(self.settings["controller_file"]):
1033                                 cmd("/bin/bash "+self.settings["controller_file"]+" run","run script failed.",env=self.env)
1034                                 touch(self.settings["autoresume_path"]+"run_local")
1035
1036                 except CatalystError:
1037                         self.unbind()
1038                         raise CatalystError,"Stage build aborting due to error."
1039
1040         def setup_environment(self):
1041                 # Modify the current environment. This is an ugly hack that should be
1042                 # fixed. We need this to use the os.system() call since we can't
1043                 # specify our own environ:
1044                 for x in self.settings.keys():
1045                         # "/" is replaced with "_", "-" is replaced with "_", and "." is
1046                         # replaced with "_"
1047                         varname="clst_"+string.replace(x,"/","_")
1048                         varname=string.replace(varname,"-","_")
1049                         varname=string.replace(varname,".","_")
1050                         if type(self.settings[x])==types.StringType:
1051                                 # prefix to prevent namespace clashes:
1052                                 #os.environ[varname]=self.settings[x]
1053                                 self.env[varname]=self.settings[x]
1054                         elif type(self.settings[x])==types.ListType:
1055                                 #os.environ[varname]=string.join(self.settings[x])
1056                                 self.env[varname]=string.join(self.settings[x])
1057                 if self.settings.has_key("makeopts"):
1058                         self.env["MAKEOPTS"]=self.settings["makeopts"]
1059
1060         def run(self):
1061                 self.chroot_lock.write_lock()
1062
1063                 # Kill any pids in the chroot
1064                 self.kill_chroot_pids()
1065
1066                 # Check for mounts right away and abort if we cannot unmount them.
1067                 self.mount_safety_check()
1068
1069                 if self.settings.has_key("CLEAR_AUTORESUME"):
1070                         self.clear_autoresume()
1071                 if self.settings.has_key("PURGE"):
1072                         self.purge()
1073
1074                 for x in self.settings["action_sequence"]:
1075                         print "Running action sequence: "+x
1076                         sys.stdout.flush()
1077                         try:
1078                                 apply(getattr(self,x))
1079                         except:
1080                                 self.mount_safety_check()
1081                                 raise
1082
1083                 self.chroot_lock.unlock()
1084
1085         def unmerge(self):
1086             if self.settings.has_key("AUTORESUME") \
1087                 and os.path.exists(self.settings["autoresume_path"]+"unmerge"):
1088                     print "Resume point detected, skipping unmerge operation..."
1089             else:
1090                 if self.settings.has_key(self.settings["spec_prefix"]+"/unmerge"):
1091                     if type(self.settings[self.settings["spec_prefix"]+"/unmerge"])==types.StringType:
1092                         self.settings[self.settings["spec_prefix"]+"/unmerge"]=[self.settings[self.settings["spec_prefix"]+"/unmerge"]]
1093                     myunmerge=self.settings[self.settings["spec_prefix"]+"/unmerge"][:]
1094
1095                     for x in range(0,len(myunmerge)):
1096                     #surround args with quotes for passing to bash,
1097                     #allows things like "<" to remain intact
1098                         myunmerge[x]="'"+myunmerge[x]+"'"
1099                     myunmerge=string.join(myunmerge)
1100
1101                     #before cleaning, unmerge stuff:
1102                     try:
1103                         cmd("/bin/bash "+self.settings["controller_file"]+" unmerge "+ myunmerge,\
1104                                 "Unmerge script failed.",env=self.env)
1105                         #cmd("/bin/bash "+self.settings["sharedir"]+"/targets/" \
1106                         #       +self.settings["target"]+"/unmerge.sh "+myunmerge,"Unmerge script failed.",env=self.env)
1107                         print "unmerge shell script"
1108                     except CatalystError:
1109                         self.unbind()
1110                         raise
1111
1112                 touch(self.settings["autoresume_path"]+"unmerge")
1113
1114         def target_setup(self):
1115             if self.settings.has_key("AUTORESUME") \
1116                 and os.path.exists(self.settings["autoresume_path"]+"target_setup"):
1117                     print "Resume point detected, skipping target_setup operation..."
1118             else:
1119                 print "Setting up filesystems per filesystem type"
1120                 cmd("/bin/bash "+self.settings["controller_file"]+" target_image_setup "+ self.settings["target_path"],\
1121                                 "target_image_setup script failed.",env=self.env)
1122                 touch(self.settings["autoresume_path"]+"target_setup")
1123
1124         def setup_overlay(self):
1125                 if self.settings.has_key("AUTORESUME") \
1126                 and os.path.exists(self.settings["autoresume_path"]+"setup_overlay"):
1127                         print "Resume point detected, skipping setup_overlay operation..."
1128                 else:
1129                         if self.settings.has_key(self.settings["spec_prefix"]+"/overlay"):
1130                                 for x in self.settings[self.settings["spec_prefix"]+"/overlay"]:
1131                                         if os.path.exists(x):
1132                                                 cmd("rsync -a "+x+"/ "+\
1133                                                         self.settings["target_path"], self.settings["spec_prefix"]+"overlay: "+x+" copy failed.",env=self.env)
1134                                 touch(self.settings["autoresume_path"]+"setup_overlay")
1135
1136         def create_iso(self):
1137             if self.settings.has_key("AUTORESUME") \
1138                 and os.path.exists(self.settings["autoresume_path"]+"create_iso"):
1139                     print "Resume point detected, skipping create_iso operation..."
1140             else:
1141                 # create the ISO - this is the preferred method (the iso scripts do not always work)
1142                 if self.settings.has_key("iso"):
1143                         cmd("/bin/bash "+self.settings["controller_file"]+" iso "+\
1144                                 self.settings["iso"],"ISO creation script failed.",env=self.env)
1145                         self.gen_digest_file(self.settings["iso"])
1146                         touch(self.settings["autoresume_path"]+"create_iso")
1147
1148
1149                 else:
1150                         print "WARNING livecd/iso was not defined."
1151                         print "A CD Image will not be created, skipping create-iso.sh..."
1152
1153
1154         def build_packages(self):
1155             if self.settings.has_key("AUTORESUME") \
1156                 and os.path.exists(self.settings["autoresume_path"]+"build_packages"):
1157                     print "Resume point detected, skipping build_packages operation..."
1158             else:
1159                 if self.settings.has_key(self.settings["spec_prefix"]+"/packages"):
1160                         if self.settings.has_key("AUTORESUME") \
1161                                 and os.path.exists(self.settings["autoresume_path"]+"build_packages"):
1162                                         print "Resume point detected, skipping build_packages operation..."
1163                         else:
1164                                 mypack=list_bashify(self.settings[self.settings["spec_prefix"]+"/packages"])
1165                                 try:
1166                                         cmd("/bin/bash "+self.settings["controller_file"]+\
1167                                                 " build_packages "+mypack,"Error in attempt to build packages",env=self.env)
1168                                         touch(self.settings["autoresume_path"]+"build_packages")
1169                                 except CatalystError:
1170                                         self.unbind()
1171                                         raise CatalystError,self.settings["spec_prefix"] + "build aborting due to error."
1172
1173         def build_kernel(self):
1174                 if self.settings.has_key("AUTORESUME") \
1175                 and os.path.exists(self.settings["autoresume_path"]+"build_kernel"):
1176                         print "Resume point detected, skipping build_kernel operation..."
1177                 else:
1178                         if self.settings.has_key("boot/kernel"):
1179                                 try:
1180                                         mynames=self.settings["boot/kernel"]
1181                                         if type(mynames)==types.StringType:
1182                                                 mynames=[mynames]
1183                                         # execute the script that sets up the kernel build environment
1184                                         cmd("/bin/bash "+self.settings["controller_file"]+" pre-kmerge ",\
1185                                                 "Runscript pre-kmerge failed",env=self.env)
1186
1187                                         for kname in mynames:
1188                                                 if self.settings.has_key("AUTORESUME") \
1189                                                         and os.path.exists(self.settings["autoresume_path"]+"build_kernel_"+kname):
1190                                                         print "Resume point detected, skipping build_kernel for "+kname+" operation..."
1191                                                 else:
1192                                                         try:
1193                                                                 if not os.path.exists(self.settings["boot/kernel/"+kname+"/config"]):
1194                                                                         self.unbind()
1195                                                                         raise CatalystError, "Can't find kernel config: " \
1196                                                                                 +self.settings["boot/kernel/"+kname+"/config"]
1197
1198                                                         except TypeError:
1199                                                                 raise CatalystError, "Required value boot/kernel/config not specified"
1200
1201                                                         try:
1202                                                                 cmd("cp "+self.settings["boot/kernel/"+kname+"/config"]+" "+ \
1203                                                                         self.settings["chroot_path"]+"/var/tmp/"+kname+".config", \
1204                                                                         "Couldn't copy kernel config: "+self.settings["boot/kernel/"+kname+"/config"],\
1205                                                                         env=self.env)
1206
1207                                                         except CatalystError:
1208                                                                 self.unbind()
1209
1210                                                         # If we need to pass special options to the bootloader
1211                                                         # for this kernel put them into the environment.
1212                                                         if self.settings.has_key("boot/kernel/"+kname+"/kernelopts"):
1213                                                                 myopts=self.settings["boot/kernel/"+kname+"/kernelopts"]
1214
1215                                                                 if type(myopts) != types.StringType:
1216                                                                         myopts = string.join(myopts)
1217                                                                         self.env[kname+"_kernelopts"]=myopts
1218
1219                                                                 else:
1220                                                                         self.env[kname+"_kernelopts"]=""
1221
1222                                                         if not self.settings.has_key("boot/kernel/"+kname+"/extraversion"):
1223                                                                 self.settings["boot/kernel/"+kname+"/extraversion"]=""
1224
1225                                                         self.env["clst_kextraversion"]=self.settings["boot/kernel/"+kname+"/extraversion"]
1226
1227                                                         if self.settings.has_key("boot/kernel/"+kname+"/initramfs_overlay"):
1228                                                                 if os.path.exists(self.settings["boot/kernel/"+kname+"/initramfs_overlay"]):
1229                                                                         print "Copying initramfs_overlay dir " +self.settings["boot/kernel/"+kname+"/initramfs_overlay"]
1230
1231                                                                         cmd("mkdir -p "+self.settings["chroot_path"]+"/tmp/initramfs_overlay/" + \
1232                                                                                 self.settings["boot/kernel/"+kname+"/initramfs_overlay"],env=self.env)
1233
1234                                                                         cmd("cp -R "+self.settings["boot/kernel/"+kname+"/initramfs_overlay"]+"/* " + \
1235                                                                                 self.settings["chroot_path"] + "/tmp/initramfs_overlay/" + \
1236                                                                                 self.settings["boot/kernel/"+kname+"/initramfs_overlay"],\
1237                                                                                 env=self.env)
1238
1239
1240                                                         # execute the script that builds the kernel
1241                                                         cmd("/bin/bash "+self.settings["controller_file"]+" kernel "+kname,\
1242                                                                 "Runscript kernel build failed",env=self.env)
1243
1244                                                         if self.settings.has_key("boot/kernel/"+kname+"/initramfs_overlay"):
1245                                                                 if os.path.exists(self.settings["chroot_path"]+"/tmp/initramfs_overlay/"):
1246                                                                         print "Cleaning up temporary overlay dir"
1247                                                                         cmd("rm -R "+self.settings["chroot_path"]+"/tmp/initramfs_overlay/",env=self.env)
1248
1249                                                         touch(self.settings["autoresume_path"]+"build_kernel_"+kname)
1250
1251                                                         # execute the script that cleans up the kernel build environment
1252                                                         cmd("/bin/bash "+self.settings["controller_file"]+" post-kmerge ",\
1253                                                                 "Runscript post-kmerge failed",env=self.env)
1254
1255                                         touch(self.settings["autoresume_path"]+"build_kernel")
1256
1257                                 except CatalystError:
1258                                         self.unbind()
1259                                         raise CatalystError,"build aborting due to kernel build error."
1260
1261         def bootloader(self):
1262             if self.settings.has_key("AUTORESUME") \
1263                 and os.path.exists(self.settings["autoresume_path"]+"bootloader"):
1264                     print "Resume point detected, skipping bootloader operation..."
1265             else:
1266                 try:
1267                         cmd("/bin/bash "+self.settings["controller_file"]+" bootloader " + self.settings["target_path"],\
1268                                 "Bootloader runscript failed.",env=self.env)
1269                         touch(self.settings["autoresume_path"]+"bootloader")
1270                 except CatalystError:
1271                         self.unbind()
1272                         raise CatalystError,"Runscript aborting due to error."
1273
1274         def livecd_update(self):
1275             if self.settings.has_key("AUTORESUME") \
1276                 and os.path.exists(self.settings["autoresume_path"]+"livecd_update"):
1277                     print "Resume point detected, skipping build_packages operation..."
1278             else:
1279                 try:
1280                         cmd("/bin/bash "+self.settings["controller_file"]+" livecd-update",\
1281                                 "livecd-update failed.",env=self.env)
1282                         touch(self.settings["autoresume_path"]+"livecd_update")
1283
1284                 except CatalystError:
1285                         self.unbind()
1286                         raise CatalystError,"build aborting due to livecd_update error."
1287
1288         def clear_chroot(self):
1289                 myemp=self.settings["chroot_path"]
1290                 if os.path.isdir(myemp):
1291                     print "Emptying directory",myemp
1292                     # stat the dir, delete the dir, recreate the dir and set
1293                     # the proper perms and ownership
1294                     mystat=os.stat(myemp)
1295                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1296                     if os.uname()[0] == "FreeBSD": # There's no easy way to change flags recursively in python
1297                             os.system("chflags -R noschg "+myemp)
1298                     shutil.rmtree(myemp)
1299                     os.makedirs(myemp,0755)
1300                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1301                     os.chmod(myemp,mystat[ST_MODE])
1302
1303         def clear_packages(self):
1304             if self.settings.has_key("PKGCACHE"):
1305                 print "purging the pkgcache ..."
1306
1307                 myemp=self.settings["pkgcache_path"]
1308                 if os.path.isdir(myemp):
1309                     print "Emptying directory",myemp
1310                     # stat the dir, delete the dir, recreate the dir and set
1311                     # the proper perms and ownership
1312                     mystat=os.stat(myemp)
1313                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1314                     shutil.rmtree(myemp)
1315                     os.makedirs(myemp,0755)
1316                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1317                     os.chmod(myemp,mystat[ST_MODE])
1318
1319         def clear_kerncache(self):
1320             if self.settings.has_key("KERNCACHE"):
1321                 print "purging the kerncache ..."
1322
1323                 myemp=self.settings["kerncache_path"]
1324                 if os.path.isdir(myemp):
1325                     print "Emptying directory",myemp
1326                     # stat the dir, delete the dir, recreate the dir and set
1327                     # the proper perms and ownership
1328                     mystat=os.stat(myemp)
1329                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1330                     shutil.rmtree(myemp)
1331                     os.makedirs(myemp,0755)
1332                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1333                     os.chmod(myemp,mystat[ST_MODE])
1334
1335         def clear_autoresume(self):
1336                 # clean resume points since they are no longer needed
1337                 if self.settings.has_key("AUTORESUME"):
1338                         print "Removing AutoResume Points: ..."
1339                 myemp=self.settings["autoresume_path"]
1340                 if os.path.isdir(myemp):
1341                                 if self.settings.has_key("AUTORESUME"):
1342                                         print "Emptying directory",myemp
1343                                 # stat the dir, delete the dir, recreate the dir and set
1344                                 # the proper perms and ownership
1345                                 mystat=os.stat(myemp)
1346                                 #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
1347                                 shutil.rmtree(myemp)
1348                                 os.makedirs(myemp,0755)
1349                                 os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1350                                 os.chmod(myemp,mystat[ST_MODE])
1351
1352         def gen_digest_file(self,file):
1353                 if os.path.exists(file+".DIGESTS"):
1354                         os.remove(file+".DIGESTS")
1355                 if self.settings.has_key("digests"):
1356                         if os.path.exists(file):
1357                                 myf=open(file+".DIGESTS","w")
1358                                 keys={}
1359                                 for i in self.settings["digests"].split():
1360                                         keys[i]=1
1361                                         array=keys.keys()
1362                                         array.sort()
1363                                 for j in array:
1364                                         if self.settings.has_key("VERBOSE"):
1365                                                 hash=generate_hash(file,hash_function=j,verbose=True)
1366                                         else:
1367                                                 hash=generate_hash(file,hash_function=j)
1368                                         myf.write(hash)
1369                                 myf.close()
1370
1371         def purge(self):
1372             countdown(10,"Purging Caches ...")
1373             if self.settings.has_key("PURGE"):
1374                 print "clearing autoresume ..."
1375                 self.clear_autoresume()
1376
1377                 print "clearing chroot ..."
1378                 self.clear_chroot()
1379
1380                 print "clearing package cache ..."
1381                 self.clear_packages()
1382
1383                 print "clearing kerncache ..."
1384                 self.clear_kerncache()
1385
1386 #vim: ts=4 sw=4 sta et sts=4 ai