modules/generic_stage_target.py

   1
   2 """
   3 This class does all of the chroot setup, copying of files, etc. It is
   4 the driver class for pretty much everything that Catalyst does.
   5 """
   6
   7 import os,string,imp,types,shutil
   8 from catalyst_support import *
   9 from generic_target import *
  10 from stat import *
  11 import catalyst_lock
  12 class generic_stage_target(generic_target):
  13
  14         def __init__(self,myspec,addlargs):
  15                 self.required_values.extend(["version_stamp","target","subarch",\
  16                         "rel_type","profile","snapshot","source_subpath"])
  17
  18                 self.valid_values.extend(["version_stamp","target","subarch",\
  19                         "rel_type","profile","snapshot","source_subpath","portage_confdir",\
  20                         "cflags","cxxflags","ldflags","cbuild","hostuse","portage_overlay",\
  21                         "distcc_hosts","makeopts","pkgcache_path","kerncache_path"])
  22
  23                 self.set_valid_build_kernel_vars(addlargs)
  24                 generic_target.__init__(self,myspec,addlargs)
  25
  26                 # The semantics of subarchmap and machinemap changed a bit in 2.0.3 to
  27                 # work better with vapier's CBUILD stuff. I've removed the "monolithic"
  28                 # machinemap from this file and split up its contents amongst the various
  29                 # arch/foo.py files.
  30                 #
  31                 # When register() is called on each module in the arch/ dir, it now
  32                 # returns a tuple instead of acting on the subarchmap dict that is
  33                 # passed to it. The tuple contains the values that were previously
  34                 # added to subarchmap as well as a new list of CHOSTs that go along
  35                 # with that arch. This allows us to build machinemap on the fly based
  36                 # on the keys in subarchmap and the values of the 2nd list returned
  37                 # (tmpmachinemap).
  38                 #
  39                 # Also, after talking with vapier. I have a slightly better idea of what
  40                 # certain variables are used for and what they should be set to. Neither
  41                 # 'buildarch' or 'hostarch' are used directly, so their value doesn't
  42                 # really matter. They are just compared to determine if we are
  43                 # cross-compiling. Because of this, they are just set to the name of the
  44                 # module in arch/ that the subarch is part of to make things simpler.
  45                 # The entire build process is still based off of 'subarch' like it was
  46                 # previously. -agaffney
  47
  48                 self.archmap = {}
  49                 self.subarchmap = {}
  50                 machinemap = {}
  51                 for x in [x[:-3] for x in os.listdir(self.settings["sharedir"]+"/arch/") if x.endswith(".py")]:
  52                         try:
  53                                 fh=open(self.settings["sharedir"]+"/arch/"+x+".py")
  54                                 # This next line loads the plugin as a module and assigns it to
  55                                 # archmap[x]
  56                                 self.archmap[x]=imp.load_module(x,fh,"arch/"+x+".py",(".py","r",imp.PY_SOURCE))
  57                                 # This next line registers all the subarches supported in the
  58                                 # plugin
  59                                 tmpsubarchmap, tmpmachinemap = self.archmap[x].register()
  60                                 self.subarchmap.update(tmpsubarchmap)
  61                                 for machine in tmpmachinemap:
  62                                         machinemap[machine] = x
  63                                 for subarch in tmpsubarchmap:
  64                                         machinemap[subarch] = x
  65                                 fh.close()
  66                         except IOError:
  67                                 # This message should probably change a bit, since everything in the dir should
  68                                 # load just fine. If it doesn't, it's probably a syntax error in the module
  69                                 msg("Can't find "+x+".py plugin in "+self.settings["sharedir"]+"/arch/")
  70
  71                 if self.settings.has_key("chost"):
  72                         hostmachine = self.settings["chost"].split("-")[0]
  73                         if not machinemap.has_key(hostmachine):
  74                                 raise CatalystError, "Unknown host machine type "+hostmachine
  75                         self.settings["hostarch"] = machinemap[hostmachine]
  76                 else:
  77                         hostmachine = self.settings["subarch"]
  78                         if machinemap.has_key(hostmachine):
  79                                 hostmachine = machinemap[hostmachine]
  80                         self.settings["hostarch"] = hostmachine
  81                 if self.settings.has_key("cbuild"):
  82                         buildmachine = self.settings["cbuild"].split("-")[0]
  83                 else:
  84                         buildmachine = os.uname()[4]
  85                 if not machinemap.has_key(buildmachine):
  86                         raise CatalystError, "Unknown build machine type "+buildmachine
  87                 self.settings["buildarch"] = machinemap[buildmachine]
  88                 self.settings["crosscompile"] = (self.settings["hostarch"] != self.settings["buildarch"])
  89
  90                 # Call arch constructor, pass our settings
  91                 try:
  92                         self.arch=self.subarchmap[self.settings["subarch"]](self.settings)
  93                 except KeyError:
  94                         print "Invalid subarch: "+self.settings["subarch"]
  95                         print "Choose one of the following:",
  96                         for x in self.subarchmap:
  97                                 print x,
  98                         print
  99                         sys.exit(2)
 100
 101                 print "Using target:",self.settings["target"]
 102                 # print a nice informational message:
 103                 if self.settings["buildarch"]==self.settings["hostarch"]:
 104                         print "Building natively for",self.settings["hostarch"]
 105                 elif self.settings["crosscompile"]:
 106                         print "Cross-compiling on",self.settings["buildarch"],"for different machine type",\
 107                                 self.settings["hostarch"]
 108                 else:
 109                         print "Building on",self.settings["buildarch"],"for alternate personality type",\
 110                                 self.settings["hostarch"]
 111
 112                 # This should be first to be set as other set_ options depend on this
 113                 self.set_spec_prefix()
 114
 115                 # define all of our core variables
 116                 self.set_target_profile()
 117                 self.set_target_subpath()
 118                 self.set_source_subpath()
 119
 120                 # set paths
 121                 self.set_snapshot_path()
 122                 self.set_root_path()
 123                 self.set_source_path()
 124                 self.set_snapcache_path()
 125                 self.set_chroot_path()
 126                 self.set_autoresume_path()
 127                 self.set_dest_path()
 128                 self.set_stage_path()
 129                 self.set_target_path()
 130
 131                 self.set_controller_file()
 132                 self.set_action_sequence()
 133                 self.set_use()
 134                 self.set_cleanables()
 135                 self.set_iso_volume_id()
 136                 self.set_build_kernel_vars()
 137                 self.set_fsscript()
 138                 self.set_archscript()
 139                 self.set_runscript()
 140                 self.set_install_mask()
 141                 self.set_rcadd()
 142                 self.set_rcdel()
 143                 self.set_cdtar()
 144                 self.set_fstype()
 145                 self.set_fsops()
 146                 self.set_iso()
 147                 self.set_packages()
 148                 self.set_rm()
 149                 self.set_linuxrc()
 150                 self.set_overlay()
 151                 self.set_portage_overlay()
 152                 self.set_root_overlay()
 153
 154                 # This next line checks to make sure that the specified variables exist
 155                 # on disk.
 156                 #pdb.set_trace()
 157                 file_locate(self.settings,["source_path","snapshot_path","distdir"],expand=0)
 158                 # If we are using portage_confdir, check that as well.
 159                 if self.settings.has_key("portage_confdir"):
 160                         file_locate(self.settings,["portage_confdir"],expand=0)
 161
 162                 # setup our mount points
 163                 if self.settings.has_key("SNAPCACHE"):
 164                         self.mounts=[ "/proc","/dev","/usr/portage","/usr/portage/distfiles" ]
 165                         self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
 166                                 "/usr/portage":self.settings["snapshot_cache_path"]+"/portage",\
 167                                 "/usr/portage/distfiles":self.settings["distdir"]}
 168                 else:
 169                         self.mounts=[ "/proc","/dev","/usr/portage/distfiles" ]
 170                         self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
 171                                 "/usr/portage/distfiles":self.settings["distdir"]}
 172                 if os.uname()[0] == "Linux":
 173                         self.mounts.append("/dev/pts")
 174
 175                 self.set_mounts()
 176
 177                 # Configure any user specified options (either in catalyst.conf or on
 178                 # the command line).
 179                 if self.settings.has_key("PKGCACHE"):
 180                         self.set_pkgcache_path()
 181                         print "Location of the package cache is " + self.settings["pkgcache_path"]
 182                         self.mounts.append("/usr/portage/packages")
 183                         self.mountmap["/usr/portage/packages"]=self.settings["pkgcache_path"]
 184
 185                 if self.settings.has_key("KERNCACHE"):
 186                         self.set_kerncache_path()
 187                         print "Location of the kerncache is " + self.settings["kerncache_path"]
 188                         self.mounts.append("/tmp/kerncache")
 189                         self.mountmap["/tmp/kerncache"]=self.settings["kerncache_path"]
 190
 191                 if self.settings.has_key("CCACHE"):
 192                         if os.environ.has_key("CCACHE_DIR"):
 193                                 ccdir=os.environ["CCACHE_DIR"]
 194                                 del os.environ["CCACHE_DIR"]
 195                         else:
 196                                 ccdir="/root/.ccache"
 197                         if not os.path.isdir(ccdir):
 198                                         raise CatalystError,\
 199                                                 "Compiler cache support can't be enabled (can't find "+ccdir+")"
 200                         self.mounts.append("/var/tmp/ccache")
 201                         self.mountmap["/var/tmp/ccache"]=ccdir
 202                         # for the chroot:
 203                         self.env["CCACHE_DIR"]="/var/tmp/ccache"
 204
 205         def override_cbuild(self):
 206                 if self.makeconf.has_key("CBUILD"):
 207                         self.settings["CBUILD"]=self.makeconf["CBUILD"]
 208
 209         def override_chost(self):
 210                 if self.makeconf.has_key("CHOST"):
 211                         self.settings["CHOST"]=self.makeconf["CHOST"]
 212
 213         def override_cflags(self):
 214                 if self.makeconf.has_key("CFLAGS"):
 215                         self.settings["CFLAGS"]=self.makeconf["CFLAGS"]
 216
 217         def override_cxxflags(self):
 218                 if self.makeconf.has_key("CXXFLAGS"):
 219                         self.settings["CXXFLAGS"]=self.makeconf["CXXFLAGS"]
 220
 221         def override_ldflags(self):
 222                 if self.makeconf.has_key("LDFLAGS"):
 223                         self.settings["LDFLAGS"]=self.makeconf["LDFLAGS"]
 224
 225         def set_install_mask(self):
 226                 if self.settings.has_key("install_mask"):
 227                         if type(self.settings["install_mask"]) != types.StringType:
 228                                 self.settings["install_mask"]=string.join(self.settings["install_mask"])
 229
 230         def set_spec_prefix(self):
 231                 self.settings["spec_prefix"]=self.settings["target"]
 232
 233         def set_target_profile(self):
 234                 self.settings["target_profile"]=self.settings["profile"]
 235
 236         def set_target_subpath(self):
 237                 self.settings["target_subpath"]=self.settings["rel_type"]+"/"+self.settings["target"]+\
 238                         "-"+self.settings["subarch"]+"-"+self.settings["version_stamp"]
 239
 240         def set_source_subpath(self):
 241                 if type(self.settings["source_subpath"]) != types.StringType:
 242                         raise CatalystError, "source_subpath should have been a string. Perhaps you have something wrong in your spec file?"
 243
 244         def set_pkgcache_path(self):
 245                 if self.settings.has_key("pkgcache_path"):
 246                         if type(self.settings["pkgcache_path"]) != types.StringType:
 247                                 self.settings["pkgcache_path"]=normpath(string.join(self.settings["pkgcache_path"]))
 248                 else:
 249                         self.settings["pkgcache_path"]=normpath(self.settings["storedir"]+"/packages/"+\
 250                                 self.settings["target_subpath"]+"/")
 251
 252         def set_kerncache_path(self):
 253                 if self.settings.has_key("kerncache_path"):
 254                         if type(self.settings["kerncache_path"]) != types.StringType:
 255                                 self.settings["kerncache_path"]=normpath(string.join(self.settings["kerncache_path"]))
 256                 else:
 257                         self.settings["kerncache_path"]=normpath(self.settings["storedir"]+"/kerncache/"+\
 258                                 self.settings["target_subpath"]+"/")
 259
 260         def set_target_path(self):
 261                 self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\
 262                         self.settings["target_subpath"]+".tar.bz2")
 263                 if self.settings.has_key("AUTORESUME") \
 264                         and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"):
 265                         print "Resume point detected, skipping target path setup operation..."
 266                 else:
 267                         # first clean up any existing target stuff
 268                         if os.path.isfile(self.settings["target_path"]):
 269                                 cmd("rm -f "+self.settings["target_path"], \
 270                                 "Could not remove existing file: "+self.settings["target_path"],env=self.env)
 271                                 touch(self.settings["autoresume_path"]+"setup_target_path")
 272
 273                         if not os.path.exists(self.settings["storedir"]+"/builds/"):
 274                                 os.makedirs(self.settings["storedir"]+"/builds/")
 275
 276         def set_archscript(self):
 277             if self.settings.has_key(self.settings["spec_prefix"]+"/archscript"):
 278                 print "\nWarning!!!  "
 279                 print "\t"+self.settings["spec_prefix"]+"/archscript" + " is deprecated and no longer used.\n"
 280         def set_runscript(self):
 281             if self.settings.has_key(self.settings["spec_prefix"]+"/runscript"):
 282                 print "\nWarning!!!  "
 283                 print "\t"+self.settings["spec_prefix"]+"/runscript" + " is deprecated and no longer used.\n"
 284
 285         def set_fsscript(self):
 286                 if self.settings.has_key(self.settings["spec_prefix"]+"/fsscript"):
 287                         self.settings["fsscript"]=self.settings[self.settings["spec_prefix"]+"/fsscript"]
 288                         del self.settings[self.settings["spec_prefix"]+"/fsscript"]
 289
 290         def set_rcadd(self):
 291                 if self.settings.has_key(self.settings["spec_prefix"]+"/rcadd"):
 292                         self.settings["rcadd"]=self.settings[self.settings["spec_prefix"]+"/rcadd"]
 293                         del self.settings[self.settings["spec_prefix"]+"/rcadd"]
 294
 295         def set_rcdel(self):
 296                 if self.settings.has_key(self.settings["spec_prefix"]+"/rcdel"):
 297                         self.settings["rcdel"]=self.settings[self.settings["spec_prefix"]+"/rcdel"]
 298                         del self.settings[self.settings["spec_prefix"]+"/rcdel"]
 299
 300         def set_cdtar(self):
 301                 if self.settings.has_key(self.settings["spec_prefix"]+"/cdtar"):
 302                         self.settings["cdtar"]=normpath(self.settings[self.settings["spec_prefix"]+"/cdtar"])
 303                         del self.settings[self.settings["spec_prefix"]+"/cdtar"]
 304
 305         def set_iso(self):
 306                 if self.settings.has_key(self.settings["spec_prefix"]+"/iso"):
 307                         self.settings["iso"]=normpath(self.settings[self.settings["spec_prefix"]+"/iso"])
 308                         del self.settings[self.settings["spec_prefix"]+"/iso"]
 309
 310         def set_fstype(self):
 311                 if self.settings.has_key(self.settings["spec_prefix"]+"/cdfstype"):
 312                         print "\nWarning!!!  "
 313                         print self.settings["spec_prefix"]+"/cdfstype" + " is deprecated and may be removed."
 314                         print "\tUse "+self.settings["spec_prefix"]+"/fstype" + " instead."
 315                         print "\tConverting to "+self.settings["spec_prefix"]+"/fstype" + " internally."
 316                         print "\tContinuing ....\n"
 317                         self.settings["fstype"]=self.settings[self.settings["spec_prefix"]+"/cdfstype"]
 318                         del self.settings[self.settings["spec_prefix"]+"/cdfstype"]
 319
 320                 if self.settings.has_key(self.settings["spec_prefix"]+"/fstype"):
 321                         self.settings["fstype"]=self.settings[self.settings["spec_prefix"]+"/fstype"]
 322                         del self.settings[self.settings["spec_prefix"]+"/fstype"]
 323
 324                 if not self.settings.has_key("fstype"):
 325                         self.settings["fstype"]="normal"
 326                         for x in self.valid_values:
 327                                 if x ==  self.settings["spec_prefix"]+"/fstype" or x == self.settings["spec_prefix"]+"/cdfstype":
 328                                         print "\n"+self.settings["spec_prefix"]+"/fstype is being set to the default of \"normal\"\n"
 329
 330         def set_fsops(self):
 331                 if self.settings.has_key("fstype"):
 332                         self.valid_values.append("fsops")
 333                         if self.settings.has_key(self.settings["spec_prefix"]+"/fsops"):
 334                                 self.settings["fsops"]=self.settings[self.settings["spec_prefix"]+"/fsops"]
 335                                 del self.settings[self.settings["spec_prefix"]+"/fsops"]
 336
 337         def set_source_path(self):
 338                 if self.settings.has_key("SEEDCACHE") and os.path.isdir(normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")):
 339                         self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")
 340                 else:
 341                         self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2")
 342                         if os.path.isfile(self.settings["source_path"]):
 343                                 if os.path.exists(self.settings["source_path"]): # XXX: Is this even necessary if the previous check passes?
 344                                         self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\
 345                                                 hash_function=self.settings["hash_function"],verbose=False)
 346                 print "Source path set to "+self.settings["source_path"]
 347                 if os.path.isdir(self.settings["source_path"]):
 348                         print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf"
 349                         print "\tthe source path will then be "+normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2\n")
 350
 351         def set_dest_path(self):
 352                 if self.settings.has_key("root_path"):
 353                         self.settings["destpath"]=normpath(self.settings["chroot_path"]+self.settings["root_path"])
 354                 else:
 355                         self.settings["destpath"]=normpath(self.settings["chroot_path"])
 356
 357         def set_cleanables(self):
 358                 self.settings["cleanables"]=["/etc/resolv.conf","/var/tmp/*","/tmp/*","/root/*",\
 359                                                 "/usr/portage"]
 360
 361         def set_snapshot_path(self):
 362                 self.settings["snapshot_path"]=normpath(self.settings["storedir"]+"/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
 363
 364                 if os.path.exists(self.settings["snapshot_path"]):
 365                         self.settings["snapshot_path_hash"]=generate_hash(self.settings["snapshot_path"],\
 366                                         hash_function=self.settings["hash_function"],verbose=False)
 367
 368         def set_snapcache_path(self):
 369                 if self.settings.has_key("SNAPCACHE"):
 370                         self.settings["snapshot_cache_path"]=normpath(self.settings["snapshot_cache"]+"/"+self.settings["snapshot"]+"/")
 371                         self.snapcache_lock=catalyst_lock.LockDir(self.settings["snapshot_cache_path"])
 372                         print "Caching snapshot to " + self.settings["snapshot_cache_path"]
 373
 374         def set_chroot_path(self):
 375                 # Note the trailing slash is very important and things would break without it
 376                 self.settings["chroot_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["target_subpath"]+"/")
 377                 self.chroot_lock=catalyst_lock.LockDir(self.settings["chroot_path"])
 378
 379         def set_autoresume_path(self):
 380                 self.settings["autoresume_path"]=normpath(self.settings["storedir"]+"/tmp/"+\
 381                         self.settings["rel_type"]+"/"+".autoresume-"+self.settings["target"]+\
 382                         "-"+self.settings["subarch"]+"-"+self.settings["version_stamp"]+"/")
 383                 if self.settings.has_key("AUTORESUME"):
 384                         print "The autoresume path is " + self.settings["autoresume_path"]
 385                 if not os.path.exists(self.settings["autoresume_path"]):
 386                         os.makedirs(self.settings["autoresume_path"],0755)
 387
 388         def set_controller_file(self):
 389                 self.settings["controller_file"]=normpath(self.settings["sharedir"]+"/targets/"+self.settings["target"]+"/"+self.settings["target"]+"-controller.sh")
 390
 391         def set_iso_volume_id(self):
 392                 if self.settings.has_key(self.settings["spec_prefix"]+"/volid"):
 393                         self.settings["iso_volume_id"] = self.settings[self.settings["spec_prefix"]+"/volid"]
 394                         if len(self.settings["iso_volume_id"])>32:
 395                                 raise CatalystError,"ISO VOLUME ID: volid must not exceed 32 characters."
 396                 else:
 397                         self.settings["iso_volume_id"] = "catalyst " + self.settings["snapshot"]
 398
 399         def set_action_sequence(self):
 400                 # Default action sequence for run method
 401                 self.settings["action_sequence"]=["unpack","unpack_snapshot",\
 402                                 "config_profile_link","setup_confdir","portage_overlay",\
 403                                 "base_dirs","bind","chroot_setup","setup_environment",\
 404                                 "run_local","preclean","unbind","clean"]
 405 #               if self.settings.has_key("TARBALL") or \
 406 #                       not self.settings.has_key("FETCH"):
 407                 if not self.settings.has_key("FETCH"):
 408                         self.settings["action_sequence"].append("capture")
 409                 self.settings["action_sequence"].append("clear_autoresume")
 410
 411         def set_use(self):
 412                 if self.settings.has_key(self.settings["spec_prefix"]+"/use"):
 413                         self.settings["use"]=self.settings[self.settings["spec_prefix"]+"/use"]
 414                         del self.settings[self.settings["spec_prefix"]+"/use"]
 415                 if self.settings.has_key("use"):
 416                     if type(self.settings["use"])==types.StringType:
 417                         self.settings["use"]=self.settings["use"].split()
 418
 419         def set_stage_path(self):
 420                         self.settings["stage_path"]=normpath(self.settings["chroot_path"])
 421
 422         def set_mounts(self):
 423                 pass
 424
 425         def set_packages(self):
 426                 pass
 427
 428         def set_rm(self):
 429             if self.settings.has_key(self.settings["spec_prefix"]+"/rm"):
 430                 if type(self.settings[self.settings["spec_prefix"]+"/rm"])==types.StringType:
 431                     self.settings[self.settings["spec_prefix"]+"/rm"]=self.settings[self.settings["spec_prefix"]+"/rm"].split()
 432
 433         def set_linuxrc(self):
 434             if self.settings.has_key(self.settings["spec_prefix"]+"/linuxrc"):
 435                         if type(self.settings[self.settings["spec_prefix"]+"/linuxrc"])==types.StringType:
 436                                 self.settings["linuxrc"]=self.settings[self.settings["spec_prefix"]+"/linuxrc"]
 437                                 del self.settings[self.settings["spec_prefix"]+"/linuxrc"]
 438
 439         def set_portage_overlay(self):
 440                 if self.settings.has_key("portage_overlay"):
 441                         if type(self.settings["portage_overlay"])==types.StringType:
 442                                 self.settings["portage_overlay"]=self.settings["portage_overlay"].split()
 443                         print "portage_overlay directories are set to: \"" + string.join(self.settings["portage_overlay"])+"\""
 444
 445         def set_overlay(self):
 446                 if self.settings.has_key(self.settings["spec_prefix"]+"/overlay"):
 447                         if type(self.settings[self.settings["spec_prefix"]+"/overlay"])==types.StringType:
 448                                 self.settings[self.settings["spec_prefix"]+"/overlay"]=self.settings[self.settings["spec_prefix"]+"/overlay"].split()
 449
 450         def set_root_overlay(self):
 451                 if self.settings.has_key(self.settings["spec_prefix"]+"/root_overlay"):
 452                         if type(self.settings[self.settings["spec_prefix"]+"/root_overlay"])==types.StringType:
 453                                 self.settings[self.settings["spec_prefix"]+"/root_overlay"]=self.settings[self.settings["spec_prefix"]+"/root_overlay"].split()
 454
 455
 456         def set_root_path(self):
 457                 # ROOT= variable for emerges
 458                 self.settings["root_path"]="/"
 459
 460         def set_valid_build_kernel_vars(self,addlargs):
 461                 if addlargs.has_key("boot/kernel"):
 462                         if type(addlargs["boot/kernel"]) == types.StringType:
 463                                 loopy=[addlargs["boot/kernel"]]
 464                         else:
 465                                 loopy=addlargs["boot/kernel"]
 466
 467                         for x in loopy:
 468                                 self.required_values.append("boot/kernel/"+x+"/sources")
 469                                 self.required_values.append("boot/kernel/"+x+"/config")
 470                                 self.valid_values.append("boot/kernel/"+x+"/aliases")
 471                                 self.valid_values.append("boot/kernel/"+x+"/extraversion")
 472                                 self.valid_values.append("boot/kernel/"+x+"/packages")
 473                                 if addlargs.has_key("boot/kernel/"+x+"/packages"):
 474                                         if type(addlargs["boot/kernel/"+x+"/packages"]) == types.StringType:
 475                                                 addlargs["boot/kernel/"+x+"/packages"]=[addlargs["boot/kernel/"+x+"/packages"]]
 476                                 self.valid_values.append("boot/kernel/"+x+"/use")
 477                                 self.valid_values.append("boot/kernel/"+x+"/gk_kernargs")
 478                                 self.valid_values.append("boot/kernel/"+x+"/gk_action")
 479                                 self.valid_values.append("boot/kernel/"+x+"/initramfs_overlay")
 480                                 self.valid_values.append("boot/kernel/"+x+"/softlevel")
 481                                 self.valid_values.append("boot/kernel/"+x+"/console")
 482                                 self.valid_values.append("boot/kernel/"+x+"/machine_type")
 483                                 self.valid_values.append("boot/kernel/"+x+"/postconf")
 484                                 if addlargs.has_key("boot/kernel/"+x+"/postconf"):
 485                                         print "boot/kernel/"+x+"/postconf is deprecated"
 486                                         print "\tInternally moving these ebuilds to boot/kernel/"+x+"/packages"
 487                                         print "\tPlease move them to boot/kernel/"+x+"/packages in your specfile"
 488                                         if type(addlargs["boot/kernel/"+x+"/postconf"]) == types.StringType:
 489                                                 loop2=[addlargs["boot/kernel/"+x+"/postconf"]]
 490                                         else:
 491                                                 loop2=addlargs["boot/kernel/"+x+"/postconf"]
 492
 493                                         for y in loop2:
 494                                                 if not addlargs.has_key("boot/kernel/"+x+"/packages"):
 495                                                         addlargs["boot/kernel/"+x+"/packages"]=[y]
 496                                                 else:
 497                                                         addlargs["boot/kernel/"+x+"/packages"].append(y)
 498
 499         def set_build_kernel_vars(self):
 500             if self.settings.has_key(self.settings["spec_prefix"]+"/devmanager"):
 501                 self.settings["devmanager"]=self.settings[self.settings["spec_prefix"]+"/devmanager"]
 502                 del self.settings[self.settings["spec_prefix"]+"/devmanager"]
 503
 504             if self.settings.has_key(self.settings["spec_prefix"]+"/splashtype"):
 505                 self.settings["splashtype"]=self.settings[self.settings["spec_prefix"]+"/splashtype"]
 506                 del self.settings[self.settings["spec_prefix"]+"/splashtype"]
 507
 508             if self.settings.has_key(self.settings["spec_prefix"]+"/gk_mainargs"):
 509                 self.settings["gk_mainargs"]=self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
 510                 del self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
 511
 512         def kill_chroot_pids(self):
 513             print "Checking for processes running in chroot and killing them."
 514
 515             # Force environment variables to be exported so script can see them
 516             self.setup_environment()
 517
 518             if os.path.exists(self.settings["sharedir"]+"/targets/support/kill-chroot-pids.sh"):
 519                             cmd("/bin/bash "+self.settings["sharedir"]+"/targets/support/kill-chroot-pids.sh",\
 520                                                 "kill-chroot-pids script failed.",env=self.env)
 521
 522         def mount_safety_check(self):
 523                 mypath=self.settings["chroot_path"]
 524
 525                 """
 526                 check and verify that none of our paths in mypath are mounted. We don't want to clean
 527                 up with things still mounted, and this allows us to check.
 528                 returns 1 on ok, 0 on "something is still mounted" case.
 529                 """
 530                 if not os.path.exists(mypath):
 531                         return
 532
 533                 for x in self.mounts:
 534                         if not os.path.exists(mypath+x):
 535                                 continue
 536
 537                         if ismount(mypath+x):
 538                                 #something is still mounted
 539                                 try:
 540                                         print x+" is still mounted; performing auto-bind-umount...",
 541                                         # try to umount stuff ourselves
 542                                         self.unbind()
 543                                         if ismount(mypath+x):
 544                                                 raise CatalystError, "Auto-unbind failed for "+x
 545
 546                                         else:
 547                                                 print "Auto-unbind successful..."
 548
 549                                 except CatalystError:
 550                                         raise CatalystError, "Unable to auto-unbind "+x
 551
 552         def unpack(self):
 553                 unpack=True
 554
 555                 clst_unpack_hash=read_from_clst(self.settings["autoresume_path"]+"unpack")
 556
 557                 if self.settings.has_key("SEEDCACHE"):
 558                         if os.path.isdir(self.settings["source_path"]):
 559                                 # SEEDCACHE Is a directory, use Rsync
 560                                 unpack_cmd="rsync -a --delete "+self.settings["source_path"]+" "+self.settings["chroot_path"]
 561                                 display_msg="\nStarting rsync from "+self.settings["source_path"]+"\nto "+\
 562                                         self.settings["chroot_path"]+" (This may take some time) ...\n"
 563                                 error_msg="Rsync of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 564                         else:
 565                                 # SEEDCACHE is a not a directory, try untar'ing
 566                                 print "Referenced SEEDCACHE does not appear to be a directory, trying to untar..."
 567                                 display_msg="\nStarting tar extract from "+self.settings["source_path"]+"\nto "+\
 568                                         self.settings["chroot_path"]+" (This may take some time) ...\n"
 569                                 unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"]
 570                                 error_msg="Tarball extraction of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 571                 else:
 572                         # No SEEDCACHE, use tar
 573                         display_msg="\nStarting tar extract from "+self.settings["source_path"]+"\nto "+\
 574                                 self.settings["chroot_path"]+" (This may take some time) ...\n"
 575                         unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"]
 576                         error_msg="Tarball extraction of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 577
 578
 579                 if self.settings.has_key("AUTORESUME"):
 580                         # Autoresume is Valid, SeedCache is Valid
 581                         if os.path.isdir(self.settings["source_path"]) and os.path.exists(self.settings["autoresume_path"]+"unpack"):
 582                                 unpack=False
 583                                 invalid_snapshot=False
 584
 585                         # Autoresume is Valid, Tarball is Valid
 586                         elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] == clst_unpack_hash:
 587                                 unpack=False
 588                                 invalid_snapshot=True
 589
 590                         # Autoresume is InValid, SeedCache
 591                         elif os.path.isdir(self.settings["source_path"]) and not os.path.exists(self.settings["autoresume_path"]+"unpack"):
 592                                 unpack=True
 593                                 invalid_snapshot=False
 594
 595                         # Autoresume is InValid, Tarball
 596                         elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] != clst_unpack_hash:
 597                                 unpack=True
 598                                 invalid_snapshot=True
 599                 else:
 600                         # No Autoresume,SeedCache
 601                         if self.settings.has_key("SEEDCACHE"):
 602
 603                                 # Seed cache so lets run rsync and rsync can clean up extra junk
 604                                 if os.path.isdir(self.settings["source_path"]):
 605                                         unpack=True
 606                                         invalid_snapshot=False
 607
 608                                 # Tarball so we better unpack and remove anything already there
 609                                 elif os.path.isfile(self.settings["source_path"]):
 610                                         unpack=True
 611                                         invalid_snapshot=True
 612
 613                         # No Autoresume,No SeedCache
 614                         else:
 615
 616                                 # Tarball so we better unpack and remove anything already there
 617                                 if os.path.isfile(self.settings["source_path"]):
 618                                         unpack=True
 619                                         invalid_snapshot=True
 620                                 # Should never reach this if so something is very wrong
 621                                 elif os.path.isdir(self.settings["source_path"]):
 622                                         raise CatalystError,"source path is a dir but seedcache is not enabled"
 623
 624                 if unpack:
 625                         self.mount_safety_check()
 626
 627                         if invalid_snapshot:
 628                                 if self.settings.has_key("AUTORESUME"):
 629                                         print "No Valid Resume point detected, cleaning up  ..."
 630
 631                                 self.clear_autoresume()
 632                                 self.clear_chroot()
 633
 634                         if not os.path.exists(self.settings["chroot_path"]):
 635                                 os.makedirs(self.settings["chroot_path"])
 636
 637                         if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
 638                                 os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
 639
 640                         if self.settings.has_key("PKGCACHE"):
 641                                 if not os.path.exists(self.settings["pkgcache_path"]):
 642                                         os.makedirs(self.settings["pkgcache_path"],0755)
 643
 644                         if self.settings.has_key("KERNCACHE"):
 645                                 if not os.path.exists(self.settings["kerncache_path"]):
 646                                         os.makedirs(self.settings["kerncache_path"],0755)
 647
 648                         print display_msg
 649                         cmd(unpack_cmd,error_msg,env=self.env)
 650
 651                         if self.settings.has_key("source_path_hash"):
 652                                 myf=open(self.settings["autoresume_path"]+"unpack","w")
 653                                 myf.write(self.settings["source_path_hash"])
 654                                 myf.close()
 655                         else:
 656                                 touch(self.settings["autoresume_path"]+"unpack")
 657                 else:
 658                     print "Resume point detected, skipping unpack operation..."
 659
 660
 661         def unpack_snapshot(self):
 662                 unpack=True
 663                 snapshot_hash=read_from_clst(self.settings["autoresume_path"]+"unpack_portage")
 664
 665                 if self.settings.has_key("SNAPCACHE"):
 666                         snapshot_cache_hash=read_from_clst(self.settings["snapshot_cache_path"]+"catalyst-hash")
 667                         destdir=self.settings["snapshot_cache_path"]
 668                         unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+destdir
 669                         unpack_errmsg="Error unpacking snapshot"
 670                         cleanup_msg="Cleaning up invalid snapshot cache at \n\t"+self.settings["snapshot_cache_path"]+" (This can take a long time)..."
 671                         cleanup_errmsg="Error removing existing snapshot cache directory."
 672                         self.snapshot_lock_object=self.snapcache_lock
 673
 674                         if self.settings["snapshot_path_hash"] == snapshot_cache_hash:
 675                                 print "Valid snapshot cache, skipping unpack of portage tree ..."
 676                                 unpack=False
 677
 678                 else:
 679                         destdir=normpath(self.settings["chroot_path"]+"/usr/portage")
 680                         cleanup_errmsg="Error removing existing snapshot directory."
 681                         cleanup_msg="Cleaning up existing portage tree (This can take a long time) ..."
 682                         unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+self.settings["chroot_path"]+"/usr"
 683                         unpack_errmsg="Error unpacking snapshot"
 684
 685                         if self.settings.has_key("AUTORESUME") \
 686                         and os.path.exists(self.settings["chroot_path"]+"/usr/portage/") \
 687                         and os.path.exists(self.settings["autoresume_path"]+"unpack_portage") \
 688                         and self.settings["snapshot_path_hash"] == snapshot_hash:
 689                                 print "Valid Resume point detected, skipping unpack of portage tree..."
 690                                 unpack=False
 691
 692
 693
 694                 if unpack:
 695                         if self.settings.has_key("SNAPCACHE"):
 696                             self.snapshot_lock_object.write_lock()
 697                         if os.path.exists(destdir):
 698                                 print cleanup_msg
 699                                 cleanup_cmd="rm -rf "+destdir
 700                                 cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
 701                         if not os.path.exists(destdir):
 702                                 os.makedirs(destdir,0755)
 703
 704                         print "Unpacking portage tree (This can take a long time) ..."
 705                         cmd(unpack_cmd,unpack_errmsg,env=self.env)
 706
 707                         if self.settings.has_key("SNAPCACHE"):
 708                                 myf=open(self.settings["snapshot_cache_path"]+"catalyst-hash","w")
 709                                 myf.write(self.settings["snapshot_path_hash"])
 710                                 myf.close()
 711
 712                         else:
 713                                 print "Setting snapshot autoresume point"
 714                                 myf=open(self.settings["autoresume_path"]+"unpack_portage","w")
 715                                 myf.write(self.settings["snapshot_path_hash"])
 716                                 myf.close()
 717
 718                         if self.settings.has_key("SNAPCACHE"):
 719                             self.snapshot_lock_object.unlock()
 720
 721         def config_profile_link(self):
 722                 if self.settings.has_key("AUTORESUME") \
 723                         and os.path.exists(self.settings["autoresume_path"]+"config_profile_link"):
 724                         print "Resume point detected, skipping config_profile_link operation..."
 725                 else:
 726                         print "Configuring profile link..."
 727                         cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.profile",\
 728                                         "Error zapping profile link",env=self.env)
 729                         cmd("ln -sf ../usr/portage/profiles/"+self.settings["target_profile"]+\
 730                                 " "+self.settings["chroot_path"]+"/etc/make.profile","Error creating profile link",env=self.env)
 731                         touch(self.settings["autoresume_path"]+"config_profile_link")
 732
 733         def setup_confdir(self):
 734                 if self.settings.has_key("AUTORESUME") \
 735                         and os.path.exists(self.settings["autoresume_path"]+"setup_confdir"):
 736                         print "Resume point detected, skipping setup_confdir operation..."
 737                 else:
 738                         if self.settings.has_key("portage_confdir"):
 739                                 print "Configuring /etc/portage..."
 740                                 cmd("rm -rf "+self.settings["chroot_path"]+"/etc/portage","Error zapping /etc/portage",env=self.env)
 741                                 cmd("cp -R "+self.settings["portage_confdir"]+"/ "+self.settings["chroot_path"]+\
 742                                         "/etc/portage","Error copying /etc/portage",env=self.env)
 743                                 touch(self.settings["autoresume_path"]+"setup_confdir")
 744
 745         def portage_overlay(self):
 746             # Here, we copy the contents of our overlays to /usr/local/portage. We
 747             # always copy over the overlays in case it has changed.
 748             if self.settings.has_key("portage_overlay"):
 749                 for x in self.settings["portage_overlay"]:
 750                         if os.path.exists(x):
 751                                 print "Copying overlay dir " +x
 752                                 cmd("mkdir -p "+self.settings["chroot_path"]+"/usr/local/portage","Could not make portage_overlay dir",env=self.env)
 753                                 cmd("cp -R "+x+"/* "+self.settings["chroot_path"]+"/usr/local/portage","Could not copy portage_overlay",env=self.env)
 754
 755         def root_overlay(self):
 756             # copy over the root_overlay
 757             # Always copy over the overlay incase it has changed
 758                 if self.settings.has_key(self.settings["spec_prefix"]+"/root_overlay"):
 759                         for x in self.settings[self.settings["spec_prefix"]+"/root_overlay"]:
 760                                 if os.path.exists(x):
 761                                         print "Copying root_overlay: "+x
 762                                         cmd("rsync -a "+x+"/ "+\
 763                                                 self.settings["chroot_path"], self.settings["spec_prefix"]+"/root_overlay: "+x+" copy failed.",env=self.env)
 764
 765         def base_dirs(self):
 766                 pass
 767
 768         def bind(self):
 769                 for x in self.mounts:
 770                         if not os.path.exists(self.settings["chroot_path"]+x):
 771                                 os.makedirs(self.settings["chroot_path"]+x,0755)
 772
 773                         if not os.path.exists(self.mountmap[x]):
 774                                 os.makedirs(self.mountmap[x],0755)
 775
 776                         src=self.mountmap[x]
 777                         if self.settings.has_key("SNAPCACHE") and x == "/usr/portage":
 778                                 self.snapshot_lock_object.read_lock()
 779                         if os.uname()[0] == "FreeBSD":
 780                                 if src == "/dev":
 781                                         retval=os.system("mount -t devfs none "+self.settings["chroot_path"]+x)
 782                                 else:
 783                                         retval=os.system("mount_nullfs "+src+" "+self.settings["chroot_path"]+x)
 784                         else:
 785                                 retval=os.system("mount --bind "+src+" "+self.settings["chroot_path"]+x)
 786                         if retval!=0:
 787                                 self.unbind()
 788                                 raise CatalystError,"Couldn't bind mount "+src
 789
 790
 791         def unbind(self):
 792                 ouch=0
 793                 mypath=self.settings["chroot_path"]
 794                 myrevmounts=self.mounts[:]
 795                 myrevmounts.reverse()
 796                 # unmount in reverse order for nested bind-mounts
 797                 for x in myrevmounts:
 798                         if not os.path.exists(mypath+x):
 799                                 continue
 800
 801                         if not ismount(mypath+x):
 802                                 # it's not mounted, continue
 803                                 continue
 804
 805                         retval=os.system("umount "+os.path.join(mypath,x.lstrip(os.path.sep)))
 806
 807                         if retval!=0:
 808                                 warn("First attempt to unmount: "+mypath+x+" failed.")
 809                                 warn("Killing any pids still running in the chroot")
 810
 811                                 self.kill_chroot_pids()
 812
 813                                 retval2=os.system("umount "+mypath+x)
 814                                 if retval2!=0:
 815                                     ouch=1
 816                                     warn("Couldn't umount bind mount: "+mypath+x)
 817                                     # keep trying to umount the others, to minimize damage if developer makes a mistake
 818
 819                         if self.settings.has_key("SNAPCACHE") and x == "/usr/portage":
 820                                 try:
 821                                     # Its possible the snapshot lock object isnt created yet
 822                                     # this is because mount safety check calls unbind before the target is fully initialized
 823                                     self.snapshot_lock_object.unlock()
 824                                 except:
 825                                     pass
 826                 if ouch:
 827                         """
 828                         if any bind mounts really failed, then we need to raise
 829                         this to potentially prevent an upcoming bash stage cleanup script
 830                         from wiping our bind mounts.
 831                         """
 832                         raise CatalystError,"Couldn't umount one or more bind-mounts; aborting for safety."
 833
 834         def chroot_setup(self):
 835                 self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/make.conf")
 836                 self.override_cbuild()
 837                 self.override_chost()
 838                 self.override_cflags()
 839                 self.override_cxxflags()
 840                 self.override_ldflags()
 841                 if self.settings.has_key("AUTORESUME") \
 842                         and os.path.exists(self.settings["autoresume_path"]+"chroot_setup"):
 843                         print "Resume point detected, skipping chroot_setup operation..."
 844                 else:
 845                         print "Setting up chroot..."
 846
 847                         #self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/make.conf")
 848
 849                         cmd("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc",\
 850                                 "Could not copy resolv.conf into place.",env=self.env)
 851
 852                     # copy over the envscript, if applicable
 853                         if self.settings.has_key("ENVSCRIPT"):
 854                                 if not os.path.exists(self.settings["ENVSCRIPT"]):
 855                                         raise CatalystError, "Can't find envscript "+self.settings["ENVSCRIPT"]
 856
 857                                 print "\nWarning!!!!"
 858                                 print "\tOverriding certain env variables may cause catastrophic failure."
 859                                 print "\tIf your build fails look here first as the possible problem."
 860                                 print "\tCatalyst assumes you know what you are doing when setting"
 861                                 print "\t\tthese variables."
 862                                 print "\tCatalyst Maintainers use VERY minimal envscripts if used at all"
 863                                 print "\tYou have been warned\n"
 864
 865                                 cmd("cp "+self.settings["ENVSCRIPT"]+" "+self.settings["chroot_path"]+"/tmp/envscript",\
 866                                         "Could not copy envscript into place.",env=self.env)
 867
 868                     # Copy over /etc/hosts from the host in case there are any
 869                         # specialties in there
 870                         if os.path.exists(self.settings["chroot_path"]+"/etc/hosts"):
 871                                 cmd("mv "+self.settings["chroot_path"]+"/etc/hosts "+self.settings["chroot_path"]+\
 872                                         "/etc/hosts.bck", "Could not backup /etc/hosts",env=self.env)
 873                                 cmd("cp /etc/hosts "+self.settings["chroot_path"]+"/etc/hosts", "Could not copy /etc/hosts",env=self.env)
 874                         #self.override_cbuild()
 875                         #self.override_chost()
 876                         #self.override_cflags()
 877                         #self.override_cxxflags()
 878                         #self.override_ldflags()
 879                         # Modify and write out make.conf (for the chroot)
 880                         cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.conf","Could not remove "+self.settings["chroot_path"]+"/etc/make.conf",\
 881                                 env=self.env)
 882                         myf=open(self.settings["chroot_path"]+"/etc/make.conf","w")
 883                         myf.write("# These settings were set by the catalyst build script that automatically\n# built this stage.\n")
 884                         myf.write("# Please consult /etc/make.conf.example for a more detailed example.\n")
 885                         if self.settings.has_key("CFLAGS"):
 886                                 myf.write('CFLAGS="'+self.settings["CFLAGS"]+'"\n')
 887                         if self.settings.has_key("CXXFLAGS"):
 888                                 myf.write('CXXFLAGS="'+self.settings["CXXFLAGS"]+'"\n')
 889                         else:
 890                                 myf.write('CXXFLAGS="${CFLAGS}"\n')
 891
 892                         if self.settings.has_key("LDFLAGS"):
 893                                 myf.write('LDFLAGS="'+self.settings["LDFLAGS"]+'"\n')
 894                         myf.write("# This should not be changed unless you know exactly what you are doing.  You\n# should probably be using a different stage, instead.\n")
 895                         if self.settings.has_key("CBUILD"):
 896                                 myf.write('CBUILD="'+self.settings["CBUILD"]+'"\n')
 897                         myf.write('CHOST="'+self.settings["CHOST"]+'"\n')
 898
 899                     # Figure out what our USE vars are for building
 900                         myusevars=[]
 901                         if self.settings.has_key("HOSTUSE"):
 902                                 myusevars.extend(self.settings["HOSTUSE"])
 903
 904                         if self.settings.has_key("use"):
 905                                 myusevars.extend(self.settings["use"])
 906                                 myf.write('USE="'+string.join(myusevars)+'"\n')
 907
 908                     # Setup the portage overlay
 909                         if self.settings.has_key("portage_overlay"):
 910 #                               myf.write('PORTDIR_OVERLAY="'+string.join(self.settings["portage_overlay"])+'"\n')
 911                                 myf.write('PORTDIR_OVERLAY="/usr/local/portage"\n')
 912
 913                         myf.close()
 914                         touch(self.settings["autoresume_path"]+"chroot_setup")
 915
 916         def fsscript(self):
 917                 if self.settings.has_key("AUTORESUME") \
 918                         and os.path.exists(self.settings["autoresume_path"]+"fsscript"):
 919                         print "Resume point detected, skipping fsscript operation..."
 920                 else:
 921                     if self.settings.has_key("fsscript"):
 922                         if os.path.exists(self.settings["controller_file"]):
 923                             cmd("/bin/bash "+self.settings["controller_file"]+" fsscript","fsscript script failed.",env=self.env)
 924                         touch(self.settings["autoresume_path"]+"fsscript")
 925
 926         def rcupdate(self):
 927                 if self.settings.has_key("AUTORESUME") \
 928                         and os.path.exists(self.settings["autoresume_path"]+"rcupdate"):
 929                         print "Resume point detected, skipping rcupdate operation..."
 930                 else:
 931                     if os.path.exists(self.settings["controller_file"]):
 932                         cmd("/bin/bash "+self.settings["controller_file"]+" rc-update","rc-update script failed.",env=self.env)
 933                         touch(self.settings["autoresume_path"]+"rcupdate")
 934
 935         def clean(self):
 936                 if self.settings.has_key("AUTORESUME") \
 937                         and os.path.exists(self.settings["autoresume_path"]+"clean"):
 938                         print "Resume point detected, skipping clean operation..."
 939                 else:
 940                     for x in self.settings["cleanables"]:
 941                             print "Cleaning chroot: "+x+"... "
 942                             cmd("rm -rf "+self.settings["destpath"]+x,"Couldn't clean "+x,env=self.env)
 943
 944                     # put /etc/hosts back into place
 945                     if os.path.exists(self.settings["chroot_path"]+"/etc/hosts.bck"):
 946                                 cmd("mv -f "+self.settings["chroot_path"]+"/etc/hosts.bck "+self.settings["chroot_path"]+"/etc/hosts", "Could not replace /etc/hosts",env=self.env)
 947
 948                     # remove our overlay
 949                     if os.path.exists(self.settings["chroot_path"]+"/usr/local/portage"):
 950                                 cmd("rm -rf "+self.settings["chroot_path"]+"/usr/local/portage", "Could not remove /usr/local/portage",env=self.env)
 951                                 cmd("sed -i '/^PORTDIR_OVERLAY/d' "+self.settings["chroot_path"]+"/etc/make.conf", "Could not remove PORTDIR_OVERLAY from make.conf",env=self.env)
 952
 953                     # clean up old and obsoleted files in /etc
 954                     if os.path.exists(self.settings["stage_path"]+"/etc"):
 955                                 cmd("find "+self.settings["stage_path"]+"/etc -maxdepth 1 -name \"*-\" | xargs rm -f", "Could not remove stray files in /etc",env=self.env)
 956
 957                     if os.path.exists(self.settings["controller_file"]):
 958                         cmd("/bin/bash "+self.settings["controller_file"]+" clean","clean script failed.",env=self.env)
 959                         touch(self.settings["autoresume_path"]+"clean")
 960
 961         def empty(self):
 962             if self.settings.has_key("AUTORESUME") \
 963                 and os.path.exists(self.settings["autoresume_path"]+"empty"):
 964                 print "Resume point detected, skipping empty operation..."
 965             else:
 966                 if self.settings.has_key(self.settings["spec_prefix"]+"/empty"):
 967                     if type(self.settings[self.settings["spec_prefix"]+"/empty"])==types.StringType:
 968                         self.settings[self.settings["spec_prefix"]+"/empty"]=self.settings[self.settings["spec_prefix"]+"/empty"].split()
 969                     for x in self.settings[self.settings["spec_prefix"]+"/empty"]:
 970                         myemp=self.settings["destpath"]+x
 971                         if not os.path.isdir(myemp):
 972                             print x,"not a directory or does not exist, skipping 'empty' operation."
 973                             continue
 974                         print "Emptying directory",x
 975                         # stat the dir, delete the dir, recreate the dir and set
 976                         # the proper perms and ownership
 977                         mystat=os.stat(myemp)
 978                         shutil.rmtree(myemp)
 979                         os.makedirs(myemp,0755)
 980                         os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 981                         os.chmod(myemp,mystat[ST_MODE])
 982                     touch(self.settings["autoresume_path"]+"empty")
 983
 984         def remove(self):
 985             if self.settings.has_key("AUTORESUME") \
 986                 and os.path.exists(self.settings["autoresume_path"]+"remove"):
 987                 print "Resume point detected, skipping remove operation..."
 988             else:
 989                 if self.settings.has_key(self.settings["spec_prefix"]+"/rm"):
 990                     for x in self.settings[self.settings["spec_prefix"]+"/rm"]:
 991                         # we're going to shell out for all these cleaning operations,
 992                         # so we get easy glob handling
 993                         print "livecd: removing "+x
 994                         os.system("rm -rf "+self.settings["chroot_path"]+x)
 995                 try:
 996                     if os.path.exists(self.settings["controller_file"]):
 997                             cmd("/bin/bash "+self.settings["controller_file"]+" clean",\
 998                                 "Clean  failed.",env=self.env)
 999                             touch(self.settings["autoresume_path"]+"remove")
1000                 except:
1001                     self.unbind()
1002                     raise
1003
1004
1005         def preclean(self):
1006             if self.settings.has_key("AUTORESUME") \
1007                 and os.path.exists(self.settings["autoresume_path"]+"preclean"):
1008                 print "Resume point detected, skipping preclean operation..."
1009             else:
1010                 try:
1011                         if os.path.exists(self.settings["controller_file"]):
1012                                 cmd("/bin/bash "+self.settings["controller_file"]+" preclean","preclean script failed.",env=self.env)
1013                                 touch(self.settings["autoresume_path"]+"preclean")
1014
1015                 except:
1016                         self.unbind()
1017                         raise CatalystError, "Build failed, could not execute preclean"
1018
1019         def capture(self):
1020             if self.settings.has_key("AUTORESUME") \
1021                 and os.path.exists(self.settings["autoresume_path"]+"capture"):
1022                 print "Resume point detected, skipping capture operation..."
1023             else:
1024                 """capture target in a tarball"""
1025                 mypath=self.settings["target_path"].split("/")
1026                 # remove filename from path
1027                 mypath=string.join(mypath[:-1],"/")
1028
1029                 # now make sure path exists
1030                 if not os.path.exists(mypath):
1031                         os.makedirs(mypath)
1032
1033                 print "Creating stage tarball..."
1034
1035                 cmd("tar cjpf "+self.settings["target_path"]+" -C "+self.settings["stage_path"]+\
1036                         " .","Couldn't create stage tarball",env=self.env)
1037
1038                 self.gen_digest_file(self.settings["target_path"])
1039
1040                 touch(self.settings["autoresume_path"]+"capture")
1041
1042         def run_local(self):
1043             if self.settings.has_key("AUTORESUME") \
1044                 and os.path.exists(self.settings["autoresume_path"]+"run_local"):
1045                 print "Resume point detected, skipping run_local operation..."
1046             else:
1047                 try:
1048                         if os.path.exists(self.settings["controller_file"]):
1049                                 cmd("/bin/bash "+self.settings["controller_file"]+" run","run script failed.",env=self.env)
1050                                 touch(self.settings["autoresume_path"]+"run_local")
1051
1052                 except CatalystError:
1053                         self.unbind()
1054                         raise CatalystError,"Stage build aborting due to error."
1055
1056         def setup_environment(self):
1057                 # Modify the current environment. This is an ugly hack that should be
1058                 # fixed. We need this to use the os.system() call since we can't
1059                 # specify our own environ:
1060                 for x in self.settings.keys():
1061                         # sanitize var names by doing "s|/-.|_|g"
1062                         varname="clst_"+string.replace(x,"/","_")
1063                         varname=string.replace(varname,"-","_")
1064                         varname=string.replace(varname,".","_")
1065                         if type(self.settings[x])==types.StringType:
1066                                 # prefix to prevent namespace clashes:
1067                                 #os.environ[varname]=self.settings[x]
1068                                 self.env[varname]=self.settings[x]
1069                         elif type(self.settings[x])==types.ListType:
1070                                 #os.environ[varname]=string.join(self.settings[x])
1071                                 self.env[varname]=string.join(self.settings[x])
1072                         elif type(self.settings[x])==types.BooleanType:
1073                                 if self.settings[x]:
1074                                         self.env[varname]="true"
1075                                 else:
1076                                         self.env[varname]="false"
1077                 if self.settings.has_key("makeopts"):
1078                         self.env["MAKEOPTS"]=self.settings["makeopts"]
1079
1080         def run(self):
1081                 self.chroot_lock.write_lock()
1082
1083                 # Kill any pids in the chroot
1084                 self.kill_chroot_pids()
1085
1086                 # Check for mounts right away and abort if we cannot unmount them.
1087                 self.mount_safety_check()
1088
1089                 if self.settings.has_key("CLEAR_AUTORESUME"):
1090                         self.clear_autoresume()
1091                 if self.settings.has_key("PURGE"):
1092                         self.purge()
1093
1094                 for x in self.settings["action_sequence"]:
1095                         print "--- Running action sequence: "+x
1096                         sys.stdout.flush()
1097                         try:
1098                                 apply(getattr(self,x))
1099                         except:
1100                                 self.mount_safety_check()
1101                                 raise
1102
1103                 self.chroot_lock.unlock()
1104
1105         def unmerge(self):
1106             if self.settings.has_key("AUTORESUME") \
1107                 and os.path.exists(self.settings["autoresume_path"]+"unmerge"):
1108                     print "Resume point detected, skipping unmerge operation..."
1109             else:
1110                 if self.settings.has_key(self.settings["spec_prefix"]+"/unmerge"):
1111                     if type(self.settings[self.settings["spec_prefix"]+"/unmerge"])==types.StringType:
1112                         self.settings[self.settings["spec_prefix"]+"/unmerge"]=[self.settings[self.settings["spec_prefix"]+"/unmerge"]]
1113                     myunmerge=self.settings[self.settings["spec_prefix"]+"/unmerge"][:]
1114
1115                     for x in range(0,len(myunmerge)):
1116                     #surround args with quotes for passing to bash,
1117                     #allows things like "<" to remain intact
1118                         myunmerge[x]="'"+myunmerge[x]+"'"
1119                     myunmerge=string.join(myunmerge)
1120
1121                     #before cleaning, unmerge stuff:
1122                     try:
1123                         cmd("/bin/bash "+self.settings["controller_file"]+" unmerge "+ myunmerge,\
1124                                 "Unmerge script failed.",env=self.env)
1125                         #cmd("/bin/bash "+self.settings["sharedir"]+"/targets/" \
1126                         #       +self.settings["target"]+"/unmerge.sh "+myunmerge,"Unmerge script failed.",env=self.env)
1127                         print "unmerge shell script"
1128                     except CatalystError:
1129                         self.unbind()
1130                         raise
1131
1132                 touch(self.settings["autoresume_path"]+"unmerge")
1133
1134         def target_setup(self):
1135             if self.settings.has_key("AUTORESUME") \
1136                 and os.path.exists(self.settings["autoresume_path"]+"target_setup"):
1137                     print "Resume point detected, skipping target_setup operation..."
1138             else:
1139                 print "Setting up filesystems per filesystem type"
1140                 cmd("/bin/bash "+self.settings["controller_file"]+" target_image_setup "+ self.settings["target_path"],\
1141                                 "target_image_setup script failed.",env=self.env)
1142                 touch(self.settings["autoresume_path"]+"target_setup")
1143
1144         def setup_overlay(self):
1145                 if self.settings.has_key("AUTORESUME") \
1146                 and os.path.exists(self.settings["autoresume_path"]+"setup_overlay"):
1147                         print "Resume point detected, skipping setup_overlay operation..."
1148                 else:
1149                         if self.settings.has_key(self.settings["spec_prefix"]+"/overlay"):
1150                                 for x in self.settings[self.settings["spec_prefix"]+"/overlay"]:
1151                                         if os.path.exists(x):
1152                                                 cmd("rsync -a "+x+"/ "+\
1153                                                         self.settings["target_path"], self.settings["spec_prefix"]+"overlay: "+x+" copy failed.",env=self.env)
1154                                 touch(self.settings["autoresume_path"]+"setup_overlay")
1155
1156         def create_iso(self):
1157             if self.settings.has_key("AUTORESUME") \
1158                 and os.path.exists(self.settings["autoresume_path"]+"create_iso"):
1159                     print "Resume point detected, skipping create_iso operation..."
1160             else:
1161                 # create the ISO - this is the preferred method (the iso scripts do not always work)
1162                 if self.settings.has_key("iso"):
1163                         cmd("/bin/bash "+self.settings["controller_file"]+" iso "+\
1164                                 self.settings["iso"],"ISO creation script failed.",env=self.env)
1165                         self.gen_digest_file(self.settings["iso"])
1166                         touch(self.settings["autoresume_path"]+"create_iso")
1167
1168
1169                 else:
1170                         print "WARNING livecd/iso was not defined."
1171                         print "A CD Image will not be created, skipping create-iso.sh..."
1172
1173
1174         def build_packages(self):
1175             if self.settings.has_key("AUTORESUME") \
1176                 and os.path.exists(self.settings["autoresume_path"]+"build_packages"):
1177                     print "Resume point detected, skipping build_packages operation..."
1178             else:
1179                 if self.settings.has_key(self.settings["spec_prefix"]+"/packages"):
1180                         if self.settings.has_key("AUTORESUME") \
1181                                 and os.path.exists(self.settings["autoresume_path"]+"build_packages"):
1182                                         print "Resume point detected, skipping build_packages operation..."
1183                         else:
1184                                 mypack=list_bashify(self.settings[self.settings["spec_prefix"]+"/packages"])
1185                                 try:
1186                                         cmd("/bin/bash "+self.settings["controller_file"]+\
1187                                                 " build_packages "+mypack,"Error in attempt to build packages",env=self.env)
1188                                         touch(self.settings["autoresume_path"]+"build_packages")
1189                                 except CatalystError:
1190                                         self.unbind()
1191                                         raise CatalystError,self.settings["spec_prefix"] + "build aborting due to error."
1192
1193         def build_kernel(self):
1194                 if self.settings.has_key("AUTORESUME") \
1195                 and os.path.exists(self.settings["autoresume_path"]+"build_kernel"):
1196                         print "Resume point detected, skipping build_kernel operation..."
1197                 else:
1198                         if self.settings.has_key("boot/kernel"):
1199                                 try:
1200                                         mynames=self.settings["boot/kernel"]
1201                                         if type(mynames)==types.StringType:
1202                                                 mynames=[mynames]
1203                                         # execute the script that sets up the kernel build environment
1204                                         cmd("/bin/bash "+self.settings["controller_file"]+" pre-kmerge ",\
1205                                                 "Runscript pre-kmerge failed",env=self.env)
1206
1207                                         for kname in mynames:
1208                                                 if self.settings.has_key("AUTORESUME") \
1209                                                         and os.path.exists(self.settings["autoresume_path"]+"build_kernel_"+kname):
1210                                                         print "Resume point detected, skipping build_kernel for "+kname+" operation..."
1211                                                 else:
1212                                                         try:
1213                                                                 if not os.path.exists(self.settings["boot/kernel/"+kname+"/config"]):
1214                                                                         self.unbind()
1215                                                                         raise CatalystError, "Can't find kernel config: " \
1216                                                                                 +self.settings["boot/kernel/"+kname+"/config"]
1217
1218                                                         except TypeError:
1219                                                                 raise CatalystError, "Required value boot/kernel/config not specified"
1220
1221                                                         try:
1222                                                                 cmd("cp "+self.settings["boot/kernel/"+kname+"/config"]+" "+ \
1223                                                                         self.settings["chroot_path"]+"/var/tmp/"+kname+".config", \
1224                                                                         "Couldn't copy kernel config: "+self.settings["boot/kernel/"+kname+"/config"],\
1225                                                                         env=self.env)
1226
1227                                                         except CatalystError:
1228                                                                 self.unbind()
1229
1230                                                         # If we need to pass special options to the bootloader
1231                                                         # for this kernel put them into the environment.
1232                                                         if self.settings.has_key("boot/kernel/"+kname+"/kernelopts"):
1233                                                                 myopts=self.settings["boot/kernel/"+kname+"/kernelopts"]
1234
1235                                                                 if type(myopts) != types.StringType:
1236                                                                         myopts = string.join(myopts)
1237                                                                         self.env[kname+"_kernelopts"]=myopts
1238
1239                                                                 else:
1240                                                                         self.env[kname+"_kernelopts"]=""
1241
1242                                                         if not self.settings.has_key("boot/kernel/"+kname+"/extraversion"):
1243                                                                 self.settings["boot/kernel/"+kname+"/extraversion"]=""
1244
1245                                                         self.env["clst_kextraversion"]=self.settings["boot/kernel/"+kname+"/extraversion"]
1246
1247                                                         if self.settings.has_key("boot/kernel/"+kname+"/initramfs_overlay"):
1248                                                                 if os.path.exists(self.settings["boot/kernel/"+kname+"/initramfs_overlay"]):
1249                                                                         print "Copying initramfs_overlay dir " +self.settings["boot/kernel/"+kname+"/initramfs_overlay"]
1250
1251                                                                         cmd("mkdir -p "+self.settings["chroot_path"]+"/tmp/initramfs_overlay/" + \
1252                                                                                 self.settings["boot/kernel/"+kname+"/initramfs_overlay"],env=self.env)
1253
1254                                                                         cmd("cp -R "+self.settings["boot/kernel/"+kname+"/initramfs_overlay"]+"/* " + \
1255                                                                                 self.settings["chroot_path"] + "/tmp/initramfs_overlay/" + \
1256                                                                                 self.settings["boot/kernel/"+kname+"/initramfs_overlay"],\
1257                                                                                 env=self.env)
1258
1259
1260                                                         # execute the script that builds the kernel
1261                                                         cmd("/bin/bash "+self.settings["controller_file"]+" kernel "+kname,\
1262                                                                 "Runscript kernel build failed",env=self.env)
1263
1264                                                         if self.settings.has_key("boot/kernel/"+kname+"/initramfs_overlay"):
1265                                                                 if os.path.exists(self.settings["chroot_path"]+"/tmp/initramfs_overlay/"):
1266                                                                         print "Cleaning up temporary overlay dir"
1267                                                                         cmd("rm -R "+self.settings["chroot_path"]+"/tmp/initramfs_overlay/",env=self.env)
1268
1269                                                         touch(self.settings["autoresume_path"]+"build_kernel_"+kname)
1270
1271                                                         # execute the script that cleans up the kernel build environment
1272                                                         cmd("/bin/bash "+self.settings["controller_file"]+" post-kmerge ",\
1273                                                                 "Runscript post-kmerge failed",env=self.env)
1274
1275                                         touch(self.settings["autoresume_path"]+"build_kernel")
1276
1277                                 except CatalystError:
1278                                         self.unbind()
1279                                         raise CatalystError,"build aborting due to kernel build error."
1280
1281         def bootloader(self):
1282             if self.settings.has_key("AUTORESUME") \
1283                 and os.path.exists(self.settings["autoresume_path"]+"bootloader"):
1284                     print "Resume point detected, skipping bootloader operation..."
1285             else:
1286                 try:
1287                         cmd("/bin/bash "+self.settings["controller_file"]+" bootloader " + self.settings["target_path"],\
1288                                 "Bootloader runscript failed.",env=self.env)
1289                         touch(self.settings["autoresume_path"]+"bootloader")
1290                 except CatalystError:
1291                         self.unbind()
1292                         raise CatalystError,"Runscript aborting due to error."
1293
1294         def livecd_update(self):
1295             if self.settings.has_key("AUTORESUME") \
1296                 and os.path.exists(self.settings["autoresume_path"]+"livecd_update"):
1297                     print "Resume point detected, skipping build_packages operation..."
1298             else:
1299                 try:
1300                         cmd("/bin/bash "+self.settings["controller_file"]+" livecd-update",\
1301                                 "livecd-update failed.",env=self.env)
1302                         touch(self.settings["autoresume_path"]+"livecd_update")
1303
1304                 except CatalystError:
1305                         self.unbind()
1306                         raise CatalystError,"build aborting due to livecd_update error."
1307
1308         def clear_chroot(self):
1309                 myemp=self.settings["chroot_path"]
1310                 if os.path.isdir(myemp):
1311                     print "Emptying directory",myemp
1312                     # stat the dir, delete the dir, recreate the dir and set
1313                     # the proper perms and ownership
1314                     mystat=os.stat(myemp)
1315                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1316                     if os.uname()[0] == "FreeBSD": # There's no easy way to change flags recursively in python
1317                             os.system("chflags -R noschg "+myemp)
1318                     shutil.rmtree(myemp)
1319                     os.makedirs(myemp,0755)
1320                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1321                     os.chmod(myemp,mystat[ST_MODE])
1322
1323         def clear_packages(self):
1324             if self.settings.has_key("PKGCACHE"):
1325                 print "purging the pkgcache ..."
1326
1327                 myemp=self.settings["pkgcache_path"]
1328                 if os.path.isdir(myemp):
1329                     print "Emptying directory",myemp
1330                     # stat the dir, delete the dir, recreate the dir and set
1331                     # the proper perms and ownership
1332                     mystat=os.stat(myemp)
1333                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1334                     shutil.rmtree(myemp)
1335                     os.makedirs(myemp,0755)
1336                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1337                     os.chmod(myemp,mystat[ST_MODE])
1338
1339         def clear_kerncache(self):
1340             if self.settings.has_key("KERNCACHE"):
1341                 print "purging the kerncache ..."
1342
1343                 myemp=self.settings["kerncache_path"]
1344                 if os.path.isdir(myemp):
1345                     print "Emptying directory",myemp
1346                     # stat the dir, delete the dir, recreate the dir and set
1347                     # the proper perms and ownership
1348                     mystat=os.stat(myemp)
1349                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1350                     shutil.rmtree(myemp)
1351                     os.makedirs(myemp,0755)
1352                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1353                     os.chmod(myemp,mystat[ST_MODE])
1354
1355         def clear_autoresume(self):
1356                 # clean resume points since they are no longer needed
1357                 if self.settings.has_key("AUTORESUME"):
1358                         print "Removing AutoResume Points: ..."
1359                 myemp=self.settings["autoresume_path"]
1360                 if os.path.isdir(myemp):
1361                                 if self.settings.has_key("AUTORESUME"):
1362                                         print "Emptying directory",myemp
1363                                 # stat the dir, delete the dir, recreate the dir and set
1364                                 # the proper perms and ownership
1365                                 mystat=os.stat(myemp)
1366                                 #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
1367                                 shutil.rmtree(myemp)
1368                                 os.makedirs(myemp,0755)
1369                                 os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1370                                 os.chmod(myemp,mystat[ST_MODE])
1371
1372         def gen_digest_file(self,file):
1373                 if os.path.exists(file+".DIGESTS"):
1374                         os.remove(file+".DIGESTS")
1375                 if self.settings.has_key("digests"):
1376                         if os.path.exists(file):
1377                                 myf=open(file+".DIGESTS","w")
1378                                 keys={}
1379                                 for i in self.settings["digests"].split():
1380                                         keys[i]=1
1381                                         array=keys.keys()
1382                                         array.sort()
1383                                 for j in array:
1384                                         if self.settings.has_key("VERBOSE"):
1385                                                 hash=generate_hash(file,hash_function=j,verbose=True)
1386                                         else:
1387                                                 hash=generate_hash(file,hash_function=j)
1388                                         myf.write(hash)
1389                                 myf.close()
1390
1391         def purge(self):
1392             countdown(10,"Purging Caches ...")
1393             if self.settings.has_key("PURGE"):
1394                 print "clearing autoresume ..."
1395                 self.clear_autoresume()
1396
1397                 print "clearing chroot ..."
1398                 self.clear_chroot()
1399
1400                 print "clearing package cache ..."
1401                 self.clear_packages()
1402
1403                 print "clearing kerncache ..."
1404                 self.clear_kerncache()
1405
1406 #vim: ts=4 sw=4 sta et sts=4 ai