modules/generic_stage_target.py

   1 # Copyright 1999-2005 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3 # $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.133 2006/04/25 17:13:28 rocket Exp $
   4
   5 """
   6 This class does all of the chroot setup, copying of files, etc. It is
   7 the driver class for pretty much everything that Catalyst does.
   8 """
   9
  10 import os,string,imp,types,shutil
  11 from catalyst_support import *
  12 from generic_target import *
  13 from stat import *
  14 import catalyst_lock
  15 class generic_stage_target(generic_target):
  16
  17         def __init__(self,myspec,addlargs):
  18                 self.required_values.extend(["version_stamp","target","subarch","rel_type",\
  19                         "profile","snapshot","source_subpath"])
  20
  21                 self.valid_values.extend(["version_stamp","target","subarch","rel_type","profile",\
  22                         "snapshot","source_subpath","portage_confdir","cflags","cxxflags",\
  23                         "ldflags","chost","hostuse","portage_overlay","distcc_hosts","makeopts",\
  24                         "pkgcache_path","kerncache_path"])
  25
  26                 self.set_valid_build_kernel_vars(addlargs)
  27                 generic_target.__init__(self,myspec,addlargs)
  28                 # map the mainarch we are running under to the mainarches we support for
  29                 # building stages and LiveCDs. (for example, on amd64, we can build stages for
  30                 # x86 or amd64.
  31                 targetmap={
  32                                 "x86" : ["x86"],
  33                                 "amd64" : ["x86","amd64"],
  34                                 "sparc64" : ["sparc","sparc64"],
  35                                 "ia64" : ["ia64"],
  36                                 "alpha" : ["alpha"],
  37                                 "sparc" : ["sparc"],
  38                                 "sh" : ["sh"],
  39                                 "s390" : ["s390"],
  40                                 "ppc" : ["ppc"],
  41                                 "ppc64" : ["ppc","ppc64"],
  42                                 "hppa" : ["hppa"],
  43                                 "mips" : ["mips"],
  44                                 "arm" : ["arm"]
  45                 }
  46
  47                 machinemap={
  48                                 "i386" : "x86",
  49                                 "i486" : "x86",
  50                                 "i586" : "x86",
  51                                 "i686" : "x86",
  52                                 "x86_64" : "amd64",
  53                                 "sparc64" : "sparc64",
  54                                 "ia64" : "ia64",
  55                                 "alpha" : "alpha",
  56                                 "sparc" : "sparc",
  57                                 "sh2" : "sh",
  58                                 "sh3" : "sh",
  59                                 "sh4" : "sh",
  60                                 "sh2eb" : "sh",
  61                                 "sh3eb" : "sh",
  62                                 "sh4eb" : "sh",
  63                                 "s390" : "s390",
  64                                 "ppc" : "ppc",
  65                                 "ppc64" : "ppc64",
  66                                 "parisc" : "hppa",
  67                                 "parisc64" : "hppa",
  68                                 "mips" : "mips",
  69                                 "mips64" : "mips",
  70                                 "arm" : "arm",
  71                                 "armv4l" : "arm",
  72                                 "armeb" : "arm",
  73                                 "armv5b" : "arm"
  74                 }
  75
  76                 mymachine=os.uname()[4]
  77                 if not machinemap.has_key(mymachine):
  78                         raise CatalystError, "Unknown machine type "+mymachine
  79
  80                 self.settings["hostarch"]=machinemap[mymachine]
  81                 self.archmap={}
  82                 self.subarchmap={}
  83
  84                 for x in targetmap[self.settings["hostarch"]]:
  85                         try:
  86                                 fh=open(self.settings["sharedir"]+"/arch/"+x+".py")
  87                                 # this next line loads the plugin as a module and assigns it to archmap[x]
  88                                 self.archmap[x]=imp.load_module(x,fh,"arch/"+x+".py",(".py","r",imp.PY_SOURCE))
  89                                 # this next line registers all the subarches supported in the plugin
  90                                 self.archmap[x].register(self.subarchmap)
  91                                 fh.close()
  92
  93                         except IOError:
  94                                 msg("Can't find "+x+".py plugin in "+self.settings["sharedir"]+"/arch/")
  95                 # call arch constructor, pass our settings
  96                 try:
  97                         self.arch=self.subarchmap[self.settings["subarch"]](self.settings)
  98                 except:
  99                         print "Invalid subarch: "+self.settings["subarch"]
 100                         print "Choose one of the following:",
 101                         for x in self.subarchmap:
 102                                 print x,
 103                         print
 104                         sys.exit(2)
 105
 106                 print "Using target:",self.settings["target"]
 107                 # self.settings["mainarch"] should now be set by our arch constructor, so we print
 108                 # a nice informational message:
 109                 if self.settings["mainarch"]==self.settings["hostarch"]:
 110                         print "Building natively for",self.settings["hostarch"]
 111
 112                 else:
 113                         print "Building on",self.settings["hostarch"],"for alternate machine type",\
 114                                 self.settings["mainarch"]
 115                 # This should be first to be set as other set_ options depend on this
 116                 self.set_spec_prefix()
 117
 118                 # define all of our core variables
 119                 self.set_target_profile()
 120                 self.set_target_subpath()
 121                 self.set_source_subpath()
 122
 123                 # set paths
 124                 self.set_snapshot_path()
 125                 self.set_root_path()
 126                 self.set_source_path()
 127                 self.set_snapcache_path()
 128                 self.set_chroot_path()
 129                 self.set_autoresume_path()
 130                 self.set_dest_path()
 131                 self.set_stage_path()
 132                 self.set_target_path()
 133
 134                 self.set_controller_file()
 135                 self.set_action_sequence()
 136                 self.set_use()
 137                 self.set_cleanables()
 138                 self.set_iso_volume_id()
 139                 self.set_build_kernel_vars()
 140                 self.set_fsscript()
 141                 self.set_archscript()
 142                 self.set_runscript()
 143                 self.set_install_mask()
 144                 self.set_rcadd()
 145                 self.set_rcdel()
 146                 self.set_cdtar()
 147                 self.set_fstype()
 148                 self.set_fsops()
 149                 self.set_iso()
 150                 self.set_packages()
 151                 self.set_rm()
 152                 self.set_linuxrc()
 153                 self.set_overlay()
 154                 self.set_portage_overlay()
 155                 self.set_root_overlay()
 156
 157                 # this next line checks to make sure that the specified variables exist on disk.
 158                 #pdb.set_trace()
 159                 file_locate(self.settings,["source_path","snapshot_path","distdir"],expand=0)
 160                 # if we are using portage_confdir, check that as well
 161                 if self.settings.has_key("portage_confdir"):
 162                         file_locate(self.settings,["portage_confdir"],expand=0)
 163
 164                 # setup our mount points
 165                 if self.settings.has_key("SNAPCACHE"):
 166                         self.mounts=[ "/proc","/dev","/dev/pts","/usr/portage","/usr/portage/distfiles" ]
 167                         self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
 168                                 "/usr/portage":self.settings["snapshot_cache_path"]+"/portage",\
 169                                 "/usr/portage/distfiles":self.settings["distdir"]}
 170                 else:
 171                         self.mounts=[ "/proc","/dev","/dev/pts","/usr/portage/distfiles" ]
 172                         self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
 173                                 "/usr/portage/distfiles":self.settings["distdir"]}
 174
 175                 self.set_mounts()
 176
 177                 # configure any user specified options (either in catalyst.conf or on the cmdline)
 178                 if self.settings.has_key("PKGCACHE"):
 179                         self.set_pkgcache_path()
 180                         print "Location of the package cache is " + self.settings["pkgcache_path"]
 181                         self.mounts.append("/usr/portage/packages")
 182                         self.mountmap["/usr/portage/packages"]=self.settings["pkgcache_path"]
 183
 184                 if self.settings.has_key("KERNCACHE"):
 185                         self.set_kerncache_path()
 186                         print "Location of the kerncache is " + self.settings["kerncache_path"]
 187                         self.mounts.append("/tmp/kerncache")
 188                         self.mountmap["/tmp/kerncache"]=self.settings["kerncache_path"]
 189
 190                 if self.settings.has_key("CCACHE"):
 191                         if os.environ.has_key("CCACHE_DIR"):
 192                                 ccdir=os.environ["CCACHE_DIR"]
 193                                 del os.environ["CCACHE_DIR"]
 194                         else:
 195                                 ccdir="/root/.ccache"
 196                         if not os.path.isdir(ccdir):
 197                                         raise CatalystError,\
 198                                                 "Compiler cache support can't be enabled (can't find "+ccdir+")"
 199                         self.mounts.append("/var/tmp/ccache")
 200                         self.mountmap["/var/tmp/ccache"]=ccdir
 201                         # for the chroot:
 202                         self.env["CCACHE_DIR"]="/var/tmp/ccache"
 203
 204         def override_chost(self):
 205                 #if os.environ.has_key("CHOST"):
 206                 #    self.settings["CHOST"] = os.environ["CHOST"]
 207                 if self.settings.has_key("chost"):
 208                     self.settings["CHOST"]=list_to_string(self.settings["chost"])
 209                 if self.makeconf.has_key("CHOST"):
 210                     #print "Using CHOST setting from seed stage"
 211                     self.settings["CHOST"]=self.makeconf["CHOST"]
 212
 213         def override_cflags(self):
 214                 #if os.environ.has_key("CFLAGS"):
 215                 #    self.settings["CFLAGS"] = os.environ["CFLAGS"]
 216                 if self.settings.has_key("cflags"):
 217                     self.settings["CFLAGS"]=list_to_string(self.settings["cflags"])
 218                 if self.makeconf.has_key("CFLAGS"):
 219                     #print "Using CFLAGS setting from seed stage"
 220                     self.settings["CFLAGS"]=self.makeconf["CFLAGS"]
 221
 222         def override_cxxflags(self):
 223                 #if os.environ.has_key("CXXFLAGS"):
 224                 #    self.settings["CXXFLAGS"] = os.environ["CXXFLAGS"]
 225                 if self.settings.has_key("cxxflags"):
 226                     self.settings["CXXFLAGS"]=list_to_string(self.settings["cxxflags"])
 227                 if self.makeconf.has_key("CXXFLAGS"):
 228                     #print "Using CXXFLAGS setting from seed stage"
 229                     self.settings["CXXFLAGS"]=self.makeconf["CXXFLAGS"]
 230
 231         def override_ldflags(self):
 232                 #if os.environ.has_key("LDFLAGS"):
 233                 #    self.settings["LDFLAGS"] = os.environ["LDFLAGS"]
 234                 if self.settings.has_key("ldflags"):
 235                     self.settings["LDFLAGS"]=list_to_string(self.settings["ldflags"])
 236                 if self.makeconf.has_key("LDFLAGS"):
 237                     #print "Using LDFLAGS setting from seed stage"
 238                     self.settings["LDFLAGS"]=self.makeconf["LDFLAGS"]
 239
 240         def set_install_mask(self):
 241                 if self.settings.has_key("install_mask"):
 242                         if type(self.settings["install_mask"]) != types.StringType:
 243                                 self.settings["install_mask"]=string.join(self.settings["install_mask"])
 244
 245         def set_spec_prefix(self):
 246                 self.settings["spec_prefix"]=self.settings["target"]
 247
 248         def set_target_profile(self):
 249                 self.settings["target_profile"]=self.settings["profile"]
 250
 251         def set_target_subpath(self):
 252                 self.settings["target_subpath"]=self.settings["rel_type"]+"/"+self.settings["target"]+\
 253                         "-"+self.settings["subarch"]+"-"+self.settings["version_stamp"]
 254
 255         def set_source_subpath(self):
 256                 if type(self.settings["source_subpath"]) != types.StringType:
 257                         raise CatalystError, "source_subpath should have been a string. Perhaps you have something wrong in your spec file?"
 258
 259         def set_pkgcache_path(self):
 260                 if self.settings.has_key("pkgcache_path"):
 261                         if type(self.settings["pkgcache_path"]) != types.StringType:
 262                                 self.settings["pkgcache_path"]=normpath(string.join(self.settings["pkgcache_path"]))
 263                 else:
 264                         self.settings["pkgcache_path"]=normpath(self.settings["storedir"]+"/packages/"+\
 265                                 self.settings["target_subpath"]+"/")
 266
 267         def set_kerncache_path(self):
 268                 if self.settings.has_key("kerncache_path"):
 269                         if type(self.settings["kerncache_path"]) != types.StringType:
 270                                 self.settings["kerncache_path"]=normpath(string.join(self.settings["kerncache_path"]))
 271                 else:
 272                         self.settings["kerncache_path"]=normpath(self.settings["storedir"]+"/kerncache/"+\
 273                                 self.settings["target_subpath"]+"/")
 274
 275         def set_target_path(self):
 276                 self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\
 277                         self.settings["target_subpath"]+".tar.bz2")
 278                 if self.settings.has_key("AUTORESUME") \
 279                         and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"):
 280                         print "Resume point detected, skipping target path setup operation..."
 281                 else:
 282                         # first clean up any existing target stuff
 283                         if os.path.isfile(self.settings["target_path"]):
 284                                 cmd("rm -f "+self.settings["target_path"], \
 285                                         "Could not remove existing file: "+self.settings["target_path"],env=self.env)
 286                                 touch(self.settings["autoresume_path"]+"setup_target_path")
 287
 288                         if not os.path.exists(self.settings["storedir"]+"/builds/"):
 289                                 os.makedirs(self.settings["storedir"]+"/builds/")
 290
 291         def set_archscript(self):
 292             if self.settings.has_key(self.settings["spec_prefix"]+"/archscript"):
 293                 print "\nWarning!!!  "
 294                 print "\t"+self.settings["spec_prefix"]+"/archscript" + " is deprecated and no longer used.\n"
 295         def set_runscript(self):
 296             if self.settings.has_key(self.settings["spec_prefix"]+"/runscript"):
 297                 print "\nWarning!!!  "
 298                 print "\t"+self.settings["spec_prefix"]+"/runscript" + " is deprecated and no longer used.\n"
 299
 300         def set_fsscript(self):
 301                 if self.settings.has_key(self.settings["spec_prefix"]+"/fsscript"):
 302                         self.settings["fsscript"]=self.settings[self.settings["spec_prefix"]+"/fsscript"]
 303                         del self.settings[self.settings["spec_prefix"]+"/fsscript"]
 304
 305         def set_rcadd(self):
 306                 if self.settings.has_key(self.settings["spec_prefix"]+"/rcadd"):
 307                         self.settings["rcadd"]=self.settings[self.settings["spec_prefix"]+"/rcadd"]
 308                         del self.settings[self.settings["spec_prefix"]+"/rcadd"]
 309
 310         def set_rcdel(self):
 311                 if self.settings.has_key(self.settings["spec_prefix"]+"/rcdel"):
 312                         self.settings["rcdel"]=self.settings[self.settings["spec_prefix"]+"/rcdel"]
 313                         del self.settings[self.settings["spec_prefix"]+"/rcdel"]
 314
 315         def set_cdtar(self):
 316                 if self.settings.has_key(self.settings["spec_prefix"]+"/cdtar"):
 317                         self.settings["cdtar"]=normpath(self.settings[self.settings["spec_prefix"]+"/cdtar"])
 318                         del self.settings[self.settings["spec_prefix"]+"/cdtar"]
 319
 320         def set_iso(self):
 321                 if self.settings.has_key(self.settings["spec_prefix"]+"/iso"):
 322                         self.settings["iso"]=normpath(self.settings[self.settings["spec_prefix"]+"/iso"])
 323                         del self.settings[self.settings["spec_prefix"]+"/iso"]
 324
 325         def set_fstype(self):
 326                 if self.settings.has_key(self.settings["spec_prefix"]+"/cdfstype"):
 327                         print "\nWarning!!!  "
 328                         print self.settings["spec_prefix"]+"/cdfstype" + " is deprecated and may be removed."
 329                         print "\tUse "+self.settings["spec_prefix"]+"/fstype" + " instead."
 330                         print "\tConverting to "+self.settings["spec_prefix"]+"/fstype" + " internally."
 331                         print "\tContinuing ....\n"
 332                         self.settings["fstype"]=self.settings[self.settings["spec_prefix"]+"/cdfstype"]
 333                         del self.settings[self.settings["spec_prefix"]+"/cdfstype"]
 334
 335                 if self.settings.has_key(self.settings["spec_prefix"]+"/fstype"):
 336                         self.settings["fstype"]=self.settings[self.settings["spec_prefix"]+"/fstype"]
 337                         del self.settings[self.settings["spec_prefix"]+"/fstype"]
 338
 339                 if not self.settings.has_key("fstype"):
 340                         self.settings["fstype"]="normal"
 341                         for x in self.valid_values:
 342                                 if x ==  self.settings["spec_prefix"]+"/fstype" or x == self.settings["spec_prefix"]+"/cdfstype":
 343                                         print "\n"+self.settings["spec_prefix"]+"/fstype is being set to the default of \"normal\"\n"
 344
 345         def set_fsops(self):
 346                 if self.settings.has_key("fstype"):
 347                         self.valid_values.append("fsops")
 348                         if self.settings.has_key(self.settings["spec_prefix"]+"/fs-ops"):
 349                                 print "\n\n\nWarning!!!"+self.settings["spec_prefix"]+"/fs-ops" + "is deprecated and may be removed."
 350                                 print "\tUse "+self.settings["spec_prefix"]+"/fsops" + "instead.\n\n\n"
 351                                 self.settings["fsops"]=self.settings[self.settings["spec_prefix"]+"/fs-ops"]
 352                                 del self.settings[self.settings["spec_prefix"]+"/fs-ops"]
 353
 354                         if self.settings.has_key(self.settings["spec_prefix"]+"/fsops"):
 355                                 self.settings["fsops"]=self.settings[self.settings["spec_prefix"]+"/fsops"]
 356                                 del self.settings[self.settings["spec_prefix"]+"/fsops"]
 357
 358         def set_source_path(self):
 359                 if self.settings.has_key("SEEDCACHE") and os.path.isdir(normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")):
 360                         self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")
 361                 else:
 362                         self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2")
 363
 364                         if os.path.isfile(self.settings["source_path"]):
 365                                 if os.path.exists(self.settings["source_path"]):
 366                                         self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\
 367                                                         hash_function=self.settings["hash_function"],verbose=False)
 368                 if os.path.isdir(self.settings["source_path"]):
 369                         print "Source path set to "+self.settings["source_path"]
 370                         print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf"
 371                         print "\tthe source path with then be "+normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2\n")
 372                 else:
 373                         print "Source path set to "+self.settings["source_path"]
 374
 375
 376         def set_dest_path(self):
 377                 if self.settings.has_key("root_path"):
 378                         self.settings["destpath"]=normpath(self.settings["chroot_path"]+self.settings["root_path"])
 379                 else:
 380                         self.settings["destpath"]=normpath(self.settings["chroot_path"])
 381
 382         def set_cleanables(self):
 383                 self.settings["cleanables"]=["/etc/resolv.conf","/var/tmp/*","/tmp/*","/root/*",\
 384                                                 "/usr/portage"]
 385
 386         def set_snapshot_path(self):
 387                 self.settings["snapshot_path"]=normpath(self.settings["storedir"]+"/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
 388
 389                 if os.path.exists(self.settings["snapshot_path"]):
 390                         self.settings["snapshot_path_hash"]=generate_hash(self.settings["snapshot_path"],\
 391                                         hash_function=self.settings["hash_function"],verbose=False)
 392
 393         def set_snapcache_path(self):
 394                 if self.settings.has_key("SNAPCACHE"):
 395                         self.settings["snapshot_cache_path"]=normpath(self.settings["snapshot_cache"]+"/"+self.settings["snapshot"]+"/")
 396                         self.snapcache_lock=catalyst_lock.LockDir(self.settings["snapshot_cache_path"])
 397                         print "Caching snapshot to " + self.settings["snapshot_cache_path"]
 398
 399         def set_chroot_path(self):
 400                 # Note the trailing slash is very important and things would break without it
 401                 self.settings["chroot_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["target_subpath"]+"/")
 402                 self.chroot_lock=catalyst_lock.LockDir(self.settings["chroot_path"])
 403
 404         def set_autoresume_path(self):
 405                 self.settings["autoresume_path"]=normpath(self.settings["storedir"]+"/tmp/"+\
 406                         self.settings["rel_type"]+"/"+".autoresume-"+self.settings["target"]+\
 407                         "-"+self.settings["subarch"]+"-"+self.settings["version_stamp"]+"/")
 408                 if self.settings.has_key("AUTORESUME"):
 409                         print "The autoresume path is " + self.settings["autoresume_path"]
 410                 if not os.path.exists(self.settings["autoresume_path"]):
 411                         os.makedirs(self.settings["autoresume_path"],0755)
 412
 413         def set_controller_file(self):
 414                 self.settings["controller_file"]=normpath(self.settings["sharedir"]+"/targets/"+self.settings["target"]+"/"+self.settings["target"]+"-controller.sh")
 415         def set_iso_volume_id(self):
 416                 if self.settings.has_key(self.settings["spec_prefix"]+"/volid"):
 417                         self.settings["iso_volume_id"] = string.join(self.settings[self.settings["spec_prefix"]+"/volid"])
 418                         if len(self.settings["iso_volume_id"])>32:
 419                                 raise CatalystError,"ISO VOLUME ID: volid must not exceed 32 characters."
 420                 else:
 421                         self.settings["iso_volume_id"] = "catalyst " + self.settings["snapshot"]
 422
 423         def set_action_sequence(self):
 424                 #Default action sequence for run method
 425                 self.settings["action_sequence"]=["unpack","unpack_snapshot",\
 426                                 "config_profile_link","setup_confdir","portage_overlay","bind","chroot_setup",\
 427                                 "setup_environment","run_local","preclean","unbind","clean","capture","clear_autoresume"]
 428
 429         def set_use(self):
 430                 if self.settings.has_key(self.settings["spec_prefix"]+"/use"):
 431                         self.settings["use"]=self.settings[self.settings["spec_prefix"]+"/use"]
 432                         del self.settings[self.settings["spec_prefix"]+"/use"]
 433                 if self.settings.has_key("use"):
 434                     if type(self.settings["use"])==types.StringType:
 435                         self.settings["use"]=self.settings["use"].split()
 436
 437         def set_stage_path(self):
 438                         self.settings["stage_path"]=normpath(self.settings["chroot_path"])
 439
 440         def set_mounts(self):
 441                 pass
 442
 443         def set_packages(self):
 444                 pass
 445
 446         def set_rm(self):
 447             if self.settings.has_key(self.settings["spec_prefix"]+"/rm"):
 448                 if type(self.settings[self.settings["spec_prefix"]+"/rm"])==types.StringType:
 449                     self.settings[self.settings["spec_prefix"]+"/rm"]=self.settings[self.settings["spec_prefix"]+"/rm"].split()
 450
 451         def set_linuxrc(self):
 452             if self.settings.has_key(self.settings["spec_prefix"]+"/linuxrc"):
 453                         if type(self.settings[self.settings["spec_prefix"]+"/linuxrc"])==types.StringType:
 454                                 self.settings["linuxrc"]=self.settings[self.settings["spec_prefix"]+"/linuxrc"]
 455                                 del self.settings[self.settings["spec_prefix"]+"/linuxrc"]
 456
 457         def set_portage_overlay(self):
 458                 if self.settings.has_key("portage_overlay"):
 459                         if type(self.settings["portage_overlay"])==types.StringType:
 460                                 self.settings["portage_overlay"]=self.settings["portage_overlay"].split()
 461                         print "portage_overlay directories are set to: \"" + string.join(self.settings["portage_overlay"])+"\""
 462
 463         def set_overlay(self):
 464                 if self.settings.has_key(self.settings["spec_prefix"]+"/overlay"):
 465                         if type(self.settings[self.settings["spec_prefix"]+"/overlay"])==types.StringType:
 466                                 self.settings[self.settings["spec_prefix"]+"/overlay"]=self.settings[self.settings["spec_prefix"]+"/overlay"].split()
 467
 468         def set_root_overlay(self):
 469                 if self.settings.has_key(self.settings["spec_prefix"]+"/root_overlay"):
 470                         if type(self.settings[self.settings["spec_prefix"]+"/root_overlay"])==types.StringType:
 471                                 self.settings[self.settings["spec_prefix"]+"/root_overlay"]=self.settings[self.settings["spec_prefix"]+"/root_overlay"].split()
 472
 473
 474         def set_root_path(self):
 475                 # ROOT= variable for emerges
 476                 self.settings["root_path"]="/"
 477
 478         def set_valid_build_kernel_vars(self,addlargs):
 479                 if addlargs.has_key("boot/kernel"):
 480                         if type(addlargs["boot/kernel"]) == types.StringType:
 481                                 loopy=[addlargs["boot/kernel"]]
 482                         else:
 483                                 loopy=addlargs["boot/kernel"]
 484
 485                         for x in loopy:
 486                                 self.required_values.append("boot/kernel/"+x+"/sources")
 487                                 self.required_values.append("boot/kernel/"+x+"/config")
 488                                 self.valid_values.append("boot/kernel/"+x+"/aliases")
 489                                 self.valid_values.append("boot/kernel/"+x+"/extraversion")
 490                                 self.valid_values.append("boot/kernel/"+x+"/packages")
 491                                 if addlargs.has_key("boot/kernel/"+x+"/packages"):
 492                                         if type(addlargs["boot/kernel/"+x+"/packages"]) == types.StringType:
 493                                                 addlargs["boot/kernel/"+x+"/packages"]=[addlargs["boot/kernel/"+x+"/packages"]]
 494                                 self.valid_values.append("boot/kernel/"+x+"/use")
 495                                 self.valid_values.append("boot/kernel/"+x+"/gk_kernargs")
 496                                 self.valid_values.append("boot/kernel/"+x+"/gk_action")
 497                                 self.valid_values.append("boot/kernel/"+x+"/initramfs_overlay")
 498                                 self.valid_values.append("boot/kernel/"+x+"/softlevel")
 499                                 self.valid_values.append("boot/kernel/"+x+"/console")
 500                                 self.valid_values.append("boot/kernel/"+x+"/machine_type")
 501                                 self.valid_values.append("boot/kernel/"+x+"/postconf")
 502                                 if addlargs.has_key("boot/kernel/"+x+"/postconf"):
 503                                         print "boot/kernel/"+x+"/postconf is deprecated"
 504                                         print "\tInternally moving these ebuilds to boot/kernel/"+x+"/packages"
 505                                         print "\tPlease move them to boot/kernel/"+x+"/packages in your specfile"
 506                                         if type(addlargs["boot/kernel/"+x+"/postconf"]) == types.StringType:
 507                                                 loop2=[addlargs["boot/kernel/"+x+"/postconf"]]
 508                                         else:
 509                                                 loop2=addlargs["boot/kernel/"+x+"/postconf"]
 510
 511                                         for y in loop2:
 512                                                 if not addlargs.has_key("boot/kernel/"+x+"/packages"):
 513                                                         addlargs["boot/kernel/"+x+"/packages"]=[[y]]
 514                                                 else:
 515                                                         addlargs["boot/kernel/"+x+"/packages"].append(y)
 516
 517         def set_build_kernel_vars(self):
 518             if self.settings.has_key(self.settings["spec_prefix"]+"/devmanager"):
 519                 self.settings["devmanager"]=self.settings[self.settings["spec_prefix"]+"/devmanager"]
 520                 del self.settings[self.settings["spec_prefix"]+"/devmanager"]
 521
 522             if self.settings.has_key(self.settings["spec_prefix"]+"/splashtype"):
 523                 self.settings["splashtype"]=self.settings[self.settings["spec_prefix"]+"/splashtype"]
 524                 del self.settings[self.settings["spec_prefix"]+"/splashtype"]
 525
 526             if self.settings.has_key(self.settings["spec_prefix"]+"/gk_mainargs"):
 527                 self.settings["gk_mainargs"]=self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
 528                 del self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
 529
 530         def kill_chroot_pids(self):
 531             print "Checking for processes running in chroot and killing them."
 532
 533             # Force environment variables to be exported so script can see them
 534             self.setup_environment()
 535
 536             if os.path.exists(self.settings["sharedir"]+"/targets/support/kill-chroot-pids.sh"):
 537                             cmd("/bin/bash "+self.settings["sharedir"]+"/targets/support/kill-chroot-pids.sh",\
 538                                                 "kill-chroot-pids script failed.",env=self.env)
 539
 540         def mount_safety_check(self):
 541                 mypath=self.settings["chroot_path"]
 542
 543                 """
 544                 check and verify that none of our paths in mypath are mounted. We don't want to clean
 545                 up with things still mounted, and this allows us to check.
 546                 returns 1 on ok, 0 on "something is still mounted" case.
 547                 """
 548                 if not os.path.exists(mypath):
 549                         return
 550
 551                 for x in self.mounts:
 552                         if not os.path.exists(mypath+x):
 553                                 continue
 554
 555                         if ismount(mypath+x):
 556                                 #something is still mounted
 557                                 try:
 558                                         print x+" is still mounted; performing auto-bind-umount...",
 559                                         # try to umount stuff ourselves
 560                                         self.unbind()
 561                                         if ismount(mypath+x):
 562                                                 raise CatalystError, "Auto-unbind failed for "+x
 563
 564                                         else:
 565                                                 print "Auto-unbind successful..."
 566
 567                                 except CatalystError:
 568                                         raise CatalystError, "Unable to auto-unbind "+x
 569
 570         def unpack(self):
 571                 unpack=True
 572
 573                 clst_unpack_hash=read_from_clst(self.settings["autoresume_path"]+"unpack")
 574
 575                 if self.settings.has_key("SEEDCACHE") and os.path.isdir(self.settings["source_path"]):
 576                         unpack_cmd="rsync -a --delete "+self.settings["source_path"]+" "+self.settings["chroot_path"]
 577                         display_msg="\nStarting rsync from "+self.settings["source_path"]+"\nto "+\
 578                                 self.settings["chroot_path"]+" (This may take some time) ...\n"
 579                         error_msg="Rsync of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 580                 else:
 581                         display_msg="\nStarting tar extract from "+self.settings["source_path"]+"\nto "+\
 582                                 self.settings["chroot_path"]+" (This may take some time) ...\n"
 583                         unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"]
 584                         error_msg="Tarball extraction of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 585
 586
 587                 if self.settings.has_key("AUTORESUME"):
 588                         # Autoresume is Valid, SeedCache is Valid
 589                         if os.path.isdir(self.settings["source_path"]) and os.path.exists(self.settings["autoresume_path"]+"unpack"):
 590                                 unpack=False
 591                                 invalid_snapshot=False
 592
 593                         # Autoresume is Valid, Tarball is Valid
 594                         elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] == clst_unpack_hash:
 595                                 unpack=False
 596                                 invalid_snapshot=True
 597
 598                         # Autoresume is InValid, SeedCache
 599                         elif os.path.isdir(self.settings["source_path"]) and not os.path.exists(self.settings["autoresume_path"]+"unpack"):
 600                                 unpack=True
 601                                 invalid_snapshot=False
 602
 603                         # Autoresume is InValid, Tarball
 604                         elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] != clst_unpack_hash:
 605                                 unpack=True
 606                                 invalid_snapshot=True
 607                 else:
 608                         # No Autoresume,SeedCache
 609                         if self.settings.has_key("SEEDCACHE"):
 610
 611                                 # Seed cache so lets run rsync and rsync can clean up extra junk
 612                                 if os.path.isdir(self.settings["source_path"]):
 613                                         unpack=True
 614                                         invalid_snapshot=False
 615
 616                                 # Tarball so we better unpack and remove anything already there
 617                                 elif os.path.isfile(self.settings["source_path"]):
 618                                         unpack=True
 619                                         invalid_snapshot=True
 620
 621                         # No Autoresume,No SeedCache
 622                         else:
 623
 624                                 # Tarball so we better unpack and remove anything already there
 625                                 if os.path.isfile(self.settings["source_path"]):
 626                                         unpack=True
 627                                         invalid_snapshot=True
 628                                 # Should never reach this if so something is very wrong
 629                                 elif os.path.isdir(self.settings["source_path"]):
 630                                         raise CatalystError,"source path is a dir but seedcache is not enabled"
 631
 632                 if unpack:
 633                         self.mount_safety_check()
 634
 635                         if invalid_snapshot:
 636                                 if self.settings.has_key("AUTORESUME"):
 637                                         print "No Valid Resume point detected, cleaning up  ..."
 638
 639                                 self.clear_autoresume()
 640                                 self.clear_chroot()
 641
 642                         if not os.path.exists(self.settings["chroot_path"]):
 643                                 os.makedirs(self.settings["chroot_path"])
 644
 645                         if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
 646                                 os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
 647
 648                         if self.settings.has_key("PKGCACHE"):
 649                                 if not os.path.exists(self.settings["pkgcache_path"]):
 650                                         os.makedirs(self.settings["pkgcache_path"],0755)
 651
 652                         if self.settings.has_key("KERNCACHE"):
 653                                 if not os.path.exists(self.settings["kerncache_path"]):
 654                                         os.makedirs(self.settings["kerncache_path"],0755)
 655
 656                         print display_msg
 657                         cmd(unpack_cmd,error_msg,env=self.env)
 658
 659                         if self.settings.has_key("source_path_hash"):
 660                                 myf=open(self.settings["autoresume_path"]+"unpack","w")
 661                                 myf.write(self.settings["source_path_hash"])
 662                                 myf.close()
 663                         else:
 664                                 touch(self.settings["autoresume_path"]+"unpack")
 665                 else:
 666                     print "Resume point detected, skipping unpack operation..."
 667
 668
 669         def unpack_snapshot(self):
 670                 unpack=True
 671                 snapshot_hash=read_from_clst(self.settings["autoresume_path"]+"unpack_portage")
 672
 673                 if self.settings.has_key("SNAPCACHE"):
 674                         snapshot_cache_hash=read_from_clst(self.settings["snapshot_cache_path"]+"catalyst-hash")
 675                         destdir=self.settings["snapshot_cache_path"]
 676                         unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+destdir
 677                         unpack_errmsg="Error unpacking snapshot"
 678                         cleanup_msg="Cleaning up invalid snapshot cache at \n\t"+self.settings["snapshot_cache_path"]+" (This can take a long time)..."
 679                         cleanup_errmsg="Error removing existing snapshot cache directory."
 680                         self.snapshot_lock_object=self.snapcache_lock
 681
 682                         if self.settings["snapshot_path_hash"] == snapshot_cache_hash:
 683                                 print "Valid snapshot cache, skipping unpack of portage tree ..."
 684                                 unpack=False
 685
 686                 else:
 687                         destdir=normpath(self.settings["chroot_path"]+"/usr/portage")
 688                         cleanup_errmsg="Error removing existing snapshot directory."
 689                         cleanup_msg="Cleaning up existing portage tree (This can take a long time) ..."
 690                         unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+self.settings["chroot_path"]+"/usr"
 691                         unpack_errmsg="Error unpacking snapshot"
 692
 693                         if self.settings.has_key("AUTORESUME") \
 694                         and os.path.exists(self.settings["chroot_path"]+"/usr/portage/") \
 695                         and os.path.exists(self.settings["autoresume_path"]+"unpack_portage") \
 696                         and self.settings["snapshot_path_hash"] == snapshot_hash:
 697                                 print "Valid Resume point detected, skipping unpack of portage tree..."
 698                                 unpack=False
 699
 700
 701
 702                 if unpack:
 703                         if self.settings.has_key("SNAPCACHE"):
 704                             self.snapshot_lock_object.write_lock()
 705                         if os.path.exists(destdir):
 706                                 print cleanup_msg
 707                                 cleanup_cmd="rm -rf "+destdir
 708                                 cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
 709                         if not os.path.exists(destdir):
 710                                 os.makedirs(destdir,0755)
 711
 712                         print "Unpacking portage tree (This can take a long time) ..."
 713                         cmd(unpack_cmd,unpack_errmsg,env=self.env)
 714
 715                         if self.settings.has_key("SNAPCACHE"):
 716                                 myf=open(self.settings["snapshot_cache_path"]+"catalyst-hash","w")
 717                                 myf.write(self.settings["snapshot_path_hash"])
 718                                 myf.close()
 719
 720                         else:
 721                                 print "Setting snapshot autoresume point"
 722                                 myf=open(self.settings["autoresume_path"]+"unpack_portage","w")
 723                                 myf.write(self.settings["snapshot_path_hash"])
 724                                 myf.close()
 725
 726                         if self.settings.has_key("SNAPCACHE"):
 727                             self.snapshot_lock_object.unlock()
 728
 729         def config_profile_link(self):
 730                 if self.settings.has_key("AUTORESUME") \
 731                         and os.path.exists(self.settings["autoresume_path"]+"config_profile_link"):
 732                         print "Resume point detected, skipping config_profile_link operation..."
 733                 else:
 734                         print "Configuring profile link..."
 735                         cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.profile",\
 736                                         "Error zapping profile link",env=self.env)
 737                         cmd("ln -sf ../usr/portage/profiles/"+self.settings["target_profile"]+\
 738                                 " "+self.settings["chroot_path"]+"/etc/make.profile","Error creating profile link",env=self.env)
 739                         touch(self.settings["autoresume_path"]+"config_profile_link")
 740
 741         def setup_confdir(self):
 742                 if self.settings.has_key("AUTORESUME") \
 743                         and os.path.exists(self.settings["autoresume_path"]+"setup_confdir"):
 744                         print "Resume point detected, skipping setup_confdir operation..."
 745                 else:
 746                         if self.settings.has_key("portage_confdir"):
 747                                 print "Configuring /etc/portage..."
 748                                 cmd("rm -rf "+self.settings["chroot_path"]+"/etc/portage","Error zapping /etc/portage",env=self.env)
 749                                 cmd("cp -R "+self.settings["portage_confdir"]+"/ "+self.settings["chroot_path"]+\
 750                                         "/etc/portage","Error copying /etc/portage",env=self.env)
 751                                 touch(self.settings["autoresume_path"]+"setup_confdir")
 752
 753         def portage_overlay(self):
 754             # copy over the portage overlays
 755             # Always copy over the overlay incase it has changed
 756             if self.settings.has_key("portage_overlay"):
 757                 for x in self.settings["portage_overlay"]:
 758                         if os.path.exists(x):
 759                                 print "Copying overlay dir " +x
 760                                 cmd("mkdir -p "+self.settings["chroot_path"]+x,"Could not make portage_overlay dir",env=self.env)
 761                                 #cmd("rsync -a --delete "+x+"/* "+self.settings["chroot_path"]+x,\
 762                                 cmd("rsync -a --delete "+x+"/ "+self.settings["chroot_path"]+x,\
 763                                                 "Could not copy portage_overlay",env=self.env)
 764                                 #cmd("cp -R "+x+"/* "+self.settings["chroot_path"]+x,"Could not copy portage_overlay",env=self.env)
 765
 766         def root_overlay(self):
 767             # copy over the root_overlay
 768             # Always copy over the overlay incase it has changed
 769                 if self.settings.has_key(self.settings["spec_prefix"]+"/root_overlay"):
 770                         for x in self.settings[self.settings["spec_prefix"]+"/root_overlay"]:
 771                                 if os.path.exists(x):
 772                                         print "Copying root_overlay: "+x
 773                                         cmd("rsync -a "+x+"/ "+\
 774                                                 self.settings["chroot_path"], self.settings["spec_prefix"]+"/root_overlay: "+x+" copy failed.",env=self.env)
 775
 776         def bind(self):
 777                 for x in self.mounts:
 778                         if not os.path.exists(self.settings["chroot_path"]+x):
 779                                 os.makedirs(self.settings["chroot_path"]+x,0755)
 780
 781                         if not os.path.exists(self.mountmap[x]):
 782                                 os.makedirs(self.mountmap[x],0755)
 783
 784                         src=self.mountmap[x]
 785                         if self.settings.has_key("SNAPCACHE") and x == "/usr/portage":
 786                                 self.snapshot_lock_object.read_lock()
 787                         retval=os.system("mount --bind "+src+" "+self.settings["chroot_path"]+x)
 788                         if retval!=0:
 789                                 self.unbind()
 790                                 raise CatalystError,"Couldn't bind mount "+src
 791
 792
 793         def unbind(self):
 794                 ouch=0
 795                 mypath=self.settings["chroot_path"]
 796                 myrevmounts=self.mounts[:]
 797                 myrevmounts.reverse()
 798                 # unmount in reverse order for nested bind-mounts
 799                 for x in myrevmounts:
 800                         if not os.path.exists(mypath+x):
 801                                 continue
 802
 803                         if not ismount(mypath+x):
 804                                 # it's not mounted, continue
 805                                 continue
 806
 807                         retval=os.system("umount "+mypath+x)
 808
 809                         if retval!=0:
 810                                 warn("First attempt to unmount: "+mypath+x+" failed.")
 811                                 warn("Killing any pids still running in the chroot")
 812
 813                                 self.kill_chroot_pids()
 814
 815                                 retval2=os.system("umount "+mypath+x)
 816                                 if retval2!=0:
 817                                     ouch=1
 818                                     warn("Couldn't umount bind mount: "+mypath+x)
 819                                     # keep trying to umount the others, to minimize damage if developer makes a mistake
 820
 821                         if self.settings.has_key("SNAPCACHE") and x == "/usr/portage":
 822                                 try:
 823                                     # Its possible the snapshot lock object isnt created yet
 824                                     # this is because mount safety check calls unbind before the target is fully initialized
 825                                     self.snapshot_lock_object.unlock()
 826                                 except:
 827                                     pass
 828                 if ouch:
 829                         """
 830                         if any bind mounts really failed, then we need to raise
 831                         this to potentially prevent an upcoming bash stage cleanup script
 832                         from wiping our bind mounts.
 833                         """
 834                         raise CatalystError,"Couldn't umount one or more bind-mounts; aborting for safety."
 835
 836         def chroot_setup(self):
 837                 self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/make.conf")
 838                 self.override_chost()
 839                 self.override_cflags()
 840                 self.override_cxxflags()
 841                 self.override_ldflags()
 842                 if self.settings.has_key("AUTORESUME") \
 843                         and os.path.exists(self.settings["autoresume_path"]+"chroot_setup"):
 844                         print "Resume point detected, skipping chroot_setup operation..."
 845                 else:
 846                         print "Setting up chroot..."
 847
 848                         #self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/make.conf")
 849
 850                         cmd("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc",\
 851                                 "Could not copy resolv.conf into place.",env=self.env)
 852
 853                     # copy over the envscript, if applicable
 854                         if self.settings.has_key("ENVSCRIPT"):
 855                                 if not os.path.exists(self.settings["ENVSCRIPT"]):
 856                                         raise CatalystError, "Can't find envscript "+self.settings["ENVSCRIPT"]
 857
 858                                 print "\nWarning!!!!"
 859                                 print "\tOverriding certain env variables may cause catastrophic failure."
 860                                 print "\tIf your build fails look here first as the possible problem."
 861                                 print "\tCatalyst assumes you know what you are doing when setting"
 862                                 print "\t\tthese variables."
 863                                 print "\tCatalyst Maintainers use VERY minimal envscripts if used at all"
 864                                 print "\tYou have been warned\n"
 865
 866                                 cmd("cp "+self.settings["ENVSCRIPT"]+" "+self.settings["chroot_path"]+"/tmp/envscript",\
 867                                         "Could not copy envscript into place.",env=self.env)
 868
 869                     # copy over /etc/hosts from the host in case there are any specialties in there
 870                         if os.path.exists(self.settings["chroot_path"]+"/etc/hosts"):
 871                                 cmd("mv "+self.settings["chroot_path"]+"/etc/hosts "+self.settings["chroot_path"]+\
 872                                         "/etc/hosts.bck", "Could not backup /etc/hosts",env=self.env)
 873                                 cmd("cp /etc/hosts "+self.settings["chroot_path"]+"/etc/hosts", "Could not copy /etc/hosts",env=self.env)
 874                         #self.override_chost()
 875                         #self.override_cflags()
 876                         #self.override_cxxflags()
 877                         #self.override_ldflags()
 878                         # modify and write out make.conf (for the chroot)
 879                         cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.conf","Could not remove "+self.settings["chroot_path"]+"/etc/make.conf",\
 880                                 env=self.env)
 881                         myf=open(self.settings["chroot_path"]+"/etc/make.conf","w")
 882                         myf.write("# These settings were set by the catalyst build script that automatically built this stage\n")
 883                         myf.write("# Please consult /etc/make.conf.example for a more detailed example\n")
 884                         myf.write('CFLAGS="'+self.settings["CFLAGS"]+'"\n')
 885                         myf.write('CHOST="'+self.settings["CHOST"]+'"\n')
 886
 887                         if self.settings.has_key("CXXFLAGS"):
 888                                 myf.write('CXXFLAGS="'+self.settings["CXXFLAGS"]+'"\n')
 889                         else:
 890                                 myf.write('CXXFLAGS="${CFLAGS}"\n')
 891
 892                         if self.settings.has_key("LDFLAGS"):
 893                                 myf.write('LDFLAGS="'+self.settings["LDFLAGS"]+'"\n')
 894
 895                     # figure out what our USE vars are for building
 896                         myusevars=[]
 897                         if self.settings.has_key("HOSTUSE"):
 898                                 myusevars.extend(self.settings["HOSTUSE"])
 899
 900                         if self.settings.has_key("use"):
 901                                 myusevars.extend(self.settings["use"])
 902                                 myf.write('USE="'+string.join(myusevars)+'"\n')
 903
 904                     # setup the portage overlay
 905                         if self.settings.has_key("portage_overlay"):
 906                                 myf.write('PORTDIR_OVERLAY="'+string.join(self.settings["portage_overlay"])+'"\n')
 907
 908                         myf.close()
 909                         touch(self.settings["autoresume_path"]+"chroot_setup")
 910
 911         def fsscript(self):
 912                 if self.settings.has_key("AUTORESUME") \
 913                         and os.path.exists(self.settings["autoresume_path"]+"fsscript"):
 914                         print "Resume point detected, skipping fsscript operation..."
 915                 else:
 916                     if self.settings.has_key("fsscript"):
 917                         if os.path.exists(self.settings["controller_file"]):
 918                             cmd("/bin/bash "+self.settings["controller_file"]+" fsscript","fsscript script failed.",env=self.env)
 919                         touch(self.settings["autoresume_path"]+"fsscript")
 920
 921         def rcupdate(self):
 922                 if self.settings.has_key("AUTORESUME") \
 923                         and os.path.exists(self.settings["autoresume_path"]+"rcupdate"):
 924                         print "Resume point detected, skipping rcupdate operation..."
 925                 else:
 926                     if os.path.exists(self.settings["controller_file"]):
 927                         cmd("/bin/bash "+self.settings["controller_file"]+" rc-update","rc-update script failed.",env=self.env)
 928                         touch(self.settings["autoresume_path"]+"rcupdate")
 929
 930         def clean(self):
 931                 if self.settings.has_key("AUTORESUME") \
 932                         and os.path.exists(self.settings["autoresume_path"]+"clean"):
 933                         print "Resume point detected, skipping clean operation..."
 934                 else:
 935                     for x in self.settings["cleanables"]:
 936                             print "Cleaning chroot: "+x+"... "
 937                             cmd("rm -rf "+self.settings["destpath"]+x,"Couldn't clean "+x,env=self.env)
 938
 939                     # put /etc/hosts back into place
 940                     if os.path.exists(self.settings["chroot_path"]+"/etc/hosts.bck"):
 941                             cmd("mv -f "+self.settings["chroot_path"]+"/etc/hosts.bck "+self.settings["chroot_path"]+\
 942                                           "/etc/hosts", "Could not replace /etc/hosts",env=self.env)
 943
 944                     if os.path.exists(self.settings["controller_file"]):
 945                         cmd("/bin/bash "+self.settings["controller_file"]+" clean","clean script failed.",env=self.env)
 946                         touch(self.settings["autoresume_path"]+"clean")
 947
 948         def empty(self):
 949             if self.settings.has_key("AUTORESUME") \
 950                 and os.path.exists(self.settings["autoresume_path"]+"empty"):
 951                 print "Resume point detected, skipping empty operation..."
 952             else:
 953                 if self.settings.has_key(self.settings["spec_prefix"]+"/empty"):
 954                     if type(self.settings[self.settings["spec_prefix"]+"/empty"])==types.StringType:
 955                         self.settings[self.settings["spec_prefix"]+"/empty"]=self.settings[self.settings["spec_prefix"]+"/empty"].split()
 956                     for x in self.settings[self.settings["spec_prefix"]+"/empty"]:
 957                         myemp=self.settings["destpath"]+x
 958                         if not os.path.isdir(myemp):
 959                             print x,"not a directory or does not exist, skipping 'empty' operation."
 960                             continue
 961                         print "Emptying directory",x
 962                         # stat the dir, delete the dir, recreate the dir and set
 963                         # the proper perms and ownership
 964                         mystat=os.stat(myemp)
 965                         shutil.rmtree(myemp)
 966                         os.makedirs(myemp,0755)
 967                         os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 968                         os.chmod(myemp,mystat[ST_MODE])
 969                     touch(self.settings["autoresume_path"]+"empty")
 970
 971         def remove(self):
 972             if self.settings.has_key("AUTORESUME") \
 973                 and os.path.exists(self.settings["autoresume_path"]+"remove"):
 974                 print "Resume point detected, skipping remove operation..."
 975             else:
 976                 if self.settings.has_key(self.settings["spec_prefix"]+"/rm"):
 977                     for x in self.settings[self.settings["spec_prefix"]+"/rm"]:
 978                         # we're going to shell out for all these cleaning operations,
 979                         # so we get easy glob handling
 980                         print "livecd: removing "+x
 981                         os.system("rm -rf "+self.settings["chroot_path"]+x)
 982                 try:
 983                     if os.path.exists(self.settings["controller_file"]):
 984                             cmd("/bin/bash "+self.settings["controller_file"]+" clean",\
 985                                 "Clean  failed.",env=self.env)
 986                             touch(self.settings["autoresume_path"]+"remove")
 987                 except:
 988                     self.unbind()
 989                     raise
 990
 991
 992         def preclean(self):
 993             if self.settings.has_key("AUTORESUME") \
 994                 and os.path.exists(self.settings["autoresume_path"]+"preclean"):
 995                 print "Resume point detected, skipping preclean operation..."
 996             else:
 997                 try:
 998                         if os.path.exists(self.settings["controller_file"]):
 999                                 cmd("/bin/bash "+self.settings["controller_file"]+" preclean","preclean script failed.",env=self.env)
1000                                 touch(self.settings["autoresume_path"]+"preclean")
1001
1002                 except:
1003                         self.unbind()
1004                         raise CatalystError, "Build failed, could not execute preclean"
1005
1006         def capture(self):
1007             if self.settings.has_key("AUTORESUME") \
1008                 and os.path.exists(self.settings["autoresume_path"]+"capture"):
1009                 print "Resume point detected, skipping capture operation..."
1010             else:
1011                 """capture target in a tarball"""
1012                 mypath=self.settings["target_path"].split("/")
1013                 # remove filename from path
1014                 mypath=string.join(mypath[:-1],"/")
1015
1016                 # now make sure path exists
1017                 if not os.path.exists(mypath):
1018                         os.makedirs(mypath)
1019
1020                 print "Creating stage tarball..."
1021
1022                 cmd("tar cjpf "+self.settings["target_path"]+" -C "+self.settings["stage_path"]+\
1023                         " .","Couldn't create stage tarball",env=self.env)
1024
1025                 self.gen_digest_file(self.settings["target_path"])
1026
1027                 touch(self.settings["autoresume_path"]+"capture")
1028
1029         def run_local(self):
1030             if self.settings.has_key("AUTORESUME") \
1031                 and os.path.exists(self.settings["autoresume_path"]+"run_local"):
1032                 print "Resume point detected, skipping run_local operation..."
1033             else:
1034                 try:
1035                         if os.path.exists(self.settings["controller_file"]):
1036                                 cmd("/bin/bash "+self.settings["controller_file"]+" run","run script failed.",env=self.env)
1037                                 touch(self.settings["autoresume_path"]+"run_local")
1038
1039                 except CatalystError:
1040                         self.unbind()
1041                         raise CatalystError,"Stage build aborting due to error."
1042
1043         def setup_environment(self):
1044                 # modify the current environment. This is an ugly hack that should be fixed. We need this
1045                 # to use the os.system() call since we can't specify our own environ:
1046                 for x in self.settings.keys():
1047                         # "/" is replaced with "_", "-" is also replaced with "_", "." is also replace with "_"
1048                         varname="clst_"+string.replace(x,"/","_")
1049                         varname=string.replace(varname,"-","_")
1050                         varname=string.replace(varname,".","_")
1051                         if type(self.settings[x])==types.StringType:
1052                                 # prefix to prevent namespace clashes:
1053                                 #os.environ[varname]=self.settings[x]
1054                                 self.env[varname]=self.settings[x]
1055                         elif type(self.settings[x])==types.ListType:
1056                                 #os.environ[varname]=string.join(self.settings[x])
1057                                 self.env[varname]=string.join(self.settings[x])
1058                 if self.settings.has_key("makeopts"):
1059                         self.env["MAKEOPTS"]=self.settings["makeopts"]
1060
1061         def run(self):
1062                 self.chroot_lock.write_lock()
1063
1064                 # Kill any pids in the chroot
1065                 self.kill_chroot_pids()
1066
1067                 # Check for mounts right away and abort if we cannot unmount them.
1068                 self.mount_safety_check()
1069
1070                 if self.settings.has_key("CLEAR_AUTORESUME"):
1071                         self.clear_autoresume()
1072                 if self.settings.has_key("PURGE"):
1073                         self.purge()
1074
1075                 for x in self.settings["action_sequence"]:
1076                         print "Running action sequence: "+x
1077                         sys.stdout.flush()
1078                         try:
1079                                 apply(getattr(self,x))
1080                         except:
1081                                 self.mount_safety_check()
1082                                 raise
1083
1084                 self.chroot_lock.unlock()
1085
1086         def unmerge(self):
1087             if self.settings.has_key("AUTORESUME") \
1088                 and os.path.exists(self.settings["autoresume_path"]+"unmerge"):
1089                     print "Resume point detected, skipping unmerge operation..."
1090             else:
1091                 if self.settings.has_key(self.settings["spec_prefix"]+"/unmerge"):
1092                     if type(self.settings[self.settings["spec_prefix"]+"/unmerge"])==types.StringType:
1093                         self.settings[self.settings["spec_prefix"]+"/unmerge"]=[self.settings[self.settings["spec_prefix"]+"/unmerge"]]
1094                     myunmerge=self.settings[self.settings["spec_prefix"]+"/unmerge"][:]
1095
1096                     for x in range(0,len(myunmerge)):
1097                     #surround args with quotes for passing to bash,
1098                     #allows things like "<" to remain intact
1099                         myunmerge[x]="'"+myunmerge[x]+"'"
1100                     myunmerge=string.join(myunmerge)
1101
1102                     #before cleaning, unmerge stuff:
1103                     try:
1104                         cmd("/bin/bash "+self.settings["controller_file"]+" unmerge "+ myunmerge,\
1105                                 "Unmerge script failed.",env=self.env)
1106                         #cmd("/bin/bash "+self.settings["sharedir"]+"/targets/" \
1107                         #       +self.settings["target"]+"/unmerge.sh "+myunmerge,"Unmerge script failed.",env=self.env)
1108                         print "unmerge shell script"
1109                     except CatalystError:
1110                         self.unbind()
1111                         raise
1112
1113                 touch(self.settings["autoresume_path"]+"unmerge")
1114
1115         def target_setup(self):
1116             if self.settings.has_key("AUTORESUME") \
1117                 and os.path.exists(self.settings["autoresume_path"]+"target_setup"):
1118                     print "Resume point detected, skipping target_setup operation..."
1119             else:
1120                 print "Setting up filesystems per filesystem type"
1121                 cmd("/bin/bash "+self.settings["controller_file"]+" target_image_setup "+ self.settings["target_path"],\
1122                                 "target_image_setup script failed.",env=self.env)
1123                 touch(self.settings["autoresume_path"]+"target_setup")
1124
1125         def setup_overlay(self):
1126                 if self.settings.has_key("AUTORESUME") \
1127                 and os.path.exists(self.settings["autoresume_path"]+"setup_overlay"):
1128                         print "Resume point detected, skipping setup_overlay operation..."
1129                 else:
1130                         if self.settings.has_key(self.settings["spec_prefix"]+"/overlay"):
1131                                 for x in self.settings[self.settings["spec_prefix"]+"/overlay"]:
1132                                         if os.path.exists(x):
1133                                                 cmd("rsync -a "+x+"/ "+\
1134                                                         self.settings["target_path"], self.settings["spec_prefix"]+"overlay: "+x+" copy failed.",env=self.env)
1135                                 touch(self.settings["autoresume_path"]+"setup_overlay")
1136
1137         def create_iso(self):
1138             if self.settings.has_key("AUTORESUME") \
1139                 and os.path.exists(self.settings["autoresume_path"]+"create_iso"):
1140                     print "Resume point detected, skipping create_iso operation..."
1141             else:
1142                 # create the ISO - this is the preferred method (the iso scripts do not always work)
1143                 if self.settings.has_key("iso"):
1144                         cmd("/bin/bash "+self.settings["controller_file"]+" iso "+\
1145                                 self.settings["iso"],"ISO creation script failed.",env=self.env)
1146                         self.gen_digest_file(self.settings["iso"])
1147                         touch(self.settings["autoresume_path"]+"create_iso")
1148
1149
1150                 else:
1151                         print "WARNING livecd/iso was not defined."
1152                         print "A CD Image will not be created, skipping create-iso.sh..."
1153
1154
1155         def build_packages(self):
1156             if self.settings.has_key("AUTORESUME") \
1157                 and os.path.exists(self.settings["autoresume_path"]+"build_packages"):
1158                     print "Resume point detected, skipping build_packages operation..."
1159             else:
1160                 if self.settings.has_key(self.settings["spec_prefix"]+"/packages"):
1161                         if self.settings.has_key("AUTORESUME") \
1162                                 and os.path.exists(self.settings["autoresume_path"]+"build_packages"):
1163                                         print "Resume point detected, skipping build_packages operation..."
1164                         else:
1165                                 mypack=list_bashify(self.settings[self.settings["spec_prefix"]+"/packages"])
1166                                 try:
1167                                         cmd("/bin/bash "+self.settings["controller_file"]+\
1168                                                 " build_packages "+mypack,"Error in attempt to build packages",env=self.env)
1169                                         touch(self.settings["autoresume_path"]+"build_packages")
1170                                 except CatalystError:
1171                                         self.unbind()
1172                                         raise CatalystError,self.settings["spec_prefix"] + "build aborting due to error."
1173
1174         def build_kernel(self):
1175                 if self.settings.has_key("AUTORESUME") \
1176                 and os.path.exists(self.settings["autoresume_path"]+"build_kernel"):
1177                         print "Resume point detected, skipping build_kernel operation..."
1178                 else:
1179                         if self.settings.has_key("boot/kernel"):
1180                                 try:
1181                                         mynames=self.settings["boot/kernel"]
1182                                         if type(mynames)==types.StringType:
1183                                                 mynames=[mynames]
1184                                         # execute the script that sets up the kernel build environment
1185                                         cmd("/bin/bash "+self.settings["controller_file"]+" pre-kmerge ",\
1186                                                 "Runscript pre-kmerge failed",env=self.env)
1187
1188                                         for kname in mynames:
1189                                                 if self.settings.has_key("AUTORESUME") \
1190                                                         and os.path.exists(self.settings["autoresume_path"]+"build_kernel_"+kname):
1191                                                         print "Resume point detected, skipping build_kernel for "+kname+" operation..."
1192                                                 else:
1193                                                         try:
1194                                                                 if not os.path.exists(self.settings["boot/kernel/"+kname+"/config"]):
1195                                                                         self.unbind()
1196                                                                         raise CatalystError, "Can't find kernel config: " \
1197                                                                                 +self.settings["boot/kernel/"+kname+"/config"]
1198
1199                                                         except TypeError:
1200                                                                 raise CatalystError, "Required value boot/kernel/config not specified"
1201
1202                                                         try:
1203                                                                 cmd("cp "+self.settings["boot/kernel/"+kname+"/config"]+" "+ \
1204                                                                         self.settings["chroot_path"]+"/var/tmp/"+kname+".config", \
1205                                                                         "Couldn't copy kernel config: "+self.settings["boot/kernel/"+kname+"/config"],\
1206                                                                         env=self.env)
1207
1208                                                         except CatalystError:
1209                                                                 self.unbind()
1210
1211                                                         # If we need to pass special options to the bootloader
1212                                                         # for this kernel put them into the environment.
1213                                                         if self.settings.has_key("boot/kernel/"+kname+"/kernelopts"):
1214                                                                 myopts=self.settings["boot/kernel/"+kname+"/kernelopts"]
1215
1216                                                                 if type(myopts) != types.StringType:
1217                                                                         myopts = string.join(myopts)
1218                                                                         self.env[kname+"_kernelopts"]=myopts
1219
1220                                                                 else:
1221                                                                         self.env[kname+"_kernelopts"]=""
1222
1223                                                         if not self.settings.has_key("boot/kernel/"+kname+"/extraversion"):
1224                                                                 self.settings["boot/kernel/"+kname+"/extraversion"]=""
1225
1226                                                         self.env["clst_kextraversion"]=self.settings["boot/kernel/"+kname+"/extraversion"]
1227
1228                                                         if self.settings.has_key("boot/kernel/"+kname+"/initramfs_overlay"):
1229                                                                 if os.path.exists(self.settings["boot/kernel/"+kname+"/initramfs_overlay"]):
1230                                                                         print "Copying initramfs_overlay dir " +self.settings["boot/kernel/"+kname+"/initramfs_overlay"]
1231
1232                                                                         cmd("mkdir -p "+self.settings["chroot_path"]+"/tmp/initramfs_overlay/" + \
1233                                                                                 self.settings["boot/kernel/"+kname+"/initramfs_overlay"],env=self.env)
1234
1235                                                                         cmd("cp -R "+self.settings["boot/kernel/"+kname+"/initramfs_overlay"]+"/* " + \
1236                                                                                 self.settings["chroot_path"] + "/tmp/initramfs_overlay/" + \
1237                                                                                 self.settings["boot/kernel/"+kname+"/initramfs_overlay"],\
1238                                                                                 env=self.env)
1239
1240
1241                                                         # execute the script that builds the kernel
1242                                                         cmd("/bin/bash "+self.settings["controller_file"]+" kernel "+kname,\
1243                                                                 "Runscript kernel build failed",env=self.env)
1244
1245                                                         if self.settings.has_key("boot/kernel/"+kname+"/initramfs_overlay"):
1246                                                                 if os.path.exists(self.settings["chroot_path"]+"/tmp/initramfs_overlay/"):
1247                                                                         print "Cleaning up temporary overlay dir"
1248                                                                         cmd("rm -R "+self.settings["chroot_path"]+"/tmp/initramfs_overlay/",env=self.env)
1249
1250                                                         touch(self.settings["autoresume_path"]+"build_kernel_"+kname)
1251
1252                                                         # execute the script that cleans up the kernel build environment
1253                                                         cmd("/bin/bash "+self.settings["controller_file"]+" post-kmerge ",\
1254                                                                 "Runscript post-kmerge failed",env=self.env)
1255
1256                                         touch(self.settings["autoresume_path"]+"build_kernel")
1257
1258                                 except CatalystError:
1259                                         self.unbind()
1260                                         raise CatalystError,"build aborting due to kernel build error."
1261
1262         def bootloader(self):
1263             if self.settings.has_key("AUTORESUME") \
1264                 and os.path.exists(self.settings["autoresume_path"]+"bootloader"):
1265                     print "Resume point detected, skipping bootloader operation..."
1266             else:
1267                 try:
1268                         cmd("/bin/bash "+self.settings["controller_file"]+" bootloader " + self.settings["target_path"],\
1269                                 "Bootloader runscript failed.",env=self.env)
1270                         touch(self.settings["autoresume_path"]+"bootloader")
1271                 except CatalystError:
1272                         self.unbind()
1273                         raise CatalystError,"Runscript aborting due to error."
1274
1275         def livecd_update(self):
1276             if self.settings.has_key("AUTORESUME") \
1277                 and os.path.exists(self.settings["autoresume_path"]+"livecd_update"):
1278                     print "Resume point detected, skipping build_packages operation..."
1279             else:
1280                 try:
1281                         cmd("/bin/bash "+self.settings["controller_file"]+" livecd-update",\
1282                                 "livecd-update failed.",env=self.env)
1283                         touch(self.settings["autoresume_path"]+"livecd_update")
1284
1285                 except CatalystError:
1286                         self.unbind()
1287                         raise CatalystError,"build aborting due to livecd_update error."
1288
1289         def clear_chroot(self):
1290                 myemp=self.settings["chroot_path"]
1291                 if os.path.isdir(myemp):
1292                     print "Emptying directory",myemp
1293                     # stat the dir, delete the dir, recreate the dir and set
1294                     # the proper perms and ownership
1295                     mystat=os.stat(myemp)
1296                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1297                     shutil.rmtree(myemp)
1298                     os.makedirs(myemp,0755)
1299                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1300                     os.chmod(myemp,mystat[ST_MODE])
1301
1302         def clear_packages(self):
1303             if self.settings.has_key("PKGCACHE"):
1304                 print "purging the pkgcache ..."
1305
1306                 myemp=self.settings["pkgcache_path"]
1307                 if os.path.isdir(myemp):
1308                     print "Emptying directory",myemp
1309                     # stat the dir, delete the dir, recreate the dir and set
1310                     # the proper perms and ownership
1311                     mystat=os.stat(myemp)
1312                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1313                     shutil.rmtree(myemp)
1314                     os.makedirs(myemp,0755)
1315                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1316                     os.chmod(myemp,mystat[ST_MODE])
1317
1318         def clear_kerncache(self):
1319             if self.settings.has_key("KERNCACHE"):
1320                 print "purging the kerncache ..."
1321
1322                 myemp=self.settings["kerncache_path"]
1323                 if os.path.isdir(myemp):
1324                     print "Emptying directory",myemp
1325                     # stat the dir, delete the dir, recreate the dir and set
1326                     # the proper perms and ownership
1327                     mystat=os.stat(myemp)
1328                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1329                     shutil.rmtree(myemp)
1330                     os.makedirs(myemp,0755)
1331                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1332                     os.chmod(myemp,mystat[ST_MODE])
1333
1334         def clear_autoresume(self):
1335                 # clean resume points since they are no longer needed
1336                 if self.settings.has_key("AUTORESUME"):
1337                         print "Removing AutoResume Points: ..."
1338                 myemp=self.settings["autoresume_path"]
1339                 if os.path.isdir(myemp):
1340                                 if self.settings.has_key("AUTORESUME"):
1341                                         print "Emptying directory",myemp
1342                                 # stat the dir, delete the dir, recreate the dir and set
1343                                 # the proper perms and ownership
1344                                 mystat=os.stat(myemp)
1345                                 #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
1346                                 shutil.rmtree(myemp)
1347                                 os.makedirs(myemp,0755)
1348                                 os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1349                                 os.chmod(myemp,mystat[ST_MODE])
1350
1351         def gen_digest_file(self,file):
1352                 if os.path.exists(file+".DIGESTS"):
1353                         os.remove(file+".DIGESTS")
1354                 if self.settings.has_key("digests"):
1355                         if os.path.exists(file):
1356                                 myf=open(file+".DIGESTS","w")
1357                                 keys={}
1358                                 for i in self.settings["digests"].split():
1359                                         keys[i]=1
1360                                         array=keys.keys()
1361                                         array.sort()
1362                                         for i in array:
1363                                                 if self.settings.has_key("VERBOSE"):
1364                                                         hash=generate_hash(file,hash_function=i,verbose=True)
1365                                                 else:
1366                                                         hash=generate_hash(file,hash_function=i)
1367                                                 myf.write(hash)
1368                                 myf.close()
1369
1370         def purge(self):
1371             countdown(10,"Purging Caches ...")
1372             if self.settings.has_key("PURGE"):
1373                 print "clearing autoresume ..."
1374                 self.clear_autoresume()
1375
1376                 print "clearing chroot ..."
1377                 self.clear_chroot()
1378
1379                 print "clearing package cache ..."
1380                 self.clear_packages()
1381
1382                 print "clearing kerncache ..."
1383                 self.clear_kerncache()
1384
1385 #vim: ts=4 sw=4 sta et sts=4 ai