modules/generic_stage_target.py

   1
   2 """
   3 This class does all of the chroot setup, copying of files, etc. It is
   4 the driver class for pretty much everything that Catalyst does.
   5 """
   6
   7 import os,string,imp,types,shutil
   8 from catalyst_support import *
   9 from generic_target import *
  10 from stat import *
  11 import catalyst_lock
  12 class generic_stage_target(generic_target):
  13
  14         def __init__(self,myspec,addlargs):
  15                 self.required_values.extend(["version_stamp","target","subarch",\
  16                         "rel_type","profile","snapshot","source_subpath"])
  17
  18                 self.valid_values.extend(["version_stamp","target","subarch",\
  19                         "rel_type","profile","snapshot","source_subpath","portage_confdir",\
  20                         "cflags","cxxflags","ldflags","cbuild","hostuse","portage_overlay",\
  21                         "distcc_hosts","makeopts","pkgcache_path","kerncache_path"])
  22
  23                 self.set_valid_build_kernel_vars(addlargs)
  24                 generic_target.__init__(self,myspec,addlargs)
  25
  26                 # The semantics of subarchmap and machinemap changed a bit in 2.0.3 to
  27                 # work better with vapier's CBUILD stuff. I've removed the "monolithic"
  28                 # machinemap from this file and split up its contents amongst the various
  29                 # arch/foo.py files.
  30                 #
  31                 # When register() is called on each module in the arch/ dir, it now
  32                 # returns a tuple instead of acting on the subarchmap dict that is
  33                 # passed to it. The tuple contains the values that were previously
  34                 # added to subarchmap as well as a new list of CHOSTs that go along
  35                 # with that arch. This allows us to build machinemap on the fly based
  36                 # on the keys in subarchmap and the values of the 2nd list returned
  37                 # (tmpmachinemap).
  38                 #
  39                 # Also, after talking with vapier. I have a slightly better idea of what
  40                 # certain variables are used for and what they should be set to. Neither
  41                 # 'buildarch' or 'hostarch' are used directly, so their value doesn't
  42                 # really matter. They are just compared to determine if we are
  43                 # cross-compiling. Because of this, they are just set to the name of the
  44                 # module in arch/ that the subarch is part of to make things simpler.
  45                 # The entire build process is still based off of 'subarch' like it was
  46                 # previously. -agaffney
  47
  48                 self.archmap = {}
  49                 self.subarchmap = {}
  50                 machinemap = {}
  51                 for x in [x[:-3] for x in os.listdir(self.settings["sharedir"]+"/arch/") if x.endswith(".py")]:
  52                         try:
  53                                 fh=open(self.settings["sharedir"]+"/arch/"+x+".py")
  54                                 # This next line loads the plugin as a module and assigns it to
  55                                 # archmap[x]
  56                                 self.archmap[x]=imp.load_module(x,fh,"arch/"+x+".py",(".py","r",imp.PY_SOURCE))
  57                                 # This next line registers all the subarches supported in the
  58                                 # plugin
  59                                 tmpsubarchmap, tmpmachinemap = self.archmap[x].register()
  60                                 self.subarchmap.update(tmpsubarchmap)
  61                                 for machine in tmpmachinemap:
  62                                         machinemap[machine] = x
  63                                 for subarch in tmpsubarchmap:
  64                                         machinemap[subarch] = x
  65                                 fh.close()
  66                         except IOError:
  67                                 # This message should probably change a bit, since everything in the dir should
  68                                 # load just fine. If it doesn't, it's probably a syntax error in the module
  69                                 msg("Can't find "+x+".py plugin in "+self.settings["sharedir"]+"/arch/")
  70
  71                 if self.settings.has_key("chost"):
  72                         hostmachine = self.settings["chost"].split("-")[0]
  73                         if not machinemap.has_key(hostmachine):
  74                                 raise CatalystError, "Unknown host machine type "+hostmachine
  75                         self.settings["hostarch"] = machinemap[hostmachine]
  76                 else:
  77                         hostmachine = self.settings["subarch"]
  78                         if machinemap.has_key(hostmachine):
  79                                 hostmachine = machinemap[hostmachine]
  80                         self.settings["hostarch"] = hostmachine
  81                 if self.settings.has_key("cbuild"):
  82                         buildmachine = self.settings["cbuild"].split("-")[0]
  83                 else:
  84                         buildmachine = os.uname()[4]
  85                 if not machinemap.has_key(buildmachine):
  86                         raise CatalystError, "Unknown build machine type "+buildmachine
  87                 self.settings["buildarch"] = machinemap[buildmachine]
  88                 self.settings["crosscompile"] = (self.settings["hostarch"] != self.settings["buildarch"])
  89
  90                 # Call arch constructor, pass our settings
  91                 try:
  92                         self.arch=self.subarchmap[self.settings["subarch"]](self.settings)
  93                 except KeyError:
  94                         print "Invalid subarch: "+self.settings["subarch"]
  95                         print "Choose one of the following:",
  96                         for x in self.subarchmap:
  97                                 print x,
  98                         print
  99                         sys.exit(2)
 100
 101                 print "Using target:",self.settings["target"]
 102                 # print a nice informational message:
 103                 if self.settings["buildarch"]==self.settings["hostarch"]:
 104                         print "Building natively for",self.settings["hostarch"]
 105                 elif self.settings["crosscompile"]:
 106                         print "Cross-compiling on",self.settings["buildarch"],"for different machine type",\
 107                                 self.settings["hostarch"]
 108                 else:
 109                         print "Building on",self.settings["buildarch"],"for alternate personality type",\
 110                                 self.settings["hostarch"]
 111
 112                 # This should be first to be set as other set_ options depend on this
 113                 self.set_spec_prefix()
 114
 115                 # define all of our core variables
 116                 self.set_target_profile()
 117                 self.set_target_subpath()
 118                 self.set_source_subpath()
 119
 120                 # set paths
 121                 self.set_snapshot_path()
 122                 self.set_root_path()
 123                 self.set_source_path()
 124                 self.set_snapcache_path()
 125                 self.set_chroot_path()
 126                 self.set_autoresume_path()
 127                 self.set_dest_path()
 128                 self.set_stage_path()
 129                 self.set_target_path()
 130
 131                 self.set_controller_file()
 132                 self.set_action_sequence()
 133                 self.set_use()
 134                 self.set_cleanables()
 135                 self.set_iso_volume_id()
 136                 self.set_build_kernel_vars()
 137                 self.set_fsscript()
 138                 self.set_archscript()
 139                 self.set_runscript()
 140                 self.set_install_mask()
 141                 self.set_rcadd()
 142                 self.set_rcdel()
 143                 self.set_cdtar()
 144                 self.set_fstype()
 145                 self.set_fsops()
 146                 self.set_iso()
 147                 self.set_packages()
 148                 self.set_rm()
 149                 self.set_linuxrc()
 150                 self.set_overlay()
 151                 self.set_portage_overlay()
 152                 self.set_root_overlay()
 153
 154                 # This next line checks to make sure that the specified variables exist
 155                 # on disk.
 156                 #pdb.set_trace()
 157                 file_locate(self.settings,["source_path","snapshot_path","distdir"],expand=0)
 158                 # If we are using portage_confdir, check that as well.
 159                 if self.settings.has_key("portage_confdir"):
 160                         file_locate(self.settings,["portage_confdir"],expand=0)
 161
 162                 # setup our mount points
 163                 if self.settings.has_key("SNAPCACHE"):
 164                         self.mounts=[ "/proc","/dev","/usr/portage","/usr/portage/distfiles" ]
 165                         self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
 166                                 "/usr/portage":self.settings["snapshot_cache_path"]+"/portage",\
 167                                 "/usr/portage/distfiles":self.settings["distdir"]}
 168                 else:
 169                         self.mounts=[ "/proc","/dev","/usr/portage/distfiles" ]
 170                         self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
 171                                 "/usr/portage/distfiles":self.settings["distdir"]}
 172                 if os.uname()[0] == "Linux":
 173                         self.mounts.append("/dev/pts")
 174
 175                 self.set_mounts()
 176
 177                 # Configure any user specified options (either in catalyst.conf or on
 178                 # the command line).
 179                 if self.settings.has_key("PKGCACHE"):
 180                         self.set_pkgcache_path()
 181                         print "Location of the package cache is " + self.settings["pkgcache_path"]
 182                         self.mounts.append("/usr/portage/packages")
 183                         self.mountmap["/usr/portage/packages"]=self.settings["pkgcache_path"]
 184
 185                 if self.settings.has_key("KERNCACHE"):
 186                         self.set_kerncache_path()
 187                         print "Location of the kerncache is " + self.settings["kerncache_path"]
 188                         self.mounts.append("/tmp/kerncache")
 189                         self.mountmap["/tmp/kerncache"]=self.settings["kerncache_path"]
 190
 191                 if self.settings.has_key("CCACHE"):
 192                         if os.environ.has_key("CCACHE_DIR"):
 193                                 ccdir=os.environ["CCACHE_DIR"]
 194                                 del os.environ["CCACHE_DIR"]
 195                         else:
 196                                 ccdir="/root/.ccache"
 197                         if not os.path.isdir(ccdir):
 198                                         raise CatalystError,\
 199                                                 "Compiler cache support can't be enabled (can't find "+ccdir+")"
 200                         self.mounts.append("/var/tmp/ccache")
 201                         self.mountmap["/var/tmp/ccache"]=ccdir
 202                         # for the chroot:
 203                         self.env["CCACHE_DIR"]="/var/tmp/ccache"
 204
 205         def override_cbuild(self):
 206                 if self.makeconf.has_key("CBUILD"):
 207                         self.settings["CBUILD"]=self.makeconf["CBUILD"]
 208
 209         def override_chost(self):
 210                 if self.makeconf.has_key("CHOST"):
 211                         self.settings["CHOST"]=self.makeconf["CHOST"]
 212
 213         def override_cflags(self):
 214                 if self.makeconf.has_key("CFLAGS"):
 215                         self.settings["CFLAGS"]=self.makeconf["CFLAGS"]
 216
 217         def override_cxxflags(self):
 218                 if self.makeconf.has_key("CXXFLAGS"):
 219                         self.settings["CXXFLAGS"]=self.makeconf["CXXFLAGS"]
 220
 221         def override_ldflags(self):
 222                 if self.makeconf.has_key("LDFLAGS"):
 223                         self.settings["LDFLAGS"]=self.makeconf["LDFLAGS"]
 224
 225         def set_install_mask(self):
 226                 if self.settings.has_key("install_mask"):
 227                         if type(self.settings["install_mask"]) != types.StringType:
 228                                 self.settings["install_mask"]=string.join(self.settings["install_mask"])
 229
 230         def set_spec_prefix(self):
 231                 self.settings["spec_prefix"]=self.settings["target"]
 232
 233         def set_target_profile(self):
 234                 self.settings["target_profile"]=self.settings["profile"]
 235
 236         def set_target_subpath(self):
 237                 self.settings["target_subpath"]=self.settings["rel_type"]+"/"+self.settings["target"]+\
 238                         "-"+self.settings["subarch"]+"-"+self.settings["version_stamp"]
 239
 240         def set_source_subpath(self):
 241                 if type(self.settings["source_subpath"]) != types.StringType:
 242                         raise CatalystError, "source_subpath should have been a string. Perhaps you have something wrong in your spec file?"
 243
 244         def set_pkgcache_path(self):
 245                 if self.settings.has_key("pkgcache_path"):
 246                         if type(self.settings["pkgcache_path"]) != types.StringType:
 247                                 self.settings["pkgcache_path"]=normpath(string.join(self.settings["pkgcache_path"]))
 248                 else:
 249                         self.settings["pkgcache_path"]=normpath(self.settings["storedir"]+"/packages/"+\
 250                                 self.settings["target_subpath"]+"/")
 251
 252         def set_kerncache_path(self):
 253                 if self.settings.has_key("kerncache_path"):
 254                         if type(self.settings["kerncache_path"]) != types.StringType:
 255                                 self.settings["kerncache_path"]=normpath(string.join(self.settings["kerncache_path"]))
 256                 else:
 257                         self.settings["kerncache_path"]=normpath(self.settings["storedir"]+"/kerncache/"+\
 258                                 self.settings["target_subpath"]+"/")
 259
 260         def set_target_path(self):
 261                 self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\
 262                         self.settings["target_subpath"]+".tar.bz2")
 263                 if self.settings.has_key("AUTORESUME") \
 264                         and os.path.exists(self.settings["autoresume_path"]+"setup_target_path"):
 265                         print "Resume point detected, skipping target path setup operation..."
 266                 else:
 267                         # first clean up any existing target stuff
 268                         if os.path.isfile(self.settings["target_path"]):
 269                                 cmd("rm -f "+self.settings["target_path"], \
 270                                 "Could not remove existing file: "+self.settings["target_path"],env=self.env)
 271                                 touch(self.settings["autoresume_path"]+"setup_target_path")
 272
 273                         if not os.path.exists(self.settings["storedir"]+"/builds/"):
 274                                 os.makedirs(self.settings["storedir"]+"/builds/")
 275
 276         def set_archscript(self):
 277             if self.settings.has_key(self.settings["spec_prefix"]+"/archscript"):
 278                 print "\nWarning!!!  "
 279                 print "\t"+self.settings["spec_prefix"]+"/archscript" + " is deprecated and no longer used.\n"
 280         def set_runscript(self):
 281             if self.settings.has_key(self.settings["spec_prefix"]+"/runscript"):
 282                 print "\nWarning!!!  "
 283                 print "\t"+self.settings["spec_prefix"]+"/runscript" + " is deprecated and no longer used.\n"
 284
 285         def set_fsscript(self):
 286                 if self.settings.has_key(self.settings["spec_prefix"]+"/fsscript"):
 287                         self.settings["fsscript"]=self.settings[self.settings["spec_prefix"]+"/fsscript"]
 288                         del self.settings[self.settings["spec_prefix"]+"/fsscript"]
 289
 290         def set_rcadd(self):
 291                 if self.settings.has_key(self.settings["spec_prefix"]+"/rcadd"):
 292                         self.settings["rcadd"]=self.settings[self.settings["spec_prefix"]+"/rcadd"]
 293                         del self.settings[self.settings["spec_prefix"]+"/rcadd"]
 294
 295         def set_rcdel(self):
 296                 if self.settings.has_key(self.settings["spec_prefix"]+"/rcdel"):
 297                         self.settings["rcdel"]=self.settings[self.settings["spec_prefix"]+"/rcdel"]
 298                         del self.settings[self.settings["spec_prefix"]+"/rcdel"]
 299
 300         def set_cdtar(self):
 301                 if self.settings.has_key(self.settings["spec_prefix"]+"/cdtar"):
 302                         self.settings["cdtar"]=normpath(self.settings[self.settings["spec_prefix"]+"/cdtar"])
 303                         del self.settings[self.settings["spec_prefix"]+"/cdtar"]
 304
 305         def set_iso(self):
 306                 if self.settings.has_key(self.settings["spec_prefix"]+"/iso"):
 307                         self.settings["iso"]=normpath(self.settings[self.settings["spec_prefix"]+"/iso"])
 308                         del self.settings[self.settings["spec_prefix"]+"/iso"]
 309
 310         def set_fstype(self):
 311                 if self.settings.has_key(self.settings["spec_prefix"]+"/cdfstype"):
 312                         print "\nWarning!!!  "
 313                         print self.settings["spec_prefix"]+"/cdfstype" + " is deprecated and may be removed."
 314                         print "\tUse "+self.settings["spec_prefix"]+"/fstype" + " instead."
 315                         print "\tConverting to "+self.settings["spec_prefix"]+"/fstype" + " internally."
 316                         print "\tContinuing ....\n"
 317                         self.settings["fstype"]=self.settings[self.settings["spec_prefix"]+"/cdfstype"]
 318                         del self.settings[self.settings["spec_prefix"]+"/cdfstype"]
 319
 320                 if self.settings.has_key(self.settings["spec_prefix"]+"/fstype"):
 321                         self.settings["fstype"]=self.settings[self.settings["spec_prefix"]+"/fstype"]
 322                         del self.settings[self.settings["spec_prefix"]+"/fstype"]
 323
 324                 if not self.settings.has_key("fstype"):
 325                         self.settings["fstype"]="normal"
 326                         for x in self.valid_values:
 327                                 if x ==  self.settings["spec_prefix"]+"/fstype" or x == self.settings["spec_prefix"]+"/cdfstype":
 328                                         print "\n"+self.settings["spec_prefix"]+"/fstype is being set to the default of \"normal\"\n"
 329
 330         def set_fsops(self):
 331                 if self.settings.has_key("fstype"):
 332                         self.valid_values.append("fsops")
 333                         if self.settings.has_key(self.settings["spec_prefix"]+"/fsops"):
 334                                 self.settings["fsops"]=self.settings[self.settings["spec_prefix"]+"/fsops"]
 335                                 del self.settings[self.settings["spec_prefix"]+"/fsops"]
 336
 337         def set_source_path(self):
 338                 if self.settings.has_key("SEEDCACHE") and os.path.isdir(normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")):
 339                         self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")
 340                 else:
 341                         self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2")
 342                         if os.path.isfile(self.settings["source_path"]):
 343                                 if os.path.exists(self.settings["source_path"]): # XXX: Is this even necessary if the previous check passes?
 344                                         self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\
 345                                                 hash_function=self.settings["hash_function"],verbose=False)
 346                 print "Source path set to "+self.settings["source_path"]
 347                 if os.path.isdir(self.settings["source_path"]):
 348                         print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf"
 349                         print "\tthe source path will then be "+normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2\n")
 350
 351         def set_dest_path(self):
 352                 if self.settings.has_key("root_path"):
 353                         self.settings["destpath"]=normpath(self.settings["chroot_path"]+self.settings["root_path"])
 354                 else:
 355                         self.settings["destpath"]=normpath(self.settings["chroot_path"])
 356
 357         def set_cleanables(self):
 358                 self.settings["cleanables"]=["/etc/resolv.conf","/var/tmp/*","/tmp/*","/root/*",\
 359                                                 "/usr/portage"]
 360
 361         def set_snapshot_path(self):
 362                 self.settings["snapshot_path"]=normpath(self.settings["storedir"]+"/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
 363
 364                 if os.path.exists(self.settings["snapshot_path"]):
 365                         self.settings["snapshot_path_hash"]=generate_hash(self.settings["snapshot_path"],\
 366                                         hash_function=self.settings["hash_function"],verbose=False)
 367
 368         def set_snapcache_path(self):
 369                 if self.settings.has_key("SNAPCACHE"):
 370                         self.settings["snapshot_cache_path"]=normpath(self.settings["snapshot_cache"]+"/"+self.settings["snapshot"]+"/")
 371                         self.snapcache_lock=catalyst_lock.LockDir(self.settings["snapshot_cache_path"])
 372                         print "Caching snapshot to " + self.settings["snapshot_cache_path"]
 373
 374         def set_chroot_path(self):
 375                 # Note the trailing slash is very important and things would break without it
 376                 self.settings["chroot_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["target_subpath"]+"/")
 377                 self.chroot_lock=catalyst_lock.LockDir(self.settings["chroot_path"])
 378
 379         def set_autoresume_path(self):
 380                 self.settings["autoresume_path"]=normpath(self.settings["storedir"]+"/tmp/"+\
 381                         self.settings["rel_type"]+"/"+".autoresume-"+self.settings["target"]+\
 382                         "-"+self.settings["subarch"]+"-"+self.settings["version_stamp"]+"/")
 383                 if self.settings.has_key("AUTORESUME"):
 384                         print "The autoresume path is " + self.settings["autoresume_path"]
 385                 if not os.path.exists(self.settings["autoresume_path"]):
 386                         os.makedirs(self.settings["autoresume_path"],0755)
 387
 388         def set_controller_file(self):
 389                 self.settings["controller_file"]=normpath(self.settings["sharedir"]+"/targets/"+self.settings["target"]+"/"+self.settings["target"]+"-controller.sh")
 390
 391         def set_iso_volume_id(self):
 392                 if self.settings.has_key(self.settings["spec_prefix"]+"/volid"):
 393                         self.settings["iso_volume_id"] = self.settings[self.settings["spec_prefix"]+"/volid"]
 394                         if len(self.settings["iso_volume_id"])>32:
 395                                 raise CatalystError,"ISO VOLUME ID: volid must not exceed 32 characters."
 396                 else:
 397                         self.settings["iso_volume_id"] = "catalyst " + self.settings["snapshot"]
 398
 399         def set_action_sequence(self):
 400                 # Default action sequence for run method
 401                 self.settings["action_sequence"]=["unpack","unpack_snapshot",\
 402                                 "config_profile_link","setup_confdir","portage_overlay",\
 403                                 "base_dirs","bind","chroot_setup","setup_environment",\
 404                                 "run_local","preclean","unbind","clean"]
 405 #               if self.settings.has_key("TARBALL") or \
 406 #                       not self.settings.has_key("FETCH"):
 407                 if not self.settings.has_key("FETCH"):
 408                         self.settings["action_sequence"].append("capture")
 409                 self.settings["action_sequence"].append("clear_autoresume")
 410
 411         def set_use(self):
 412                 if self.settings.has_key(self.settings["spec_prefix"]+"/use"):
 413                         self.settings["use"]=self.settings[self.settings["spec_prefix"]+"/use"]
 414                         del self.settings[self.settings["spec_prefix"]+"/use"]
 415                 if self.settings.has_key("use"):
 416                     if type(self.settings["use"])==types.StringType:
 417                         self.settings["use"]=self.settings["use"].split()
 418
 419         def set_stage_path(self):
 420                         self.settings["stage_path"]=normpath(self.settings["chroot_path"])
 421
 422         def set_mounts(self):
 423                 pass
 424
 425         def set_packages(self):
 426                 pass
 427
 428         def set_rm(self):
 429             if self.settings.has_key(self.settings["spec_prefix"]+"/rm"):
 430                 if type(self.settings[self.settings["spec_prefix"]+"/rm"])==types.StringType:
 431                     self.settings[self.settings["spec_prefix"]+"/rm"]=self.settings[self.settings["spec_prefix"]+"/rm"].split()
 432
 433         def set_linuxrc(self):
 434             if self.settings.has_key(self.settings["spec_prefix"]+"/linuxrc"):
 435                         if type(self.settings[self.settings["spec_prefix"]+"/linuxrc"])==types.StringType:
 436                                 self.settings["linuxrc"]=self.settings[self.settings["spec_prefix"]+"/linuxrc"]
 437                                 del self.settings[self.settings["spec_prefix"]+"/linuxrc"]
 438
 439         def set_portage_overlay(self):
 440                 if self.settings.has_key("portage_overlay"):
 441                         if type(self.settings["portage_overlay"])==types.StringType:
 442                                 self.settings["portage_overlay"]=self.settings["portage_overlay"].split()
 443                         print "portage_overlay directories are set to: \"" + string.join(self.settings["portage_overlay"])+"\""
 444
 445         def set_overlay(self):
 446                 if self.settings.has_key(self.settings["spec_prefix"]+"/overlay"):
 447                         if type(self.settings[self.settings["spec_prefix"]+"/overlay"])==types.StringType:
 448                                 self.settings[self.settings["spec_prefix"]+"/overlay"]=self.settings[self.settings["spec_prefix"]+"/overlay"].split()
 449
 450         def set_root_overlay(self):
 451                 if self.settings.has_key(self.settings["spec_prefix"]+"/root_overlay"):
 452                         if type(self.settings[self.settings["spec_prefix"]+"/root_overlay"])==types.StringType:
 453                                 self.settings[self.settings["spec_prefix"]+"/root_overlay"]=self.settings[self.settings["spec_prefix"]+"/root_overlay"].split()
 454
 455
 456         def set_root_path(self):
 457                 # ROOT= variable for emerges
 458                 self.settings["root_path"]="/"
 459
 460         def set_valid_build_kernel_vars(self,addlargs):
 461                 if addlargs.has_key("boot/kernel"):
 462                         if type(addlargs["boot/kernel"]) == types.StringType:
 463                                 loopy=[addlargs["boot/kernel"]]
 464                         else:
 465                                 loopy=addlargs["boot/kernel"]
 466
 467                         for x in loopy:
 468                                 self.required_values.append("boot/kernel/"+x+"/sources")
 469                                 self.required_values.append("boot/kernel/"+x+"/config")
 470                                 self.valid_values.append("boot/kernel/"+x+"/aliases")
 471                                 self.valid_values.append("boot/kernel/"+x+"/extraversion")
 472                                 self.valid_values.append("boot/kernel/"+x+"/packages")
 473                                 if addlargs.has_key("boot/kernel/"+x+"/packages"):
 474                                         if type(addlargs["boot/kernel/"+x+"/packages"]) == types.StringType:
 475                                                 addlargs["boot/kernel/"+x+"/packages"]=[addlargs["boot/kernel/"+x+"/packages"]]
 476                                 self.valid_values.append("boot/kernel/"+x+"/use")
 477                                 self.valid_values.append("boot/kernel/"+x+"/gk_kernargs")
 478                                 self.valid_values.append("boot/kernel/"+x+"/gk_action")
 479                                 self.valid_values.append("boot/kernel/"+x+"/initramfs_overlay")
 480                                 self.valid_values.append("boot/kernel/"+x+"/softlevel")
 481                                 self.valid_values.append("boot/kernel/"+x+"/console")
 482                                 self.valid_values.append("boot/kernel/"+x+"/machine_type")
 483                                 self.valid_values.append("boot/kernel/"+x+"/postconf")
 484                                 if addlargs.has_key("boot/kernel/"+x+"/postconf"):
 485                                         print "boot/kernel/"+x+"/postconf is deprecated"
 486                                         print "\tInternally moving these ebuilds to boot/kernel/"+x+"/packages"
 487                                         print "\tPlease move them to boot/kernel/"+x+"/packages in your specfile"
 488                                         if type(addlargs["boot/kernel/"+x+"/postconf"]) == types.StringType:
 489                                                 loop2=[addlargs["boot/kernel/"+x+"/postconf"]]
 490                                         else:
 491                                                 loop2=addlargs["boot/kernel/"+x+"/postconf"]
 492
 493                                         for y in loop2:
 494                                                 if not addlargs.has_key("boot/kernel/"+x+"/packages"):
 495                                                         addlargs["boot/kernel/"+x+"/packages"]=[y]
 496                                                 else:
 497                                                         addlargs["boot/kernel/"+x+"/packages"].append(y)
 498
 499         def set_build_kernel_vars(self):
 500             if self.settings.has_key(self.settings["spec_prefix"]+"/devmanager"):
 501                 self.settings["devmanager"]=self.settings[self.settings["spec_prefix"]+"/devmanager"]
 502                 del self.settings[self.settings["spec_prefix"]+"/devmanager"]
 503
 504             if self.settings.has_key(self.settings["spec_prefix"]+"/splashtype"):
 505                 self.settings["splashtype"]=self.settings[self.settings["spec_prefix"]+"/splashtype"]
 506                 del self.settings[self.settings["spec_prefix"]+"/splashtype"]
 507
 508             if self.settings.has_key(self.settings["spec_prefix"]+"/gk_mainargs"):
 509                 self.settings["gk_mainargs"]=self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
 510                 del self.settings[self.settings["spec_prefix"]+"/gk_mainargs"]
 511
 512         def kill_chroot_pids(self):
 513             print "Checking for processes running in chroot and killing them."
 514
 515             # Force environment variables to be exported so script can see them
 516             self.setup_environment()
 517
 518             if os.path.exists(self.settings["sharedir"]+"/targets/support/kill-chroot-pids.sh"):
 519                             cmd("/bin/bash "+self.settings["sharedir"]+"/targets/support/kill-chroot-pids.sh",\
 520                                                 "kill-chroot-pids script failed.",env=self.env)
 521
 522         def mount_safety_check(self):
 523                 mypath=self.settings["chroot_path"]
 524
 525                 """
 526                 check and verify that none of our paths in mypath are mounted. We don't want to clean
 527                 up with things still mounted, and this allows us to check.
 528                 returns 1 on ok, 0 on "something is still mounted" case.
 529                 """
 530                 if not os.path.exists(mypath):
 531                         return
 532
 533                 for x in self.mounts:
 534                         if not os.path.exists(mypath+x):
 535                                 continue
 536
 537                         if ismount(mypath+x):
 538                                 #something is still mounted
 539                                 try:
 540                                         print x+" is still mounted; performing auto-bind-umount...",
 541                                         # try to umount stuff ourselves
 542                                         self.unbind()
 543                                         if ismount(mypath+x):
 544                                                 raise CatalystError, "Auto-unbind failed for "+x
 545
 546                                         else:
 547                                                 print "Auto-unbind successful..."
 548
 549                                 except CatalystError:
 550                                         raise CatalystError, "Unable to auto-unbind "+x
 551
 552         def unpack(self):
 553                 unpack=True
 554
 555                 clst_unpack_hash=read_from_clst(self.settings["autoresume_path"]+"unpack")
 556
 557                 if self.settings.has_key("SEEDCACHE"):
 558                         if os.path.isdir(self.settings["source_path"]):
 559                                 # SEEDCACHE Is a directory, use Rsync
 560                                 unpack_cmd="rsync -a --delete "+self.settings["source_path"]+" "+self.settings["chroot_path"]
 561                                 display_msg="\nStarting rsync from "+self.settings["source_path"]+"\nto "+\
 562                                         self.settings["chroot_path"]+" (This may take some time) ...\n"
 563                                 error_msg="Rsync of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 564                         else:
 565                                 # SEEDCACHE is a not a directory, try untar'ing
 566                                 print "Referenced SEEDCACHE does not appear to be a directory, trying to untar..."
 567                                 display_msg="\nStarting tar extract from "+self.settings["source_path"]+"\nto "+\
 568                                         self.settings["chroot_path"]+" (This may take some time) ...\n"
 569                                 unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"]
 570                                 error_msg="Tarball extraction of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 571                 else:
 572                         # No SEEDCACHE, use tar
 573                         display_msg="\nStarting tar extract from "+self.settings["source_path"]+"\nto "+\
 574                                 self.settings["chroot_path"]+" (This may take some time) ...\n"
 575                         unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"]
 576                         error_msg="Tarball extraction of "+self.settings["source_path"]+" to "+self.settings["chroot_path"]+" failed."
 577
 578
 579                 if self.settings.has_key("AUTORESUME"):
 580                         # Autoresume is Valid, SeedCache is Valid
 581                         if os.path.isdir(self.settings["source_path"]) and os.path.exists(self.settings["autoresume_path"]+"unpack"):
 582                                 unpack=False
 583                                 invalid_snapshot=False
 584
 585                         # Autoresume is Valid, Tarball is Valid
 586                         elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] == clst_unpack_hash:
 587                                 unpack=False
 588                                 invalid_snapshot=True
 589
 590                         # Autoresume is InValid, SeedCache
 591                         elif os.path.isdir(self.settings["source_path"]) and not os.path.exists(self.settings["autoresume_path"]+"unpack"):
 592                                 unpack=True
 593                                 invalid_snapshot=False
 594
 595                         # Autoresume is InValid, Tarball
 596                         elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] != clst_unpack_hash:
 597                                 unpack=True
 598                                 invalid_snapshot=True
 599                 else:
 600                         # No Autoresume,SeedCache
 601                         if self.settings.has_key("SEEDCACHE"):
 602
 603                                 # Seed cache so lets run rsync and rsync can clean up extra junk
 604                                 if os.path.isdir(self.settings["source_path"]):
 605                                         unpack=True
 606                                         invalid_snapshot=False
 607
 608                                 # Tarball so we better unpack and remove anything already there
 609                                 elif os.path.isfile(self.settings["source_path"]):
 610                                         unpack=True
 611                                         invalid_snapshot=True
 612
 613                         # No Autoresume,No SeedCache
 614                         else:
 615
 616                                 # Tarball so we better unpack and remove anything already there
 617                                 if os.path.isfile(self.settings["source_path"]):
 618                                         unpack=True
 619                                         invalid_snapshot=True
 620                                 # Should never reach this if so something is very wrong
 621                                 elif os.path.isdir(self.settings["source_path"]):
 622                                         raise CatalystError,"source path is a dir but seedcache is not enabled"
 623
 624                 if unpack:
 625                         self.mount_safety_check()
 626
 627                         if invalid_snapshot:
 628                                 if self.settings.has_key("AUTORESUME"):
 629                                         print "No Valid Resume point detected, cleaning up  ..."
 630
 631                                 self.clear_autoresume()
 632                                 self.clear_chroot()
 633
 634                         if not os.path.exists(self.settings["chroot_path"]):
 635                                 os.makedirs(self.settings["chroot_path"])
 636
 637                         if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
 638                                 os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
 639
 640                         if self.settings.has_key("PKGCACHE"):
 641                                 if not os.path.exists(self.settings["pkgcache_path"]):
 642                                         os.makedirs(self.settings["pkgcache_path"],0755)
 643
 644                         if self.settings.has_key("KERNCACHE"):
 645                                 if not os.path.exists(self.settings["kerncache_path"]):
 646                                         os.makedirs(self.settings["kerncache_path"],0755)
 647
 648                         print display_msg
 649                         cmd(unpack_cmd,error_msg,env=self.env)
 650
 651                         if self.settings.has_key("source_path_hash"):
 652                                 myf=open(self.settings["autoresume_path"]+"unpack","w")
 653                                 myf.write(self.settings["source_path_hash"])
 654                                 myf.close()
 655                         else:
 656                                 touch(self.settings["autoresume_path"]+"unpack")
 657                 else:
 658                     print "Resume point detected, skipping unpack operation..."
 659
 660
 661         def unpack_snapshot(self):
 662                 unpack=True
 663                 snapshot_hash=read_from_clst(self.settings["autoresume_path"]+"unpack_portage")
 664
 665                 if self.settings.has_key("SNAPCACHE"):
 666                         snapshot_cache_hash=read_from_clst(self.settings["snapshot_cache_path"]+"catalyst-hash")
 667                         destdir=self.settings["snapshot_cache_path"]
 668                         unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+destdir
 669                         unpack_errmsg="Error unpacking snapshot"
 670                         cleanup_msg="Cleaning up invalid snapshot cache at \n\t"+self.settings["snapshot_cache_path"]+" (This can take a long time)..."
 671                         cleanup_errmsg="Error removing existing snapshot cache directory."
 672                         self.snapshot_lock_object=self.snapcache_lock
 673
 674                         if self.settings["snapshot_path_hash"] == snapshot_cache_hash:
 675                                 print "Valid snapshot cache, skipping unpack of portage tree ..."
 676                                 unpack=False
 677
 678                 else:
 679                         destdir=normpath(self.settings["chroot_path"]+"/usr/portage")
 680                         cleanup_errmsg="Error removing existing snapshot directory."
 681                         cleanup_msg="Cleaning up existing portage tree (This can take a long time) ..."
 682                         unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+self.settings["chroot_path"]+"/usr"
 683                         unpack_errmsg="Error unpacking snapshot"
 684
 685                         if self.settings.has_key("AUTORESUME") \
 686                         and os.path.exists(self.settings["chroot_path"]+"/usr/portage/") \
 687                         and os.path.exists(self.settings["autoresume_path"]+"unpack_portage") \
 688                         and self.settings["snapshot_path_hash"] == snapshot_hash:
 689                                 print "Valid Resume point detected, skipping unpack of portage tree..."
 690                                 unpack=False
 691
 692
 693
 694                 if unpack:
 695                         if self.settings.has_key("SNAPCACHE"):
 696                             self.snapshot_lock_object.write_lock()
 697                         if os.path.exists(destdir):
 698                                 print cleanup_msg
 699                                 cleanup_cmd="rm -rf "+destdir
 700                                 cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
 701                         if not os.path.exists(destdir):
 702                                 os.makedirs(destdir,0755)
 703
 704                         print "Unpacking portage tree (This can take a long time) ..."
 705                         cmd(unpack_cmd,unpack_errmsg,env=self.env)
 706
 707                         if self.settings.has_key("SNAPCACHE"):
 708                                 myf=open(self.settings["snapshot_cache_path"]+"catalyst-hash","w")
 709                                 myf.write(self.settings["snapshot_path_hash"])
 710                                 myf.close()
 711
 712                         else:
 713                                 print "Setting snapshot autoresume point"
 714                                 myf=open(self.settings["autoresume_path"]+"unpack_portage","w")
 715                                 myf.write(self.settings["snapshot_path_hash"])
 716                                 myf.close()
 717
 718                         if self.settings.has_key("SNAPCACHE"):
 719                             self.snapshot_lock_object.unlock()
 720
 721         def config_profile_link(self):
 722                 if self.settings.has_key("AUTORESUME") \
 723                         and os.path.exists(self.settings["autoresume_path"]+"config_profile_link"):
 724                         print "Resume point detected, skipping config_profile_link operation..."
 725                 else:
 726                         print "Configuring profile link..."
 727                         cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.profile",\
 728                                         "Error zapping profile link",env=self.env)
 729                         cmd("ln -sf ../usr/portage/profiles/"+self.settings["target_profile"]+\
 730                                 " "+self.settings["chroot_path"]+"/etc/make.profile","Error creating profile link",env=self.env)
 731                         touch(self.settings["autoresume_path"]+"config_profile_link")
 732
 733         def setup_confdir(self):
 734                 if self.settings.has_key("AUTORESUME") \
 735                         and os.path.exists(self.settings["autoresume_path"]+"setup_confdir"):
 736                         print "Resume point detected, skipping setup_confdir operation..."
 737                 else:
 738                         if self.settings.has_key("portage_confdir"):
 739                                 print "Configuring /etc/portage..."
 740                                 cmd("rm -rf "+self.settings["chroot_path"]+"/etc/portage","Error zapping /etc/portage",env=self.env)
 741                                 cmd("cp -R "+self.settings["portage_confdir"]+"/ "+self.settings["chroot_path"]+\
 742                                         "/etc/portage","Error copying /etc/portage",env=self.env)
 743                                 touch(self.settings["autoresume_path"]+"setup_confdir")
 744
 745         def portage_overlay(self):
 746             # Here, we copy the contents of our overlays to /usr/local/portage. We
 747             # always copy over the overlays in case it has changed.
 748             if self.settings.has_key("portage_overlay"):
 749                 for x in self.settings["portage_overlay"]:
 750                         if os.path.exists(x):
 751                                 print "Copying overlay dir " +x
 752                                 cmd("mkdir -p "+self.settings["chroot_path"]+"/usr/local/portage","Could not make portage_overlay dir",env=self.env)
 753                                 cmd("cp -R "+x+"/* "+self.settings["chroot_path"]+"/usr/local/portage","Could not copy portage_overlay",env=self.env)
 754
 755         def root_overlay(self):
 756             # copy over the root_overlay
 757             # Always copy over the overlay incase it has changed
 758                 if self.settings.has_key(self.settings["spec_prefix"]+"/root_overlay"):
 759                         for x in self.settings[self.settings["spec_prefix"]+"/root_overlay"]:
 760                                 if os.path.exists(x):
 761                                         print "Copying root_overlay: "+x
 762                                         cmd("rsync -a "+x+"/ "+\
 763                                                 self.settings["chroot_path"], self.settings["spec_prefix"]+"/root_overlay: "+x+" copy failed.",env=self.env)
 764
 765         def base_dirs(self):
 766                 pass
 767
 768         def bind(self):
 769                 for x in self.mounts:
 770                         if not os.path.exists(self.settings["chroot_path"]+x):
 771                                 os.makedirs(self.settings["chroot_path"]+x,0755)
 772
 773                         if not os.path.exists(self.mountmap[x]):
 774                                 os.makedirs(self.mountmap[x],0755)
 775
 776                         src=self.mountmap[x]
 777                         if self.settings.has_key("SNAPCACHE") and x == "/usr/portage":
 778                                 self.snapshot_lock_object.read_lock()
 779                         if os.uname()[0] == "FreeBSD":
 780                                 if src == "/dev":
 781                                         retval=os.system("mount -t devfs none "+self.settings["chroot_path"]+x)
 782                                 else:
 783                                         retval=os.system("mount_nullfs "+src+" "+self.settings["chroot_path"]+x)
 784                         else:
 785                                 retval=os.system("mount --bind "+src+" "+self.settings["chroot_path"]+x)
 786                         if retval!=0:
 787                                 self.unbind()
 788                                 raise CatalystError,"Couldn't bind mount "+src
 789
 790
 791         def unbind(self):
 792                 ouch=0
 793                 mypath=self.settings["chroot_path"]
 794                 myrevmounts=self.mounts[:]
 795                 myrevmounts.reverse()
 796                 # unmount in reverse order for nested bind-mounts
 797                 for x in myrevmounts:
 798                         if not os.path.exists(mypath+x):
 799                                 continue
 800
 801                         if not ismount(mypath+x):
 802                                 # it's not mounted, continue
 803                                 continue
 804
 805                         retval=os.system("umount "+os.path.join(mypath,x.lstrip(os.path.sep)))
 806
 807                         if retval!=0:
 808                                 warn("First attempt to unmount: "+mypath+x+" failed.")
 809                                 warn("Killing any pids still running in the chroot")
 810
 811                                 self.kill_chroot_pids()
 812
 813                                 retval2=os.system("umount "+mypath+x)
 814                                 if retval2!=0:
 815                                     ouch=1
 816                                     warn("Couldn't umount bind mount: "+mypath+x)
 817                                     # keep trying to umount the others, to minimize damage if developer makes a mistake
 818
 819                         if self.settings.has_key("SNAPCACHE") and x == "/usr/portage":
 820                                 try:
 821                                     # Its possible the snapshot lock object isnt created yet
 822                                     # this is because mount safety check calls unbind before the target is fully initialized
 823                                     self.snapshot_lock_object.unlock()
 824                                 except:
 825                                     pass
 826                 if ouch:
 827                         """
 828                         if any bind mounts really failed, then we need to raise
 829                         this to potentially prevent an upcoming bash stage cleanup script
 830                         from wiping our bind mounts.
 831                         """
 832                         raise CatalystError,"Couldn't umount one or more bind-mounts; aborting for safety."
 833
 834         def chroot_setup(self):
 835                 self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/make.conf")
 836                 self.override_cbuild()
 837                 self.override_chost()
 838                 self.override_cflags()
 839                 self.override_cxxflags()
 840                 self.override_ldflags()
 841                 if self.settings.has_key("AUTORESUME") \
 842                         and os.path.exists(self.settings["autoresume_path"]+"chroot_setup"):
 843                         print "Resume point detected, skipping chroot_setup operation..."
 844                 else:
 845                         print "Setting up chroot..."
 846
 847                         #self.makeconf=read_makeconf(self.settings["chroot_path"]+"/etc/make.conf")
 848
 849                         cmd("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc",\
 850                                 "Could not copy resolv.conf into place.",env=self.env)
 851
 852                     # copy over the envscript, if applicable
 853                         if self.settings.has_key("ENVSCRIPT"):
 854                                 if not os.path.exists(self.settings["ENVSCRIPT"]):
 855                                         raise CatalystError, "Can't find envscript "+self.settings["ENVSCRIPT"]
 856
 857                                 print "\nWarning!!!!"
 858                                 print "\tOverriding certain env variables may cause catastrophic failure."
 859                                 print "\tIf your build fails look here first as the possible problem."
 860                                 print "\tCatalyst assumes you know what you are doing when setting"
 861                                 print "\t\tthese variables."
 862                                 print "\tCatalyst Maintainers use VERY minimal envscripts if used at all"
 863                                 print "\tYou have been warned\n"
 864
 865                                 cmd("cp "+self.settings["ENVSCRIPT"]+" "+self.settings["chroot_path"]+"/tmp/envscript",\
 866                                         "Could not copy envscript into place.",env=self.env)
 867
 868                         # Setup metadata_overlay
 869                         if self.settings.has_key("METADATA_OVERLAY") and not self.settings.has_key("portage_confdir"):
 870                                 if not os.path.exists(self.settings["chroot_path"] + "/etc/portage"):
 871                                         cmd("mkdir " + self.settings["chroot_path"] + "/etc/portage")
 872                                 myf = open(self.settings["chroot_path"] + "/etc/portage/modules", "a")
 873                                 myf.write("portdbapi.auxdbmodule = cache.metadata_overlay.database\n")
 874                                 myf.close()
 875
 876                     # Copy over /etc/hosts from the host in case there are any
 877                         # specialties in there
 878                         if os.path.exists(self.settings["chroot_path"]+"/etc/hosts"):
 879                                 cmd("mv "+self.settings["chroot_path"]+"/etc/hosts "+self.settings["chroot_path"]+\
 880                                         "/etc/hosts.bck", "Could not backup /etc/hosts",env=self.env)
 881                                 cmd("cp /etc/hosts "+self.settings["chroot_path"]+"/etc/hosts", "Could not copy /etc/hosts",env=self.env)
 882                         #self.override_cbuild()
 883                         #self.override_chost()
 884                         #self.override_cflags()
 885                         #self.override_cxxflags()
 886                         #self.override_ldflags()
 887                         # Modify and write out make.conf (for the chroot)
 888                         cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.conf","Could not remove "+self.settings["chroot_path"]+"/etc/make.conf",\
 889                                 env=self.env)
 890                         myf=open(self.settings["chroot_path"]+"/etc/make.conf","w")
 891                         myf.write("# These settings were set by the catalyst build script that automatically\n# built this stage.\n")
 892                         myf.write("# Please consult /etc/make.conf.example for a more detailed example.\n")
 893                         if self.settings.has_key("CFLAGS"):
 894                                 myf.write('CFLAGS="'+self.settings["CFLAGS"]+'"\n')
 895                         if self.settings.has_key("CXXFLAGS"):
 896                                 myf.write('CXXFLAGS="'+self.settings["CXXFLAGS"]+'"\n')
 897                         else:
 898                                 myf.write('CXXFLAGS="${CFLAGS}"\n')
 899
 900                         if self.settings.has_key("LDFLAGS"):
 901                                 myf.write('LDFLAGS="'+self.settings["LDFLAGS"]+'"\n')
 902                         myf.write("# This should not be changed unless you know exactly what you are doing.  You\n# should probably be using a different stage, instead.\n")
 903                         if self.settings.has_key("CBUILD"):
 904                                 myf.write('CBUILD="'+self.settings["CBUILD"]+'"\n')
 905                         myf.write('# WARNING: Changing your CHOST is not something that should be done lightly.\n# Please consult http://www.gentoo.org/doc/en/change-chost.xml before changing it.\nCHOST="'+self.settings["CHOST"]+'"\n')
 906
 907                     # Figure out what our USE vars are for building
 908                         myusevars=[]
 909                         if self.settings.has_key("HOSTUSE"):
 910                                 myusevars.extend(self.settings["HOSTUSE"])
 911
 912                         if self.settings.has_key("use"):
 913                                 myusevars.extend(self.settings["use"])
 914                                 myf.write('USE="'+string.join(myusevars)+'"\n')
 915
 916                     # Setup the portage overlay
 917                         if self.settings.has_key("portage_overlay"):
 918 #                               myf.write('PORTDIR_OVERLAY="'+string.join(self.settings["portage_overlay"])+'"\n')
 919                                 myf.write('PORTDIR_OVERLAY="/usr/local/portage"\n')
 920
 921                         myf.close()
 922                         touch(self.settings["autoresume_path"]+"chroot_setup")
 923
 924         def fsscript(self):
 925                 if self.settings.has_key("AUTORESUME") \
 926                         and os.path.exists(self.settings["autoresume_path"]+"fsscript"):
 927                         print "Resume point detected, skipping fsscript operation..."
 928                 else:
 929                     if self.settings.has_key("fsscript"):
 930                         if os.path.exists(self.settings["controller_file"]):
 931                             cmd("/bin/bash "+self.settings["controller_file"]+" fsscript","fsscript script failed.",env=self.env)
 932                         touch(self.settings["autoresume_path"]+"fsscript")
 933
 934         def rcupdate(self):
 935                 if self.settings.has_key("AUTORESUME") \
 936                         and os.path.exists(self.settings["autoresume_path"]+"rcupdate"):
 937                         print "Resume point detected, skipping rcupdate operation..."
 938                 else:
 939                     if os.path.exists(self.settings["controller_file"]):
 940                         cmd("/bin/bash "+self.settings["controller_file"]+" rc-update","rc-update script failed.",env=self.env)
 941                         touch(self.settings["autoresume_path"]+"rcupdate")
 942
 943         def clean(self):
 944                 if self.settings.has_key("AUTORESUME") \
 945                         and os.path.exists(self.settings["autoresume_path"]+"clean"):
 946                         print "Resume point detected, skipping clean operation..."
 947                 else:
 948                     for x in self.settings["cleanables"]:
 949                             print "Cleaning chroot: "+x+"... "
 950                             cmd("rm -rf "+self.settings["destpath"]+x,"Couldn't clean "+x,env=self.env)
 951
 952                     # put /etc/hosts back into place
 953                     if os.path.exists(self.settings["chroot_path"]+"/etc/hosts.bck"):
 954                                 cmd("mv -f "+self.settings["chroot_path"]+"/etc/hosts.bck "+self.settings["chroot_path"]+"/etc/hosts", "Could not replace /etc/hosts",env=self.env)
 955
 956                     # remove our overlay
 957                     if os.path.exists(self.settings["chroot_path"]+"/usr/local/portage"):
 958                                 cmd("rm -rf "+self.settings["chroot_path"]+"/usr/local/portage", "Could not remove /usr/local/portage",env=self.env)
 959                                 cmd("sed -i '/^PORTDIR_OVERLAY/d' "+self.settings["chroot_path"]+"/etc/make.conf", "Could not remove PORTDIR_OVERLAY from make.conf",env=self.env)
 960
 961                     # clean up old and obsoleted files in /etc
 962                     if os.path.exists(self.settings["stage_path"]+"/etc"):
 963                                 cmd("find "+self.settings["stage_path"]+"/etc -maxdepth 1 -name \"*-\" | xargs rm -f", "Could not remove stray files in /etc",env=self.env)
 964
 965                     if os.path.exists(self.settings["controller_file"]):
 966                         cmd("/bin/bash "+self.settings["controller_file"]+" clean","clean script failed.",env=self.env)
 967                         touch(self.settings["autoresume_path"]+"clean")
 968
 969         def empty(self):
 970             if self.settings.has_key("AUTORESUME") \
 971                 and os.path.exists(self.settings["autoresume_path"]+"empty"):
 972                 print "Resume point detected, skipping empty operation..."
 973             else:
 974                 if self.settings.has_key(self.settings["spec_prefix"]+"/empty"):
 975                     if type(self.settings[self.settings["spec_prefix"]+"/empty"])==types.StringType:
 976                         self.settings[self.settings["spec_prefix"]+"/empty"]=self.settings[self.settings["spec_prefix"]+"/empty"].split()
 977                     for x in self.settings[self.settings["spec_prefix"]+"/empty"]:
 978                         myemp=self.settings["destpath"]+x
 979                         if not os.path.isdir(myemp):
 980                             print x,"not a directory or does not exist, skipping 'empty' operation."
 981                             continue
 982                         print "Emptying directory",x
 983                         # stat the dir, delete the dir, recreate the dir and set
 984                         # the proper perms and ownership
 985                         mystat=os.stat(myemp)
 986                         shutil.rmtree(myemp)
 987                         os.makedirs(myemp,0755)
 988                         os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 989                         os.chmod(myemp,mystat[ST_MODE])
 990                     touch(self.settings["autoresume_path"]+"empty")
 991
 992         def remove(self):
 993             if self.settings.has_key("AUTORESUME") \
 994                 and os.path.exists(self.settings["autoresume_path"]+"remove"):
 995                 print "Resume point detected, skipping remove operation..."
 996             else:
 997                 if self.settings.has_key(self.settings["spec_prefix"]+"/rm"):
 998                     for x in self.settings[self.settings["spec_prefix"]+"/rm"]:
 999                         # we're going to shell out for all these cleaning operations,
1000                         # so we get easy glob handling
1001                         print "livecd: removing "+x
1002                         os.system("rm -rf "+self.settings["chroot_path"]+x)
1003                 try:
1004                     if os.path.exists(self.settings["controller_file"]):
1005                             cmd("/bin/bash "+self.settings["controller_file"]+" clean",\
1006                                 "Clean  failed.",env=self.env)
1007                             touch(self.settings["autoresume_path"]+"remove")
1008                 except:
1009                     self.unbind()
1010                     raise
1011
1012
1013         def preclean(self):
1014             if self.settings.has_key("AUTORESUME") \
1015                 and os.path.exists(self.settings["autoresume_path"]+"preclean"):
1016                 print "Resume point detected, skipping preclean operation..."
1017             else:
1018                 try:
1019                         if os.path.exists(self.settings["controller_file"]):
1020                                 cmd("/bin/bash "+self.settings["controller_file"]+" preclean","preclean script failed.",env=self.env)
1021                                 touch(self.settings["autoresume_path"]+"preclean")
1022
1023                 except:
1024                         self.unbind()
1025                         raise CatalystError, "Build failed, could not execute preclean"
1026
1027         def capture(self):
1028             if self.settings.has_key("AUTORESUME") \
1029                 and os.path.exists(self.settings["autoresume_path"]+"capture"):
1030                 print "Resume point detected, skipping capture operation..."
1031             else:
1032                 """capture target in a tarball"""
1033                 mypath=self.settings["target_path"].split("/")
1034                 # remove filename from path
1035                 mypath=string.join(mypath[:-1],"/")
1036
1037                 # now make sure path exists
1038                 if not os.path.exists(mypath):
1039                         os.makedirs(mypath)
1040
1041                 print "Creating stage tarball..."
1042
1043                 cmd("tar cjpf "+self.settings["target_path"]+" -C "+self.settings["stage_path"]+\
1044                         " .","Couldn't create stage tarball",env=self.env)
1045
1046                 self.gen_digest_file(self.settings["target_path"])
1047
1048                 touch(self.settings["autoresume_path"]+"capture")
1049
1050         def run_local(self):
1051             if self.settings.has_key("AUTORESUME") \
1052                 and os.path.exists(self.settings["autoresume_path"]+"run_local"):
1053                 print "Resume point detected, skipping run_local operation..."
1054             else:
1055                 try:
1056                         if os.path.exists(self.settings["controller_file"]):
1057                                 cmd("/bin/bash "+self.settings["controller_file"]+" run","run script failed.",env=self.env)
1058                                 touch(self.settings["autoresume_path"]+"run_local")
1059
1060                 except CatalystError:
1061                         self.unbind()
1062                         raise CatalystError,"Stage build aborting due to error."
1063
1064         def setup_environment(self):
1065                 # Modify the current environment. This is an ugly hack that should be
1066                 # fixed. We need this to use the os.system() call since we can't
1067                 # specify our own environ:
1068                 for x in self.settings.keys():
1069                         # sanitize var names by doing "s|/-.|_|g"
1070                         varname="clst_"+string.replace(x,"/","_")
1071                         varname=string.replace(varname,"-","_")
1072                         varname=string.replace(varname,".","_")
1073                         if type(self.settings[x])==types.StringType:
1074                                 # prefix to prevent namespace clashes:
1075                                 #os.environ[varname]=self.settings[x]
1076                                 self.env[varname]=self.settings[x]
1077                         elif type(self.settings[x])==types.ListType:
1078                                 #os.environ[varname]=string.join(self.settings[x])
1079                                 self.env[varname]=string.join(self.settings[x])
1080                         elif type(self.settings[x])==types.BooleanType:
1081                                 if self.settings[x]:
1082                                         self.env[varname]="true"
1083                                 else:
1084                                         self.env[varname]="false"
1085                 if self.settings.has_key("makeopts"):
1086                         self.env["MAKEOPTS"]=self.settings["makeopts"]
1087
1088         def run(self):
1089                 self.chroot_lock.write_lock()
1090
1091                 # Kill any pids in the chroot
1092                 self.kill_chroot_pids()
1093
1094                 # Check for mounts right away and abort if we cannot unmount them.
1095                 self.mount_safety_check()
1096
1097                 if self.settings.has_key("CLEAR_AUTORESUME"):
1098                         self.clear_autoresume()
1099                 if self.settings.has_key("PURGE"):
1100                         self.purge()
1101
1102                 for x in self.settings["action_sequence"]:
1103                         print "--- Running action sequence: "+x
1104                         sys.stdout.flush()
1105                         try:
1106                                 apply(getattr(self,x))
1107                         except:
1108                                 self.mount_safety_check()
1109                                 raise
1110
1111                 self.chroot_lock.unlock()
1112
1113         def unmerge(self):
1114             if self.settings.has_key("AUTORESUME") \
1115                 and os.path.exists(self.settings["autoresume_path"]+"unmerge"):
1116                     print "Resume point detected, skipping unmerge operation..."
1117             else:
1118                 if self.settings.has_key(self.settings["spec_prefix"]+"/unmerge"):
1119                     if type(self.settings[self.settings["spec_prefix"]+"/unmerge"])==types.StringType:
1120                         self.settings[self.settings["spec_prefix"]+"/unmerge"]=[self.settings[self.settings["spec_prefix"]+"/unmerge"]]
1121                     myunmerge=self.settings[self.settings["spec_prefix"]+"/unmerge"][:]
1122
1123                     for x in range(0,len(myunmerge)):
1124                     #surround args with quotes for passing to bash,
1125                     #allows things like "<" to remain intact
1126                         myunmerge[x]="'"+myunmerge[x]+"'"
1127                     myunmerge=string.join(myunmerge)
1128
1129                     #before cleaning, unmerge stuff:
1130                     try:
1131                         cmd("/bin/bash "+self.settings["controller_file"]+" unmerge "+ myunmerge,\
1132                                 "Unmerge script failed.",env=self.env)
1133                         #cmd("/bin/bash "+self.settings["sharedir"]+"/targets/" \
1134                         #       +self.settings["target"]+"/unmerge.sh "+myunmerge,"Unmerge script failed.",env=self.env)
1135                         print "unmerge shell script"
1136                     except CatalystError:
1137                         self.unbind()
1138                         raise
1139
1140                 touch(self.settings["autoresume_path"]+"unmerge")
1141
1142         def target_setup(self):
1143             if self.settings.has_key("AUTORESUME") \
1144                 and os.path.exists(self.settings["autoresume_path"]+"target_setup"):
1145                     print "Resume point detected, skipping target_setup operation..."
1146             else:
1147                 print "Setting up filesystems per filesystem type"
1148                 cmd("/bin/bash "+self.settings["controller_file"]+" target_image_setup "+ self.settings["target_path"],\
1149                                 "target_image_setup script failed.",env=self.env)
1150                 touch(self.settings["autoresume_path"]+"target_setup")
1151
1152         def setup_overlay(self):
1153                 if self.settings.has_key("AUTORESUME") \
1154                 and os.path.exists(self.settings["autoresume_path"]+"setup_overlay"):
1155                         print "Resume point detected, skipping setup_overlay operation..."
1156                 else:
1157                         if self.settings.has_key(self.settings["spec_prefix"]+"/overlay"):
1158                                 for x in self.settings[self.settings["spec_prefix"]+"/overlay"]:
1159                                         if os.path.exists(x):
1160                                                 cmd("rsync -a "+x+"/ "+\
1161                                                         self.settings["target_path"], self.settings["spec_prefix"]+"overlay: "+x+" copy failed.",env=self.env)
1162                                 touch(self.settings["autoresume_path"]+"setup_overlay")
1163
1164         def create_iso(self):
1165             if self.settings.has_key("AUTORESUME") \
1166                 and os.path.exists(self.settings["autoresume_path"]+"create_iso"):
1167                     print "Resume point detected, skipping create_iso operation..."
1168             else:
1169                 # create the ISO - this is the preferred method (the iso scripts do not always work)
1170                 if self.settings.has_key("iso"):
1171                         cmd("/bin/bash "+self.settings["controller_file"]+" iso "+\
1172                                 self.settings["iso"],"ISO creation script failed.",env=self.env)
1173                         self.gen_digest_file(self.settings["iso"])
1174                         touch(self.settings["autoresume_path"]+"create_iso")
1175
1176
1177                 else:
1178                         print "WARNING livecd/iso was not defined."
1179                         print "A CD Image will not be created, skipping create-iso.sh..."
1180
1181
1182         def build_packages(self):
1183             if self.settings.has_key("AUTORESUME") \
1184                 and os.path.exists(self.settings["autoresume_path"]+"build_packages"):
1185                     print "Resume point detected, skipping build_packages operation..."
1186             else:
1187                 if self.settings.has_key(self.settings["spec_prefix"]+"/packages"):
1188                         if self.settings.has_key("AUTORESUME") \
1189                                 and os.path.exists(self.settings["autoresume_path"]+"build_packages"):
1190                                         print "Resume point detected, skipping build_packages operation..."
1191                         else:
1192                                 mypack=list_bashify(self.settings[self.settings["spec_prefix"]+"/packages"])
1193                                 try:
1194                                         cmd("/bin/bash "+self.settings["controller_file"]+\
1195                                                 " build_packages "+mypack,"Error in attempt to build packages",env=self.env)
1196                                         touch(self.settings["autoresume_path"]+"build_packages")
1197                                 except CatalystError:
1198                                         self.unbind()
1199                                         raise CatalystError,self.settings["spec_prefix"] + "build aborting due to error."
1200
1201         def build_kernel(self):
1202                 if self.settings.has_key("AUTORESUME") \
1203                 and os.path.exists(self.settings["autoresume_path"]+"build_kernel"):
1204                         print "Resume point detected, skipping build_kernel operation..."
1205                 else:
1206                         if self.settings.has_key("boot/kernel"):
1207                                 try:
1208                                         mynames=self.settings["boot/kernel"]
1209                                         if type(mynames)==types.StringType:
1210                                                 mynames=[mynames]
1211                                         # execute the script that sets up the kernel build environment
1212                                         cmd("/bin/bash "+self.settings["controller_file"]+" pre-kmerge ",\
1213                                                 "Runscript pre-kmerge failed",env=self.env)
1214
1215                                         for kname in mynames:
1216                                                 if self.settings.has_key("AUTORESUME") \
1217                                                         and os.path.exists(self.settings["autoresume_path"]+"build_kernel_"+kname):
1218                                                         print "Resume point detected, skipping build_kernel for "+kname+" operation..."
1219                                                 else:
1220                                                         try:
1221                                                                 if not os.path.exists(self.settings["boot/kernel/"+kname+"/config"]):
1222                                                                         self.unbind()
1223                                                                         raise CatalystError, "Can't find kernel config: " \
1224                                                                                 +self.settings["boot/kernel/"+kname+"/config"]
1225
1226                                                         except TypeError:
1227                                                                 raise CatalystError, "Required value boot/kernel/config not specified"
1228
1229                                                         try:
1230                                                                 cmd("cp "+self.settings["boot/kernel/"+kname+"/config"]+" "+ \
1231                                                                         self.settings["chroot_path"]+"/var/tmp/"+kname+".config", \
1232                                                                         "Couldn't copy kernel config: "+self.settings["boot/kernel/"+kname+"/config"],\
1233                                                                         env=self.env)
1234
1235                                                         except CatalystError:
1236                                                                 self.unbind()
1237
1238                                                         # If we need to pass special options to the bootloader
1239                                                         # for this kernel put them into the environment.
1240                                                         if self.settings.has_key("boot/kernel/"+kname+"/kernelopts"):
1241                                                                 myopts=self.settings["boot/kernel/"+kname+"/kernelopts"]
1242
1243                                                                 if type(myopts) != types.StringType:
1244                                                                         myopts = string.join(myopts)
1245                                                                         self.env[kname+"_kernelopts"]=myopts
1246
1247                                                                 else:
1248                                                                         self.env[kname+"_kernelopts"]=""
1249
1250                                                         if not self.settings.has_key("boot/kernel/"+kname+"/extraversion"):
1251                                                                 self.settings["boot/kernel/"+kname+"/extraversion"]=""
1252
1253                                                         self.env["clst_kextraversion"]=self.settings["boot/kernel/"+kname+"/extraversion"]
1254
1255                                                         if self.settings.has_key("boot/kernel/"+kname+"/initramfs_overlay"):
1256                                                                 if os.path.exists(self.settings["boot/kernel/"+kname+"/initramfs_overlay"]):
1257                                                                         print "Copying initramfs_overlay dir " +self.settings["boot/kernel/"+kname+"/initramfs_overlay"]
1258
1259                                                                         cmd("mkdir -p "+self.settings["chroot_path"]+"/tmp/initramfs_overlay/" + \
1260                                                                                 self.settings["boot/kernel/"+kname+"/initramfs_overlay"],env=self.env)
1261
1262                                                                         cmd("cp -R "+self.settings["boot/kernel/"+kname+"/initramfs_overlay"]+"/* " + \
1263                                                                                 self.settings["chroot_path"] + "/tmp/initramfs_overlay/" + \
1264                                                                                 self.settings["boot/kernel/"+kname+"/initramfs_overlay"],\
1265                                                                                 env=self.env)
1266
1267
1268                                                         # execute the script that builds the kernel
1269                                                         cmd("/bin/bash "+self.settings["controller_file"]+" kernel "+kname,\
1270                                                                 "Runscript kernel build failed",env=self.env)
1271
1272                                                         if self.settings.has_key("boot/kernel/"+kname+"/initramfs_overlay"):
1273                                                                 if os.path.exists(self.settings["chroot_path"]+"/tmp/initramfs_overlay/"):
1274                                                                         print "Cleaning up temporary overlay dir"
1275                                                                         cmd("rm -R "+self.settings["chroot_path"]+"/tmp/initramfs_overlay/",env=self.env)
1276
1277                                                         touch(self.settings["autoresume_path"]+"build_kernel_"+kname)
1278
1279                                                         # execute the script that cleans up the kernel build environment
1280                                                         cmd("/bin/bash "+self.settings["controller_file"]+" post-kmerge ",\
1281                                                                 "Runscript post-kmerge failed",env=self.env)
1282
1283                                         touch(self.settings["autoresume_path"]+"build_kernel")
1284
1285                                 except CatalystError:
1286                                         self.unbind()
1287                                         raise CatalystError,"build aborting due to kernel build error."
1288
1289         def bootloader(self):
1290             if self.settings.has_key("AUTORESUME") \
1291                 and os.path.exists(self.settings["autoresume_path"]+"bootloader"):
1292                     print "Resume point detected, skipping bootloader operation..."
1293             else:
1294                 try:
1295                         cmd("/bin/bash "+self.settings["controller_file"]+" bootloader " + self.settings["target_path"],\
1296                                 "Bootloader runscript failed.",env=self.env)
1297                         touch(self.settings["autoresume_path"]+"bootloader")
1298                 except CatalystError:
1299                         self.unbind()
1300                         raise CatalystError,"Runscript aborting due to error."
1301
1302         def livecd_update(self):
1303             if self.settings.has_key("AUTORESUME") \
1304                 and os.path.exists(self.settings["autoresume_path"]+"livecd_update"):
1305                     print "Resume point detected, skipping build_packages operation..."
1306             else:
1307                 try:
1308                         cmd("/bin/bash "+self.settings["controller_file"]+" livecd-update",\
1309                                 "livecd-update failed.",env=self.env)
1310                         touch(self.settings["autoresume_path"]+"livecd_update")
1311
1312                 except CatalystError:
1313                         self.unbind()
1314                         raise CatalystError,"build aborting due to livecd_update error."
1315
1316         def clear_chroot(self):
1317                 myemp=self.settings["chroot_path"]
1318                 if os.path.isdir(myemp):
1319                     print "Emptying directory",myemp
1320                     # stat the dir, delete the dir, recreate the dir and set
1321                     # the proper perms and ownership
1322                     mystat=os.stat(myemp)
1323                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1324                     if os.uname()[0] == "FreeBSD": # There's no easy way to change flags recursively in python
1325                             os.system("chflags -R noschg "+myemp)
1326                     shutil.rmtree(myemp)
1327                     os.makedirs(myemp,0755)
1328                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1329                     os.chmod(myemp,mystat[ST_MODE])
1330
1331         def clear_packages(self):
1332             if self.settings.has_key("PKGCACHE"):
1333                 print "purging the pkgcache ..."
1334
1335                 myemp=self.settings["pkgcache_path"]
1336                 if os.path.isdir(myemp):
1337                     print "Emptying directory",myemp
1338                     # stat the dir, delete the dir, recreate the dir and set
1339                     # the proper perms and ownership
1340                     mystat=os.stat(myemp)
1341                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1342                     shutil.rmtree(myemp)
1343                     os.makedirs(myemp,0755)
1344                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1345                     os.chmod(myemp,mystat[ST_MODE])
1346
1347         def clear_kerncache(self):
1348             if self.settings.has_key("KERNCACHE"):
1349                 print "purging the kerncache ..."
1350
1351                 myemp=self.settings["kerncache_path"]
1352                 if os.path.isdir(myemp):
1353                     print "Emptying directory",myemp
1354                     # stat the dir, delete the dir, recreate the dir and set
1355                     # the proper perms and ownership
1356                     mystat=os.stat(myemp)
1357                     #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
1358                     shutil.rmtree(myemp)
1359                     os.makedirs(myemp,0755)
1360                     os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1361                     os.chmod(myemp,mystat[ST_MODE])
1362
1363         def clear_autoresume(self):
1364                 # clean resume points since they are no longer needed
1365                 if self.settings.has_key("AUTORESUME"):
1366                         print "Removing AutoResume Points: ..."
1367                 myemp=self.settings["autoresume_path"]
1368                 if os.path.isdir(myemp):
1369                                 if self.settings.has_key("AUTORESUME"):
1370                                         print "Emptying directory",myemp
1371                                 # stat the dir, delete the dir, recreate the dir and set
1372                                 # the proper perms and ownership
1373                                 mystat=os.stat(myemp)
1374                                 if os.uname()[0] == "FreeBSD":
1375                                         cmd("chflags -R noschg "+myemp, "Could not remove immutable flag for file " + myemp)
1376                                 #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
1377                                 shutil.rmtree(myemp)
1378                                 os.makedirs(myemp,0755)
1379                                 os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
1380                                 os.chmod(myemp,mystat[ST_MODE])
1381
1382         def gen_digest_file(self,file):
1383                 if os.path.exists(file+".DIGESTS"):
1384                         os.remove(file+".DIGESTS")
1385                 if self.settings.has_key("digests"):
1386                         if os.path.exists(file):
1387                                 myf=open(file+".DIGESTS","w")
1388                                 keys={}
1389                                 for i in self.settings["digests"].split():
1390                                         keys[i]=1
1391                                         array=keys.keys()
1392                                         array.sort()
1393                                 for j in array:
1394                                         if self.settings.has_key("VERBOSE"):
1395                                                 hash=generate_hash(file,hash_function=j,verbose=True)
1396                                         else:
1397                                                 hash=generate_hash(file,hash_function=j)
1398                                         myf.write(hash)
1399                                 myf.close()
1400
1401         def purge(self):
1402             countdown(10,"Purging Caches ...")
1403             if self.settings.has_key("PURGE"):
1404                 print "clearing autoresume ..."
1405                 self.clear_autoresume()
1406
1407                 print "clearing chroot ..."
1408                 self.clear_chroot()
1409
1410                 print "clearing package cache ..."
1411                 self.clear_packages()
1412
1413                 print "clearing kerncache ..."
1414                 self.clear_kerncache()
1415
1416 #vim: ts=4 sw=4 sta et sts=4 ai