Add support to cache the snapshot dir. add snapcache to options. add snapshot_cache...

git-svn-id: svn+ssh://svn.gentoo.org/var/svnroot/catalyst/trunk@814 d1e1f19c-881f-0410-ab34-b69fee027534

diff --git a/ChangeLog b/ChangeLog
index ca5fbe6f1e29d29fdad88055981a349d516cf678..b1e7e0d6be493499e731fc392a86cbadfdb7b265 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
 # Copyright 2002-2005 Gentoo Foundation; Distributed under the GPL v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.310 2005/07/27 19:17:18 wolf31o2 Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.311 2005/07/27 20:30:50 rocket Exp $
+
+  27 Jul 2005; Eric Edgar <rocket@gentoo.org> catalyst,
+  modules/catalyst_support.py, modules/generic_stage_target.py:
+  Add support to cache the snapshot dir. add snapcache to options. add
+  snapshot_cache= to override the default location of the cache in
+  catalyst.conf (eg snapshot_cache="/mnt/catalyst/snapshot")
 
   27 Jul 2005; Chris Gianelloni <wolf31o2@gentoo.org>
   +livecd/files/Getting_Online.txt, +livecd/files/README.txt:

diff --git a/catalyst b/catalyst
index 551b3065472798d554d8a6db79176184585159f9..ccb42d0c48a12827abd5cc3ea78b4836355c06cb 100755 (executable)
--- a/catalyst
+++ b/catalyst
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # Copyright 1999-2005 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/catalyst,v 1.86 2005/07/06 18:28:12 rocket Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/catalyst,v 1.87 2005/07/27 20:30:50 rocket Exp $
 
 # Maintained in full by:
 # Eric Edgar <rocket@gentoo.org>
@@ -52,7 +52,8 @@ def parse_config(myconfig):
 
        confdefaults={ "storedir":"/var/tmp/catalyst",\
                "sharedir":"/usr/share/catalyst","distdir":"/usr/portage/distfiles",\
-               "portdir":"/usr/portage","options":""}
+               "portdir":"/usr/portage","options":"",\
+               "snapshot_cache":"/var/tmp/catalyst/snapshot_cache" }
                
        # first, try the one passed (presumably from the cmdline)
        if myconfig:
@@ -100,6 +101,10 @@ def parse_config(myconfig):
        if "pkgcache" in string.split(conf_values["options"]):
                print "Package cache support enabled."
                conf_values["PKGCACHE"]="1"
+       
+       if "snapcache" in string.split(conf_values["options"]):
+               print "Snapshot cache support enabled."
+               conf_values["SNAPCACHE"]="1"
 
        if "kerncache" in string.split(conf_values["options"]):
                print "Kernel cache support enabled."

diff --git a/modules/catalyst_support.py b/modules/catalyst_support.py
index 1a7e19161695c21cb6428946231e0b75f716ad69..92ba97a7413b350443c907b460d7cf9be3949665 100644 (file)
--- a/modules/catalyst_support.py
+++ b/modules/catalyst_support.py
@@ -1,6 +1,6 @@
 # Copyright 1999-2005 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.51 2005/07/07 21:35:00 rocket Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.52 2005/07/27 20:30:50 rocket Exp $
 
 import sys,string,os,types,re,signal,traceback,md5,time
 selinux_capable = False
@@ -112,6 +112,8 @@ valid_config_file_values.append("options")
 valid_config_file_values.append("DEBUG")
 valid_config_file_values.append("VERBOSE")
 valid_config_file_values.append("PURGE")
+valid_config_file_values.append("SNAPCACHE")
+valid_config_file_values.append("snapshot_cache")
 
 verbosity=1
 
@@ -411,62 +413,6 @@ def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,retur
         cleanup(mypid)
         return 0
 
-
-
-
-
-
-
-
-
-
-
-#def spawn(mystring,debug=0,fd_pipes=None):
-#      """
-#      apparently, os.system mucks up return values, so this code
-#      should fix that.
-#
-#      Taken from portage.py - thanks to carpaski@gentoo.org
-#      """
-#      print "Running command \""+mystring+"\""
-#      myargs=[]
-#      mycommand = "/bin/bash"
-#      if debug:
-#              myargs=["bash","-x","-c",mystring]
-#      else:
-#              myargs=["bash","-c",mystring]
-#      
-#      mypid=os.fork()
-#      if mypid==0:
-#              if fd_pipes:
-#                      os.dup2(fd_pipes[0], 0) # stdin  -- (Read)/Write
-#                      os.dup2(fd_pipes[1], 1) # stdout -- Read/(Write)
-#                      os.dup2(fd_pipes[2], 2) # stderr -- Read/(Write)
-#              try:
-#                      os.execvp(mycommand,myargs)
-#              except Exception, e:
-#                      raise CatalystError,myexc
-#              
-               # If the execve fails, we need to report it, and exit
-               # *carefully* --- report error here
-#              os._exit(1)
-#              sys.exit(1)
-#              return # should never get reached
-#      try:
-#              retval=os.waitpid(mypid,0)[1]
-#              if (retval & 0xff)==0:
-#                      return (retval >> 8) # return exit code
-#              else:
-#                      return ((retval & 0xff) << 8) # interrupted by signal
-#      
-#      except: 
-#              os.kill(mypid,signal.SIGTERM)
-#              if os.waitpid(mypid,os.WNOHANG)[1] == 0:
-#              # feisty bugger, still alive.
-#                      os.kill(mypid,signal.SIGKILL)
-#              raise
-#
-
 def cmd(mycmd,myexc=""):
        try:
                sys.stdout.flush()

diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py
index 3383c59968f3890ff5b302b3ea82a3c8d2e2c487..b0eb1d441f07003216e26681914cd9fc93d64bd8 100644 (file)
--- a/modules/generic_stage_target.py
+++ b/modules/generic_stage_target.py
@@ -1,6 +1,6 @@
 # Copyright 1999-2005 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.51 2005/07/06 18:48:03 rocket Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.52 2005/07/27 20:30:50 rocket Exp $
 
 """
 This class does all of the chroot setup, copying of files, etc. It is
@@ -105,6 +105,7 @@ class generic_stage_target(generic_target):
 
                # set paths
                self.set_snapshot_path()
+               self.set_snapcache_path()
                self.set_source_path()
                self.set_chroot_path()
                self.set_autoresume_path()
@@ -140,9 +141,16 @@ class generic_stage_target(generic_target):
                        file_locate(self.settings,["portage_confdir"],expand=0)
                
                # setup our mount points
-               self.mounts=[ "/proc","/dev","/dev/pts","/usr/portage/distfiles" ]
-               self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
-                       "/usr/portage/distfiles":self.settings["distdir"]}
+               if self.settings.has_key("SNAPCACHE"):
+                       self.mounts=[ "/proc","/dev","/dev/pts","/usr/portage","/usr/portage/distfiles" ]
+                       self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
+                               "/usr/portage":self.settings["snapshot_cache_path"]+"/portage",\
+                               "/usr/portage/distfiles":self.settings["distdir"]}
+               else:
+                       self.mounts=[ "/proc","/dev","/dev/pts","/usr/portage/distfiles" ]
+                       self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\
+                               "/usr/portage/distfiles":self.settings["distdir"]}
+
                self.set_mounts()
 
                # configure any user specified options (either in catalyst.conf or on the cmdline)
@@ -311,6 +319,11 @@ class generic_stage_target(generic_target):
                if os.path.exists(self.settings["snapshot_path"]):
                    self.settings["snapshot_path_md5sum"]=calc_md5(self.settings["snapshot_path"])
        
+       def set_snapcache_path(self):
+               if self.settings.has_key("SNAPCACHE"):
+                   self.settings["snapshot_cache_path"]=self.settings["snapshot_cache"]+"/"+self.settings["snapshot"]+"/"
+                   print "Caching snapshot to " + self.settings["snapshot_cache_path"]
+       
        def set_chroot_path(self):
                self.settings["chroot_path"]=self.settings["storedir"]+"/tmp/"+self.settings["target_subpath"]+"/"
        
@@ -485,7 +498,41 @@ class generic_stage_target(generic_target):
                                    myf.write(self.settings["source_path_md5sum"])
                                    myf.close()
        
-       def unpack_snapshot(self):                      
+       def unpack_snapshot(self):              
+
+               if self.settings.has_key("SNAPCACHE"):
+                       if os.path.exists(self.settings["snapshot_cache_path"]+"catalyst-md5sum"):
+                               snapshot_cache_md5sum=read_from_clst(self.settings["snapshot_cache_path"]+"catalyst-md5sum")
+                               if self.settings["snapshot_path_md5sum"] == snapshot_cache_md5sum:
+                                   print "Valid snapshot cache, skipping unpack of portage tree..."
+                               else:
+                                   print "Cleaning up invalid cache at "+self.settings["snapshot_cache_path"]
+
+                                   cmd("rm -rf "+self.settings["snapshot_cache_path"],\
+                                          "Error removing existing snapshot directory.")
+                                   if not os.path.exists(self.settings["snapshot_cache_path"]):
+                                       os.makedirs(self.settings["snapshot_cache_path"],0755)
+                                   print "Unpacking portage tree to snapshot cache ..."
+                                   cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+\
+                                       self.settings["snapshot_cache_path"],"Error unpacking snapshot")
+                                   myf=open(self.settings["snapshot_cache_path"]+"catalyst-md5sum","w")
+                                   myf.write(self.settings["snapshot_path_md5sum"])
+                                   myf.close()
+                       else:
+                           if os.path.exists(self.settings["snapshot_cache_path"]):
+                               print "Cleaning up existing snapshot cache  ..."
+                               cmd("rm -rf "+self.settings["snapshot_cache_path"],\
+                                          "Error removing existing snapshot directory.")
+                           
+                           if not os.path.exists(self.settings["snapshot_cache_path"]):
+                               os.makedirs(self.settings["snapshot_cache_path"],0755)
+                           print "Unpacking portage tree to snapshot cache ..."
+                           cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+\
+                               self.settings["snapshot_cache_path"],"Error unpacking snapshot")
+                           myf=open(self.settings["snapshot_cache_path"]+"catalyst-md5sum","w")
+                           myf.write(self.settings["snapshot_path_md5sum"])
+                           myf.close()
+               else:
                        if os.path.exists(self.settings["autoresume_path"]+"unpack_portage"):
                            clst_unpack_portage_md5sum=read_from_clst(self.settings["autoresume_path"]+"unpack_portage")
                        
@@ -497,8 +544,8 @@ class generic_stage_target(generic_target):
                        else:
                                if os.path.exists(self.settings["chroot_path"]+"/usr/portage"):
                                        print "Cleaning up existing portage tree ..."
-                               cmd("rm -rf "+self.settings["chroot_path"]+"/usr/portage",\
-                                       "Error removing existing snapshot directory.")
+                                       cmd("rm -rf "+self.settings["chroot_path"]+"/usr/portage",\
+                                          "Error removing existing snapshot directory.")
                        
                                print "Unpacking portage tree ..."
                                cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+\
@@ -567,6 +614,7 @@ class generic_stage_target(generic_target):
                        if retval!=0:
                                self.unbind()
                                raise CatalystError,"Couldn't bind mount "+src
+                           
        
        def unbind(self):
                ouch=0