From 53bf5b316276f363f377d65d71b24cc31ec6d031 Mon Sep 17 00:00:00 2001 From: Eric Edgar Date: Wed, 27 Jul 2005 20:30:50 +0000 Subject: [PATCH] Add support to cache the snapshot dir. add snapcache to options. add snapshot_cache= to override the default location of the cache in catalyst.conf (eg snapshot_cache="/mnt/catalyst/snapshot") git-svn-id: svn+ssh://svn.gentoo.org/var/svnroot/catalyst/trunk@814 d1e1f19c-881f-0410-ab34-b69fee027534 --- ChangeLog | 8 ++++- catalyst | 9 +++-- modules/catalyst_support.py | 60 ++----------------------------- modules/generic_stage_target.py | 62 +++++++++++++++++++++++++++++---- 4 files changed, 72 insertions(+), 67 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca5fbe6f..b1e7e0d6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,11 @@ # Copyright 2002-2005 Gentoo Foundation; Distributed under the GPL v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.310 2005/07/27 19:17:18 wolf31o2 Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.311 2005/07/27 20:30:50 rocket Exp $ + + 27 Jul 2005; Eric Edgar catalyst, + modules/catalyst_support.py, modules/generic_stage_target.py: + Add support to cache the snapshot dir. add snapcache to options. add + snapshot_cache= to override the default location of the cache in + catalyst.conf (eg snapshot_cache="/mnt/catalyst/snapshot") 27 Jul 2005; Chris Gianelloni +livecd/files/Getting_Online.txt, +livecd/files/README.txt: diff --git a/catalyst b/catalyst index 551b3065..ccb42d0c 100755 --- a/catalyst +++ b/catalyst @@ -1,7 +1,7 @@ #!/usr/bin/python # Copyright 1999-2005 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/catalyst,v 1.86 2005/07/06 18:28:12 rocket Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/catalyst,v 1.87 2005/07/27 20:30:50 rocket Exp $ # Maintained in full by: # Eric Edgar @@ -52,7 +52,8 @@ def parse_config(myconfig): confdefaults={ "storedir":"/var/tmp/catalyst",\ "sharedir":"/usr/share/catalyst","distdir":"/usr/portage/distfiles",\ - "portdir":"/usr/portage","options":""} + "portdir":"/usr/portage","options":"",\ + "snapshot_cache":"/var/tmp/catalyst/snapshot_cache" } # first, try the one passed (presumably from the cmdline) if myconfig: @@ -100,6 +101,10 @@ def parse_config(myconfig): if "pkgcache" in string.split(conf_values["options"]): print "Package cache support enabled." conf_values["PKGCACHE"]="1" + + if "snapcache" in string.split(conf_values["options"]): + print "Snapshot cache support enabled." + conf_values["SNAPCACHE"]="1" if "kerncache" in string.split(conf_values["options"]): print "Kernel cache support enabled." diff --git a/modules/catalyst_support.py b/modules/catalyst_support.py index 1a7e1916..92ba97a7 100644 --- a/modules/catalyst_support.py +++ b/modules/catalyst_support.py @@ -1,6 +1,6 @@ # Copyright 1999-2005 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.51 2005/07/07 21:35:00 rocket Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.52 2005/07/27 20:30:50 rocket Exp $ import sys,string,os,types,re,signal,traceback,md5,time selinux_capable = False @@ -112,6 +112,8 @@ valid_config_file_values.append("options") valid_config_file_values.append("DEBUG") valid_config_file_values.append("VERBOSE") valid_config_file_values.append("PURGE") +valid_config_file_values.append("SNAPCACHE") +valid_config_file_values.append("snapshot_cache") verbosity=1 @@ -411,62 +413,6 @@ def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,retur cleanup(mypid) return 0 - - - - - - - - - - -#def spawn(mystring,debug=0,fd_pipes=None): -# """ -# apparently, os.system mucks up return values, so this code -# should fix that. -# -# Taken from portage.py - thanks to carpaski@gentoo.org -# """ -# print "Running command \""+mystring+"\"" -# myargs=[] -# mycommand = "/bin/bash" -# if debug: -# myargs=["bash","-x","-c",mystring] -# else: -# myargs=["bash","-c",mystring] -# -# mypid=os.fork() -# if mypid==0: -# if fd_pipes: -# os.dup2(fd_pipes[0], 0) # stdin -- (Read)/Write -# os.dup2(fd_pipes[1], 1) # stdout -- Read/(Write) -# os.dup2(fd_pipes[2], 2) # stderr -- Read/(Write) -# try: -# os.execvp(mycommand,myargs) -# except Exception, e: -# raise CatalystError,myexc -# - # If the execve fails, we need to report it, and exit - # *carefully* --- report error here -# os._exit(1) -# sys.exit(1) -# return # should never get reached -# try: -# retval=os.waitpid(mypid,0)[1] -# if (retval & 0xff)==0: -# return (retval >> 8) # return exit code -# else: -# return ((retval & 0xff) << 8) # interrupted by signal -# -# except: -# os.kill(mypid,signal.SIGTERM) -# if os.waitpid(mypid,os.WNOHANG)[1] == 0: -# # feisty bugger, still alive. -# os.kill(mypid,signal.SIGKILL) -# raise -# - def cmd(mycmd,myexc=""): try: sys.stdout.flush() diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py index 3383c599..b0eb1d44 100644 --- a/modules/generic_stage_target.py +++ b/modules/generic_stage_target.py @@ -1,6 +1,6 @@ # Copyright 1999-2005 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.51 2005/07/06 18:48:03 rocket Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.52 2005/07/27 20:30:50 rocket Exp $ """ This class does all of the chroot setup, copying of files, etc. It is @@ -105,6 +105,7 @@ class generic_stage_target(generic_target): # set paths self.set_snapshot_path() + self.set_snapcache_path() self.set_source_path() self.set_chroot_path() self.set_autoresume_path() @@ -140,9 +141,16 @@ class generic_stage_target(generic_target): file_locate(self.settings,["portage_confdir"],expand=0) # setup our mount points - self.mounts=[ "/proc","/dev","/dev/pts","/usr/portage/distfiles" ] - self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\ - "/usr/portage/distfiles":self.settings["distdir"]} + if self.settings.has_key("SNAPCACHE"): + self.mounts=[ "/proc","/dev","/dev/pts","/usr/portage","/usr/portage/distfiles" ] + self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\ + "/usr/portage":self.settings["snapshot_cache_path"]+"/portage",\ + "/usr/portage/distfiles":self.settings["distdir"]} + else: + self.mounts=[ "/proc","/dev","/dev/pts","/usr/portage/distfiles" ] + self.mountmap={"/proc":"/proc", "/dev":"/dev", "/dev/pts":"/dev/pts",\ + "/usr/portage/distfiles":self.settings["distdir"]} + self.set_mounts() # configure any user specified options (either in catalyst.conf or on the cmdline) @@ -311,6 +319,11 @@ class generic_stage_target(generic_target): if os.path.exists(self.settings["snapshot_path"]): self.settings["snapshot_path_md5sum"]=calc_md5(self.settings["snapshot_path"]) + def set_snapcache_path(self): + if self.settings.has_key("SNAPCACHE"): + self.settings["snapshot_cache_path"]=self.settings["snapshot_cache"]+"/"+self.settings["snapshot"]+"/" + print "Caching snapshot to " + self.settings["snapshot_cache_path"] + def set_chroot_path(self): self.settings["chroot_path"]=self.settings["storedir"]+"/tmp/"+self.settings["target_subpath"]+"/" @@ -485,7 +498,41 @@ class generic_stage_target(generic_target): myf.write(self.settings["source_path_md5sum"]) myf.close() - def unpack_snapshot(self): + def unpack_snapshot(self): + + if self.settings.has_key("SNAPCACHE"): + if os.path.exists(self.settings["snapshot_cache_path"]+"catalyst-md5sum"): + snapshot_cache_md5sum=read_from_clst(self.settings["snapshot_cache_path"]+"catalyst-md5sum") + if self.settings["snapshot_path_md5sum"] == snapshot_cache_md5sum: + print "Valid snapshot cache, skipping unpack of portage tree..." + else: + print "Cleaning up invalid cache at "+self.settings["snapshot_cache_path"] + + cmd("rm -rf "+self.settings["snapshot_cache_path"],\ + "Error removing existing snapshot directory.") + if not os.path.exists(self.settings["snapshot_cache_path"]): + os.makedirs(self.settings["snapshot_cache_path"],0755) + print "Unpacking portage tree to snapshot cache ..." + cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+\ + self.settings["snapshot_cache_path"],"Error unpacking snapshot") + myf=open(self.settings["snapshot_cache_path"]+"catalyst-md5sum","w") + myf.write(self.settings["snapshot_path_md5sum"]) + myf.close() + else: + if os.path.exists(self.settings["snapshot_cache_path"]): + print "Cleaning up existing snapshot cache ..." + cmd("rm -rf "+self.settings["snapshot_cache_path"],\ + "Error removing existing snapshot directory.") + + if not os.path.exists(self.settings["snapshot_cache_path"]): + os.makedirs(self.settings["snapshot_cache_path"],0755) + print "Unpacking portage tree to snapshot cache ..." + cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+\ + self.settings["snapshot_cache_path"],"Error unpacking snapshot") + myf=open(self.settings["snapshot_cache_path"]+"catalyst-md5sum","w") + myf.write(self.settings["snapshot_path_md5sum"]) + myf.close() + else: if os.path.exists(self.settings["autoresume_path"]+"unpack_portage"): clst_unpack_portage_md5sum=read_from_clst(self.settings["autoresume_path"]+"unpack_portage") @@ -497,8 +544,8 @@ class generic_stage_target(generic_target): else: if os.path.exists(self.settings["chroot_path"]+"/usr/portage"): print "Cleaning up existing portage tree ..." - cmd("rm -rf "+self.settings["chroot_path"]+"/usr/portage",\ - "Error removing existing snapshot directory.") + cmd("rm -rf "+self.settings["chroot_path"]+"/usr/portage",\ + "Error removing existing snapshot directory.") print "Unpacking portage tree ..." cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+\ @@ -567,6 +614,7 @@ class generic_stage_target(generic_target): if retval!=0: self.unbind() raise CatalystError,"Couldn't bind mount "+src + def unbind(self): ouch=0 -- 2.26.2