# Copyright 2002-2005 Gentoo Foundation; Distributed under the GPL v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.469 2005/12/19 19:10:04 wolf31o2 Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.470 2005/12/19 20:53:13 rocket Exp $
+
+ 19 Dec 2005; Eric Edgar <rocket@gentoo.org> catalyst, files/catalyst.conf,
+ modules/catalyst_support.py, modules/generic_stage_target.py,
+ modules/livecd_stage2_target.py, modules/stage2_target.py:
+ Change the internal hash checking to be quicker and more memory efficient.
+ Add additional hash digests options.
19 Dec 2005; Chris Gianelloni <wolf31o2@gentoo.org>
targets/tinderbox/tinderbox-chroot.sh,
#!/usr/bin/python -OO
# Copyright 1999-2005 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/catalyst,v 1.115 2005/12/19 15:48:00 rocket Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/catalyst,v 1.116 2005/12/19 20:53:13 rocket Exp $
# Maintained in full by:
# Eric Edgar <rocket@gentoo.org>
import os,sys,imp,string,getopt
import pdb
+
__maintainer__="Chris Gianelloni <wolf31o2@gentoo.org>"
__version__="2.0_rc9"
confdefaults={ "storedir":"/var/tmp/catalyst",\
"sharedir":"/usr/share/catalyst","distdir":"/usr/portage/distfiles",\
"portdir":"/usr/portage","options":"",\
- "snapshot_cache":"/var/tmp/catalyst/snapshot_cache" }
+ "snapshot_cache":"/var/tmp/catalyst/snapshot_cache",\
+ "hash_function":"crc32"}
# first, try the one passed (presumably from the cmdline)
if myconfig:
print "Envscript support enabled."
conf_values["ENVSCRIPT"]=myconf["envscript"]
- if "md5" in string.split(conf_values["options"]):
- print "MD5 .digests file creation support enabled."
- conf_values["MD5"]="1"
-
- if "sha" in string.split(conf_values["options"]):
- print "SHA .digests file creation support enabled."
- conf_values["SHA"]="1"
+ if myconf.has_key("digests"):
+ conf_values["digests"]=myconf["digests"]
def import_modules():
# import catalyst's own modules (i.e. catalyst_support and the arch modules)
parse_config(myconfig)
sys.path.append(conf_values["sharedir"]+"/modules")
from catalyst_support import *
-
+
+ # Start checking that digests are valid now that the hash_map was imported from catalyst_support
+ if conf_values.has_key("digests"):
+ for i in conf_values["digests"].split():
+ if not hash_map.has_key(i):
+ print "Valid digest entries:"
+ print hash_map.keys()
+ raise CatalystError, i+" is not a valid digest entry"
+
# import the rest of the catalyst modules
targetmap=import_modules()
# Copyright 1999-2005 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/files/catalyst.conf,v 1.19 2005/12/09 17:22:57 wolf31o2 Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/files/catalyst.conf,v 1.20 2005/12/19 20:53:13 rocket Exp $
# Simple desriptions of catalyst settings. Please refer to the online
# documentation for more information.
# WARNING: moving parts of the portage tree from within fsscript *will* break
# your cache. The cache is unlinked before any empty or rm processing, though.
#
-# DIGESTS CREATION
-# md5 = Create a .digests file containing the md5 of the output object
-# sha = Create a .digests file containing the sha1 of the output object
# (These options can be used together)
-options="autoresume kerncache md5 pkgcache seedcache sha snapcache"
+options="autoresume kerncache pkgcache seedcache snapcache"
# sharedir specifies where all of the catalyst runtime executables are. Most
# users do not need to change this.
# enabled in the options.
# snapshot_cache="<path_to_new_cache_dir>"
+# Internal hash function catalyst should use for things like autoresume,seedcache,etc.
+# crc32 is the default and fastest. You shouldnt ever need to change this unless your os
+# doesnt support it.
+# Supported options: sha1, sha224, ripemd128, ripemd320, sha384, crc32, ripemd256, sha256, sha512, ripemd160, md5
+# hash_function="crc32"
+
+# DIGESTS CREATION
+# Creates a .digests file containing the hash output from any of the supported options below. Adding them all
+# May take a long time.
+# Supported options: sha1, sha224, ripemd128, ripemd320, sha384, crc32, ripemd256, sha256, sha512, ripemd160, md5
+# digests="sha1 sha224 ripemd128 ripemd320 sha384 crc32 ripemd256 sha256 sha512 ripemd160 md5"
+
# envscript allows users to set options such as http proxies, MAKEOPTS,
# GENTOO_MIRRORS, or any other environment variables needed for building.
# The envscript file sets environment variables like so:
# Copyright 1999-2005 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.63 2005/12/05 18:13:12 rocket Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.64 2005/12/19 20:53:13 rocket Exp $
-import sys,string,os,types,re,signal,traceback,md5,sha,time
+import sys,string,os,types,re,signal,traceback,time
+#import md5,sha
selinux_capable = False
#userpriv_capable = (os.getuid() == 0)
#fakeroot_capable = False
return r
# hexify()
-# A function to calculate the md5 sum of a file
-def calc_md5(file,verbose=False):
- m = md5.new()
- f = open(file, 'r')
- for line in f.readlines():
- m.update(line)
- f.close()
- md5sum = hexify(m.digest())
- if verbose:
- print "MD5 (%s) = %s" % (file, md5sum)
- return md5sum
-# calc_md5
-
-def calc_sha(file,verbose=False):
- m = sha.new()
- f = open(file, 'r')
- for line in f.readlines():
- m.update(line)
- f.close()
- shaval = hexify(m.digest())
- if verbose:
- print "SHA (%s) = %s" % (file, shaval)
- return shaval
-
+def generate_hash(file,hash_function="crc32",verbose=False):
+ try:
+ return hash_map[hash_function][0](file,hash_map[hash_function][1],hash_map[hash_function][2],verbose)
+ except:
+ raise CatalystError,"Error generating hash, is appropriate utility installed on your system?"
+
+def calc_hash(file,cmd,id_string="MD5",verbose=False):
+ a=os.popen(cmd+" "+file)
+ mylines=a.readlines()
+ a.close()
+ mylines=mylines[0].split()
+ result=mylines[0]
+ if verbose:
+ print id_string+" (%s) = %s" % (file, result)
+ return result
+
+def calc_hash2(file,cmd,id_string="MD5",verbose=False):
+ a=os.popen(cmd+" "+file)
+ a.readline()
+ mylines=a.readline().split()
+ a.close()
+ result=mylines[0]
+ if verbose:
+ print id_string+" (%s) = %s" % (file, result)
+ return result
+
+#This has map must be defined after the function calc_hash
+#It is possible to call different functions from this but they must be defined before hash_map
+hash_map={"md5":[calc_hash,"md5sum","MD5"],\
+ "crc32":[calc_hash,"crc32","CRC32"],\
+ "sha1":[calc_hash,"sha1sum","SHA1"],\
+ "sha224":[calc_hash2,"shash -a SHA224","SHA224"],\
+ "sha256":[calc_hash2,"shash -a SHA256","SHA256"],\
+ "sha384":[calc_hash2,"shash -a SHA384","SHA384"],\
+ "sha512":[calc_hash2,"shash -a SHA512","SHA512"],\
+ "ripemd128":[calc_hash2,"shash -a RIPEMD128","RIPEMD128"],\
+ "ripemd160":[calc_hash2,"shash -a RIPEMD160","RIPEMD160"],\
+ "ripemd256":[calc_hash2,"shash -a RIPEMD256","RIPEMD256"],\
+ "ripemd320":[calc_hash2,"shash -a RIPEMD320","RIPEMD320"]}
+
def read_from_clst(file):
line = ''
myline = ''
valid_config_file_values.append("DISTCC")
valid_config_file_values.append("ENVSCRIPT")
valid_config_file_values.append("AUTORESUME")
-valid_config_file_values.append("SHA")
-valid_config_file_values.append("MD5")
valid_config_file_values.append("FETCH")
valid_config_file_values.append("CLEAR_AUTORESUME")
valid_config_file_values.append("options")
valid_config_file_values.append("PURGE")
valid_config_file_values.append("SNAPCACHE")
valid_config_file_values.append("snapshot_cache")
+valid_config_file_values.append("hash_function")
+valid_config_file_values.append("digests")
valid_config_file_values.append("SEEDCACHE")
verbosity=1
# Copyright 1999-2005 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.100 2005/12/16 14:53:29 rocket Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.101 2005/12/19 20:53:13 rocket Exp $
"""
This class does all of the chroot setup, copying of files, etc. It is
# this next line checks to make sure that the specified variables exist on disk.
#pdb.set_trace()
file_locate(self.settings,["source_path","snapshot_path","distdir"],expand=0)
-
# if we are using portage_confdir, check that as well
if self.settings.has_key("portage_confdir"):
file_locate(self.settings,["portage_confdir"],expand=0)
if os.path.isfile(self.settings["source_path"]):
if os.path.exists(self.settings["source_path"]):
- self.settings["source_path_md5sum"]=calc_md5(self.settings["source_path"],True)
- self.settings["source_path_sha"]=calc_sha(self.settings["source_path"],True)
+ self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\
+ hash_function=self.settings["hash_function"],verbose=False)
if os.path.isdir(self.settings["source_path"]):
print "Source path set to "+self.settings["source_path"]
print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf"
def set_snapshot_path(self):
self.settings["snapshot_path"]=normpath(self.settings["storedir"]+"/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
if os.path.exists(self.settings["snapshot_path"]):
- self.settings["snapshot_path_md5sum"]=calc_md5(self.settings["snapshot_path"],True)
- self.settings["snapshot_path_sha"]=calc_sha(self.settings["snapshot_path"],True)
+ self.settings["snapshot_path_hash"]=generate_hash(self.settings["snapshot_path"],\
+ hash_function=self.settings["hash_function"],verbose=False)
def set_snapcache_path(self):
if self.settings.has_key("SNAPCACHE"):
def unpack(self):
unpack=True
- clst_unpack_md5sum=read_from_clst(self.settings["autoresume_path"]+"unpack")
+ clst_unpack_hash=read_from_clst(self.settings["autoresume_path"]+"unpack")
if self.settings.has_key("SEEDCACHE") and os.path.isdir(self.settings["source_path"]):
unpack_cmd="rsync -a --delete "+self.settings["source_path"]+" "+self.settings["chroot_path"]
invalid_snapshot=False
# Autoresume is Valid, Tarball is Valid
- elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_md5sum"] == clst_unpack_md5sum:
+ elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] == clst_unpack_hash:
unpack=False
invalid_snapshot=True
invalid_snapshot=False
# Autoresume is InValid, Tarball
- elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_md5sum"] != clst_unpack_md5sum:
+ elif os.path.isfile(self.settings["source_path"]) and self.settings["source_path_hash"] != clst_unpack_hash:
unpack=True
invalid_snapshot=True
else:
print display_msg
cmd(unpack_cmd,error_msg,env=self.env)
- if self.settings.has_key("source_path_md5sum"):
+ if self.settings.has_key("source_path_hash"):
myf=open(self.settings["autoresume_path"]+"unpack","w")
- myf.write(self.settings["source_path_md5sum"])
+ myf.write(self.settings["source_path_hash"])
myf.close()
else:
touch(self.settings["autoresume_path"]+"unpack")
def unpack_snapshot(self):
unpack=True
- snapshot_md5sum=read_from_clst(self.settings["autoresume_path"]+"unpack_portage")
+ snapshot_hash=read_from_clst(self.settings["autoresume_path"]+"unpack_portage")
if self.settings.has_key("SNAPCACHE"):
- snapshot_cache_md5sum=read_from_clst(self.settings["snapshot_cache_path"]+"catalyst-md5sum")
+ snapshot_cache_hash=read_from_clst(self.settings["snapshot_cache_path"]+"catalyst-hash")
destdir=self.settings["snapshot_cache_path"]
unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+destdir
unpack_errmsg="Error unpacking snapshot"
cleanup_errmsg="Error removing existing snapshot cache directory."
self.snapshot_lock_object=self.snapcache_lock
- if self.settings["snapshot_path_md5sum"] == snapshot_cache_md5sum:
+ if self.settings["snapshot_path_hash"] == snapshot_cache_hash:
print "Valid snapshot cache, skipping unpack of portage tree ..."
unpack=False
if self.settings.has_key("AUTORESUME") \
and os.path.exists(self.settings["chroot_path"]+"/usr/portage/") \
and os.path.exists(self.settings["autoresume_path"]+"unpack_portage") \
- and self.settings["snapshot_path_md5sum"] == snapshot_md5sum:
+ and self.settings["snapshot_path_hash"] == snapshot_hash:
print "Valid Resume point detected, skipping unpack of portage tree..."
unpack=False
cmd(unpack_cmd,unpack_errmsg,env=self.env)
if self.settings.has_key("SNAPCACHE"):
- myf=open(self.settings["snapshot_cache_path"]+"catalyst-md5sum","w")
- myf.write(self.settings["snapshot_path_md5sum"])
+ myf=open(self.settings["snapshot_cache_path"]+"catalyst-hash","w")
+ myf.write(self.settings["snapshot_path_hash"])
myf.close()
else:
print "Setting snapshot autoresume point"
myf=open(self.settings["autoresume_path"]+"unpack_portage","w")
- myf.write(self.settings["snapshot_path_md5sum"])
+ myf.write(self.settings["snapshot_path_hash"])
myf.close()
if self.settings.has_key("SNAPCACHE"):
cmd("tar cjf "+self.settings["target_path"]+" -C "+self.settings["stage_path"]+\
" .","Couldn't create stage tarball",env=self.env)
- self.gen_digest_file(self.settings["target_path"]+".digests")
+ self.gen_digest_file(self.settings["target_path"])
touch(self.settings["autoresume_path"]+"capture")
os.chmod(myemp,mystat[ST_MODE])
def gen_digest_file(self,file):
- if os.path.exists(file+".digests"):
- os.remove(file+".digests")
- if self.settings.has_key("SHA") or self.settings.has_key("MD5"):
- if os.path.exists(file):
- myf=open(file+".digests","w")
-
- if self.settings.has_key("MD5"):
- if self.settings.has_key("VERBOSE"):
- md5=calc_md5(file,True)
- else:
- md5=calc_md5(file)
- myf.write("MD5: "+md5+"\n")
-
- if self.settings.has_key("SHA"):
- if self.settings.has_key("VERBOSE"):
- sha=calc_sha(file,True)
- else:
- sha=calc_sha(file)
- myf.write("SHA: "+sha+"\n")
-
- myf.close()
+ if os.path.exists(file+".digests"):
+ os.remove(file+".digests")
+ if self.settings.has_key("digests"):
+ if os.path.exists(file):
+ myf=open(file+".digests","w")
+ for i in self.settings["digests"].split():
+
+ if self.settings.has_key("VERBOSE"):
+ hash=generate_hash(file,hash_function=i,verbose=True)
+ else:
+ hash=generate_hash(file,hash_function=i)
+ myf.write(hash_map[i][2]+": "+hash+"\n")
+ myf.close()
def purge(self):
countdown(10,"Purging Caches ...")
# Copyright 1999-2005 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/modules/livecd_stage2_target.py,v 1.57 2005/12/16 14:53:29 rocket Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/modules/livecd_stage2_target.py,v 1.58 2005/12/19 20:53:13 rocket Exp $
"""
Builder class for a LiveCD stage2 build.
def set_source_path(self):
self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2")
if os.path.isfile(self.settings["source_path"]):
- self.settings["source_path_md5sum"]=calc_md5(self.settings["source_path"])
+ self.settings["source_path_hash"]=generate_hash(self.settings["source_path"])
else:
self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")
if not os.path.exists(self.settings["source_path"]):
def unpack(self):
unpack=True
- clst_unpack_md5sum=read_from_clst(self.settings["autoresume_path"]+"unpack")
+ clst_unpack_hash=read_from_clst(self.settings["autoresume_path"]+"unpack")
if os.path.isdir(self.settings["source_path"]):
unpack_cmd="rsync -a --delete "+self.settings["source_path"]+" "+self.settings["chroot_path"]
os.path.exists(self.settings["autoresume_path"]+"unpack"):
print "Resume point detected, skipping unpack operation..."
unpack=False
- elif self.settings.has_key("source_path_md5sum"):
- if self.settings["source_path_md5sum"] != clst_unpack_md5sum:
+ elif self.settings.has_key("source_path_hash"):
+ if self.settings["source_path_hash"] != clst_unpack_hash:
invalid_snapshot=True
if unpack:
print display_msg
cmd(unpack_cmd,error_msg,env=self.env)
- if self.settings.has_key("source_path_md5sum"):
+ if self.settings.has_key("source_path_hash"):
myf=open(self.settings["autoresume_path"]+"unpack","w")
- myf.write(self.settings["source_path_md5sum"])
+ myf.write(self.settings["source_path_hash"])
myf.close()
else:
touch(self.settings["autoresume_path"]+"unpack")
# Copyright 1999-2005 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/modules/stage2_target.py,v 1.10 2005/12/16 14:42:07 rocket Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/modules/stage2_target.py,v 1.11 2005/12/19 20:53:13 rocket Exp $
"""
Builder class for a stage2 installation tarball build.
self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2")
if os.path.isfile(self.settings["source_path"]):
if os.path.exists(self.settings["source_path"]):
- self.settings["source_path_md5sum"]=calc_md5(self.settings["source_path"])
+ self.settings["source_path_hash"]=generate_hash(self.settings["source_path"])
def override_chost(self):
if os.environ.has_key("CHOST"):
self.settings["CHOST"] = os.environ["CHOST"]