From 7a957e8ebd6e9e567436e1867f3a412836fedbc7 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 11 Jan 2009 16:46:48 -0600 Subject: [PATCH] Move all hash and contents functions to catalyst.hash --- ChangeLog | 7 ++ modules/catalyst/hash.py | 116 +++++++++++++++++++++++ modules/catalyst/support.py | 112 ++-------------------- modules/catalyst/target/generic_stage.py | 11 ++- modules/catalyst/target/livecd_stage2.py | 3 +- modules/catalyst/target/stage2.py | 3 +- modules/catalyst/util.py | 25 ----- 7 files changed, 141 insertions(+), 136 deletions(-) create mode 100644 modules/catalyst/hash.py diff --git a/ChangeLog b/ChangeLog index 4d70f790..dfd9f24c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,13 @@ # Copyright 2002-2009 Gentoo Foundation; 2008-2009 Various authors (see AUTHORS) # Distributed under the GPL v2 + 11 Jan 2009; Andrew Gaffney + +modules/catalyst/hash.py, modules/catalyst/support.py, + modules/catalyst/target/generic_stage.py, + modules/catalyst/target/livecd_stage2.py, + modules/catalyst/target/stage2.py, modules/catalyst/util.py: + Move all hash and contents functions to catalyst.hash + 11 Jan 2009; Andrew Gaffney modules/catalyst/support.py, modules/catalyst/target/generic_stage.py, modules/catalyst/util.py: diff --git a/modules/catalyst/hash.py b/modules/catalyst/hash.py new file mode 100644 index 00000000..44c272de --- /dev/null +++ b/modules/catalyst/hash.py @@ -0,0 +1,116 @@ +""" +This module contains functions for generating the CONTENTS and hash files +""" + +import os + +def generate_contents(file, contents_function="auto", verbose=False): + try: + _ = contents_function + if _ == 'auto' and file.endswith('.iso'): + _ = 'isoinfo-l' + if (_ in ['tar-tv','auto']): + if file.endswith('.tgz') or file.endswith('.tar.gz'): + _ = 'tar-tvz' + elif file.endswith('.tbz2') or file.endswith('.tar.bz2'): + _ = 'tar-tvj' + elif file.endswith('.tar'): + _ = 'tar-tv' + + if _ == 'auto': + warn('File %r has unknown type for automatic detection.' % (file, )) + return None + else: + contents_function = _ + _ = contents_map[contents_function] + return _[0](file,_[1],verbose) + except: + raise CatalystError, \ + "Error generating contents, is appropriate utility (%s) installed on your system?" \ + % (contents_function, ) + +def calc_contents(file, cmd, verbose): + cmd = cmd % { 'file': file } + a = os.popen(cmd) + result = "".join(a.readlines()) + a.close() + if verbose: + print result + return result + +# This has map must be defined after the function calc_content +# It is possible to call different functions from this but they must be defined +# before hash_map +# Key,function,cmd +contents_map = { + # 'find' is disabled because it requires the source path, which is not + # always available + #"find" :[calc_contents,"find %(path)s"], + "tar-tv":[calc_contents,"tar tvf %(file)s"], + "tar-tvz":[calc_contents,"tar tvzf %(file)s"], + "tar-tvj":[calc_contents,"tar tvjf %(file)s"], + "isoinfo-l":[calc_contents,"isoinfo -l -i %(file)s"], + # isoinfo-f should be a last resort only + "isoinfo-f":[calc_contents,"isoinfo -f -i %(file)s"], +} + +def generate_hash(file, hash_function="crc32", verbose=False): + try: + return hash_map[hash_function][0](file, hash_map[hash_function][1], hash_map[hash_function][2], \ + hash_map[hash_function][3],verbose) + except: + raise CatalystError, "Error generating hash, is appropriate utility installed on your system?" + +def calc_hash(file, cmd, cmd_args, id_string="MD5", verbose=False): + a = os.popen(cmd + " " + cmd_args + " " + file) + result = a.readline().split()[0] + a.close() + if verbose: + print "%s (%s) = %s" % (id_string, file, result) + return result + +def calc_hash2(file, cmd, cmd_args, id_string="MD5", verbose=False): + a = os.popen(cmd + " " + cmd_args + " " + file) + header = a.readline() + myline = a.readline().split() + a.close() + hash = myline[0] + short_file = os.path.split(myline[1])[1] + result = header + hash + " " + short_file + "\n" + if verbose: + print "%s (%s) = %s" % (header, short_file, result) + return result + +# This has map must be defined after the function calc_hash +# It is possible to call different functions from this but they must be defined +# before hash_map +# Key,function,cmd,cmd_args,Print string +hash_map={ + "adler32":[calc_hash2,"shash","-a ADLER32","ADLER32"], + "crc32":[calc_hash2,"shash","-a CRC32","CRC32"], + "crc32b":[calc_hash2,"shash","-a CRC32B","CRC32B"], + "gost":[calc_hash2,"shash","-a GOST","GOST"], + "haval128":[calc_hash2,"shash","-a HAVAL128","HAVAL128"], + "haval160":[calc_hash2,"shash","-a HAVAL160","HAVAL160"], + "haval192":[calc_hash2,"shash","-a HAVAL192","HAVAL192"], + "haval224":[calc_hash2,"shash","-a HAVAL224","HAVAL224"], + "haval256":[calc_hash2,"shash","-a HAVAL256","HAVAL256"], + "md2":[calc_hash2,"shash","-a MD2","MD2"], + "md4":[calc_hash2,"shash","-a MD4","MD4"], + "md5":[calc_hash2,"shash","-a MD5","MD5"], + "ripemd128":[calc_hash2,"shash","-a RIPEMD128","RIPEMD128"], + "ripemd160":[calc_hash2,"shash","-a RIPEMD160","RIPEMD160"], + "ripemd256":[calc_hash2,"shash","-a RIPEMD256","RIPEMD256"], + "ripemd320":[calc_hash2,"shash","-a RIPEMD320","RIPEMD320"], + "sha1":[calc_hash2,"shash","-a SHA1","SHA1"], + "sha224":[calc_hash2,"shash","-a SHA224","SHA224"], + "sha256":[calc_hash2,"shash","-a SHA256","SHA256"], + "sha384":[calc_hash2,"shash","-a SHA384","SHA384"], + "sha512":[calc_hash2,"shash","-a SHA512","SHA512"], + "snefru128":[calc_hash2,"shash","-a SNEFRU128","SNEFRU128"], + "snefru256":[calc_hash2,"shash","-a SNEFRU256","SNEFRU256"], + "tiger":[calc_hash2,"shash","-a TIGER","TIGER"], + "tiger128":[calc_hash2,"shash","-a TIGER128","TIGER128"], + "tiger160":[calc_hash2,"shash","-a TIGER160","TIGER160"], + "whirlpool":[calc_hash2,"shash","-a WHIRLPOOL","WHIRLPOOL"], +} diff --git a/modules/catalyst/support.py b/modules/catalyst/support.py index 923c4663..7c67d59c 100644 --- a/modules/catalyst/support.py +++ b/modules/catalyst/support.py @@ -1,8 +1,7 @@ -import sys,string,os,types,re,signal,traceback,time +import sys, os, types, re, signal, traceback from catalyst.output import warn import catalyst.util -#import md5,sha selinux_capable = False #userpriv_capable = (os.getuid() == 0) @@ -10,22 +9,17 @@ selinux_capable = False BASH_BINARY = "/bin/bash" try: - import resource - max_fd_limit=resource.getrlimit(RLIMIT_NOFILE) + import resource + max_fd_limit=resource.getrlimit(RLIMIT_NOFILE) except SystemExit, e: - raise + raise except: - # hokay, no resource module. - max_fd_limit=256 + # hokay, no resource module. + max_fd_limit=256 # pids this process knows of. spawned_pids = [] -try: - import urllib -except SystemExit, e: - raise - def cleanup(pids,block_exceptions=True): """function to go through and reap the list of pids passed to it""" global spawned_pids @@ -53,96 +47,6 @@ def cleanup(pids,block_exceptions=True): except IndexError: pass -def calc_contents(file,cmd,verbose): - args={ 'file': file } - cmd=cmd % dict(args) - a=os.popen(cmd) - mylines=a.readlines() - a.close() - result="".join(mylines) - if verbose: - print result - return result - -# This has map must be defined after the function calc_content -# It is possible to call different functions from this but they must be defined -# before hash_map -# Key,function,cmd -contents_map={ - # 'find' is disabled because it requires the source path, which is not - # always available - #"find" :[calc_contents,"find %(path)s"], - "tar-tv":[calc_contents,"tar tvf %(file)s"], - "tar-tvz":[calc_contents,"tar tvzf %(file)s"], - "tar-tvj":[calc_contents,"tar tvjf %(file)s"], - "isoinfo-l":[calc_contents,"isoinfo -l -i %(file)s"], - # isoinfo-f should be a last resort only - "isoinfo-f":[calc_contents,"isoinfo -f -i %(file)s"], -} - -def generate_hash(file,hash_function="crc32",verbose=False): - try: - return hash_map[hash_function][0](file,hash_map[hash_function][1],hash_map[hash_function][2],\ - hash_map[hash_function][3],verbose) - except: - raise CatalystError,"Error generating hash, is appropriate utility installed on your system?" - -def calc_hash(file,cmd,cmd_args,id_string="MD5",verbose=False): - a=os.popen(cmd+" "+cmd_args+" "+file) - mylines=a.readlines() - a.close() - mylines=mylines[0].split() - result=mylines[0] - if verbose: - print id_string+" (%s) = %s" % (file, result) - return result - -def calc_hash2(file,cmd,cmd_args,id_string="MD5",verbose=False): - a=os.popen(cmd+" "+cmd_args+" "+file) - header=a.readline() - mylines=a.readline().split() - hash=mylines[0] - short_file=os.path.split(mylines[1])[1] - a.close() - result=header+hash+" "+short_file+"\n" - if verbose: - print header+" (%s) = %s" % (short_file, result) - return result - -# This has map must be defined after the function calc_hash -# It is possible to call different functions from this but they must be defined -# before hash_map -# Key,function,cmd,cmd_args,Print string -hash_map={ - "adler32":[calc_hash2,"shash","-a ADLER32","ADLER32"],\ - "crc32":[calc_hash2,"shash","-a CRC32","CRC32"],\ - "crc32b":[calc_hash2,"shash","-a CRC32B","CRC32B"],\ - "gost":[calc_hash2,"shash","-a GOST","GOST"],\ - "haval128":[calc_hash2,"shash","-a HAVAL128","HAVAL128"],\ - "haval160":[calc_hash2,"shash","-a HAVAL160","HAVAL160"],\ - "haval192":[calc_hash2,"shash","-a HAVAL192","HAVAL192"],\ - "haval224":[calc_hash2,"shash","-a HAVAL224","HAVAL224"],\ - "haval256":[calc_hash2,"shash","-a HAVAL256","HAVAL256"],\ - "md2":[calc_hash2,"shash","-a MD2","MD2"],\ - "md4":[calc_hash2,"shash","-a MD4","MD4"],\ - "md5":[calc_hash2,"shash","-a MD5","MD5"],\ - "ripemd128":[calc_hash2,"shash","-a RIPEMD128","RIPEMD128"],\ - "ripemd160":[calc_hash2,"shash","-a RIPEMD160","RIPEMD160"],\ - "ripemd256":[calc_hash2,"shash","-a RIPEMD256","RIPEMD256"],\ - "ripemd320":[calc_hash2,"shash","-a RIPEMD320","RIPEMD320"],\ - "sha1":[calc_hash2,"shash","-a SHA1","SHA1"],\ - "sha224":[calc_hash2,"shash","-a SHA224","SHA224"],\ - "sha256":[calc_hash2,"shash","-a SHA256","SHA256"],\ - "sha384":[calc_hash2,"shash","-a SHA384","SHA384"],\ - "sha512":[calc_hash2,"shash","-a SHA512","SHA512"],\ - "snefru128":[calc_hash2,"shash","-a SNEFRU128","SNEFRU128"],\ - "snefru256":[calc_hash2,"shash","-a SNEFRU256","SNEFRU256"],\ - "tiger":[calc_hash2,"shash","-a TIGER","TIGER"],\ - "tiger128":[calc_hash2,"shash","-a TIGER128","TIGER128"],\ - "tiger160":[calc_hash2,"shash","-a TIGER160","TIGER160"],\ - "whirlpool":[calc_hash2,"shash","-a WHIRLPOOL","WHIRLPOOL"],\ - } - required_config_file_values=["storedir","sharedir","distdir","portdir"] valid_config_file_values=required_config_file_values[:] valid_config_file_values.append("PKGCACHE") @@ -239,7 +143,7 @@ def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \ mydata=fd.readlines() fd.close() if emulate_gso: - mydata=string.join(mydata) + mydata="".join(mydata) if len(mydata) and mydata[-1] == "\n": mydata=mydata[:-1] retval=os.waitpid(mypid[0],0)[1] @@ -399,7 +303,7 @@ def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,retur raise except Exception, e: if not func_call: - raise str(e)+":\n "+myc+" "+string.join(myargs) + raise str(e)+":\n "+myc+" "+"".join(myargs) print "func call failed" # If the execve fails, we need to report it, and exit diff --git a/modules/catalyst/target/generic_stage.py b/modules/catalyst/target/generic_stage.py index 1e18ac51..56499077 100644 --- a/modules/catalyst/target/generic_stage.py +++ b/modules/catalyst/target/generic_stage.py @@ -12,6 +12,7 @@ import catalyst.lock import catalyst.arch from catalyst.output import warn import catalyst.util +import catalyst.hash class generic_stage_target(generic_target): @@ -364,7 +365,7 @@ class generic_stage_target(generic_target): # XXX: Is this even necessary if the previous check passes? if os.path.exists(self.settings["source_path"]): self.settings["source_path_hash"]=\ - generate_hash(self.settings["source_path"],\ + catalyst.hash.generate_hash(self.settings["source_path"],\ hash_function=self.settings["hash_function"],\ verbose=False) print "Source path set to "+self.settings["source_path"] @@ -392,7 +393,7 @@ class generic_stage_target(generic_target): if os.path.exists(self.settings["snapshot_path"]): self.settings["snapshot_path_hash"]=\ - generate_hash(self.settings["snapshot_path"],\ + catalyst.hash.generate_hash(self.settings["snapshot_path"],\ hash_function=self.settings["hash_function"],verbose=False) def set_snapcache_path(self): @@ -1586,7 +1587,7 @@ class generic_stage_target(generic_target): array=keys.keys() array.sort() for j in array: - contents=catalyst.util.generate_contents(file,contents_function=j,\ + contents=catalyst.hash.generate_contents(file,contents_function=j,\ verbose=("VERBOSE" in self.settings)) if contents: myf.write(contents) @@ -1607,12 +1608,12 @@ class generic_stage_target(generic_target): if os.path.exists(f): if "all" in array: for k in hash_map.keys(): - hash=generate_hash(f,hash_function=k,verbose=\ + hash=catalyst.hash.generate_hash(f,hash_function=k,verbose=\ ("VERBOSE" in self.settings)) myf.write(hash) else: for j in array: - hash=generate_hash(f,hash_function=j,verbose=\ + hash=catalyst.hash.generate_hash(f,hash_function=j,verbose=\ ("VERBOSE" in self.settings)) myf.write(hash) myf.close() diff --git a/modules/catalyst/target/livecd_stage2.py b/modules/catalyst/target/livecd_stage2.py index b86d265e..91f51cc8 100644 --- a/modules/catalyst/target/livecd_stage2.py +++ b/modules/catalyst/target/livecd_stage2.py @@ -7,6 +7,7 @@ import os,string,types,stat,shutil from catalyst.support import * from generic_stage import * import catalyst.util +import catalyst.hash class livecd_stage2_target(generic_stage_target): def __init__(self,spec,addlargs): @@ -33,7 +34,7 @@ class livecd_stage2_target(generic_stage_target): def set_source_path(self): self.settings["source_path"]=catalyst.util.normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2") if os.path.isfile(self.settings["source_path"]): - self.settings["source_path_hash"]=generate_hash(self.settings["source_path"]) + self.settings["source_path_hash"]=catalyst.hash.generate_hash(self.settings["source_path"]) else: self.settings["source_path"]=catalyst.util.normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/") if not os.path.exists(self.settings["source_path"]): diff --git a/modules/catalyst/target/stage2.py b/modules/catalyst/target/stage2.py index 90877847..5e8cb4ef 100644 --- a/modules/catalyst/target/stage2.py +++ b/modules/catalyst/target/stage2.py @@ -6,6 +6,7 @@ Builder class for a stage2 installation tarball build. from catalyst.support import * from generic_stage import * import catalyst.util +import catalyst.hash class stage2_target(generic_stage_target): def __init__(self,spec,addlargs): @@ -21,7 +22,7 @@ class stage2_target(generic_stage_target): if os.path.isfile(self.settings["source_path"]): if os.path.exists(self.settings["source_path"]): # XXX: Is this even necessary if the previous check passes? - self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\ + self.settings["source_path_hash"]=catalyst.hash.generate_hash(self.settings["source_path"],\ hash_function=self.settings["hash_function"],verbose=False) print "Source path set to "+self.settings["source_path"] if os.path.isdir(self.settings["source_path"]): diff --git a/modules/catalyst/util.py b/modules/catalyst/util.py index 5bc6cb7c..e715ed74 100644 --- a/modules/catalyst/util.py +++ b/modules/catalyst/util.py @@ -103,31 +103,6 @@ def touch(myfile): except IOError: raise CatalystError, "Could not touch " + myfile + "." -def generate_contents(file, contents_function="auto", verbose=False): - try: - _ = contents_function - if _ == 'auto' and file.endswith('.iso'): - _ = 'isoinfo-l' - if (_ in ['tar-tv','auto']): - if file.endswith('.tgz') or file.endswith('.tar.gz'): - _ = 'tar-tvz' - elif file.endswith('.tbz2') or file.endswith('.tar.bz2'): - _ = 'tar-tvj' - elif file.endswith('.tar'): - _ = 'tar-tv' - - if _ == 'auto': - warn('File %r has unknown type for automatic detection.' % (file, )) - return None - else: - contents_function = _ - _ = contents_map[contents_function] - return _[0](file,_[1],verbose) - except: - raise CatalystError, \ - "Error generating contents, is appropriate utility (%s) installed on your system?" \ - % (contents_function, ) - def countdown(secs=5, doing="Starting"): if secs: print ">>> Waiting " + secs + " seconds before starting..." -- 2.26.2