From e337857ffb575302368d13fc63c5d511aa27c244 Mon Sep 17 00:00:00 2001 From: Brian Dolbec Date: Fri, 3 Jan 2014 10:42:26 -0800 Subject: [PATCH] Initial separation and creation of a hash_utils.py module --- catalyst/hash_utils.py | 137 +++++++++++++++++++++++ catalyst/main.py | 30 +++-- catalyst/support.py | 62 ---------- catalyst/targets/generic_stage_target.py | 36 +++--- catalyst/targets/livecd_stage2_target.py | 4 +- catalyst/targets/stage2_target.py | 7 +- 6 files changed, 184 insertions(+), 92 deletions(-) create mode 100644 catalyst/hash_utils.py diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py new file mode 100644 index 00000000..b575ace8 --- /dev/null +++ b/catalyst/hash_utils.py @@ -0,0 +1,137 @@ + +import os +from collections import namedtuple +from subprocess import Popen, PIPE + +from support import CatalystError + + +# Use HashMap.fields for the value legend +# fields = ["func", "cmd", "args", "id"] +HASH_DEFINITIONS = { + "adler32" :["calc_hash2", "shash", ["-a", "ADLER32"], "ADLER32"], + "crc32" :["calc_hash2", "shash", ["-a", "CRC32"], "CRC32"], + "crc32b" :["calc_hash2", "shash", ["-a", "CRC32B"], "CRC32B"], + "gost" :["calc_hash2", "shash", ["-a", "GOST"], "GOST"], + "haval128" :["calc_hash2", "shash", ["-a", "HAVAL128"], "HAVAL128"], + "haval160" :["calc_hash2", "shash", ["-a", "HAVAL160"], "HAVAL160"], + "haval192" :["calc_hash2", "shash", ["-a", "HAVAL192"], "HAVAL192"], + "haval224" :["calc_hash2", "shash", ["-a", "HAVAL224"], "HAVAL224"], + "haval256" :["calc_hash2", "shash", ["-a", "HAVAL256"], "HAVAL256"], + "md2" :["calc_hash2", "shash", ["-a", "MD2"], "MD2"], + "md4" :["calc_hash2", "shash", ["-a", "MD4"], "MD4"], + "md5" :["calc_hash2", "shash", ["-a", "MD5"], "MD5"], + "ripemd128":["calc_hash2", "shash", ["-a", "RIPEMD128"], "RIPEMD128"], + "ripemd160":["calc_hash2", "shash", ["-a", "RIPEMD160"], "RIPEMD160"], + "ripemd256":["calc_hash2", "shash", ["-a", "RIPEMD256"], "RIPEMD256"], + "ripemd320":["calc_hash2", "shash", ["-a", "RIPEMD320"], "RIPEMD320"], + "sha1" :["calc_hash2", "shash", ["-a", "SHA1"], "SHA1"], + "sha224" :["calc_hash2", "shash", ["-a", "SHA224"], "SHA224"], + "sha256" :["calc_hash2", "shash", ["-a", "SHA256"], "SHA256"], + "sha384" :["calc_hash2", "shash", ["-a", "SHA384"], "SHA384"], + "sha512" :["calc_hash2", "shash", ["-a", "SHA512"], "SHA512"], + "snefru128":["calc_hash2", "shash", ["-a", "SNEFRU128"], "SNEFRU128"], + "snefru256":["calc_hash2", "shash", ["-a", "SNEFRU256"], "SNEFRU256"], + "tiger" :["calc_hash2", "shash", ["-a", "TIGER"], "TIGER"], + "tiger128" :["calc_hash2", "shash", ["-a", "TIGER128"], "TIGER128"], + "tiger160" :["calc_hash2", "shash", ["-a", "TIGER160"], "TIGER160"], + "whirlpool":["calc_hash2", "shash", ["-a", "WHIRLPOOL"], "WHIRLPOOL"], + } + + +class HashMap(object): + '''Class for handling + Catalyst's hash generation''' + + fields = ["func", "cmd", "args", "id"] + + + def __init__(self, hashes=None): + '''Class init + + @param hashes: dictionary of Key:[function, cmd, cmd_args, Print string] + @param fields: list of ordered field names for the hashes + eg: ["func", "cmd", "args", "id"] + ''' + if hashes is None: + hashes = {} + self.hash_map = {} + + # create the hash definition namedtuple classes + for name in list(hashes): + obj = namedtuple(name, self.fields) + obj.__slots__ = () + self.hash_map[name] = obj._make(hashes[name]) + del obj + + + def generate_hash(self, file_, hash_="crc32", verbose=False): + '''Prefered method of generating a hash for the passed in file_ + + @param file_: the file to generate the hash for + @param hash_: the hash algorythm to use + @param verbose: boolean + @returns the hash result + ''' + try: + return getattr(self, self.hash_map[hash_].func)( + file_, + hash_, + verbose + ) + except: + raise CatalystError,"Error generating hash, is appropriate " + \ + "utility installed on your system?" + + + def calc_hash(self, file_, hash_, verbose=False): + ''' + Calculate the hash for "file_" + + @param file_: the file to generate the hash for + @param hash_: the hash algorythm to use + @param verbose: boolean + @returns the hash result + ''' + _hash = self.hash_map[hash_] + args = [_hash.cmd] + args.extend(_hash.args) + args.append(file_) + source = Popen(args, stdout=PIPE) + mylines = source.communicate()[0] + mylines=mylines[0].split() + result=mylines[0] + if verbose: + print _hash.id + " (%s) = %s" % (file_, result) + return result + + + def calc_hash2(self, file_, hash_type, verbose=False): + ''' + Calculate the hash for "file_" + + @param file_: the file to generate the hash for + @param hash_: the hash algorythm to use + @param verbose: boolean + @returns the hash result + ''' + _hash = self.hash_map[hash_type] + args = [_hash.cmd] + args.extend(_hash.args) + args.append(file_) + #print("DEBUG: calc_hash2; args =", args) + source = Popen(args, stdout=PIPE) + output = source.communicate() + lines = output[0].split('\n') + #print("DEBUG: calc_hash2; output =", output) + header = lines[0] + h_f = lines[1].split() + hash_result = h_f[0] + short_file = os.path.split(h_f[1])[1] + result = header + "\n" + hash_result + " " + short_file + "\n" + if verbose: + print header + " (%s) = %s" % (short_file, result) + return result + + + diff --git a/catalyst/main.py b/catalyst/main.py index 6b909898..7bcf2cb9 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -22,7 +22,10 @@ from . import __version__ import catalyst.config import catalyst.util from catalyst.support import (required_build_targets, - valid_build_targets, CatalystError, hash_map, find_binary, LockInUse) + valid_build_targets, CatalystError, find_binary, LockInUse) + +from hash_utils import HashMap, HASH_DEFINITIONS + conf_values={} @@ -345,40 +348,43 @@ def main(): # import configuration file and import our main module using those settings parse_config(myconfig) - # Start checking that digests are valid now that the hash_map was imported - # from catalyst.support + # initialze our hash and contents generators + hash_map = HashMap(HASH_DEFINITIONS) + conf_values["hash_map"] = hash_map + + # Start checking that digests are valid now that hash_map is initialized if "digests" in conf_values: for i in conf_values["digests"].split(): - if i not in hash_map: + if i not in HASH_DEFINITIONS: print print i+" is not a valid digest entry" print "Valid digest entries:" - print hash_map.keys() + print HASH_DEFINITIONS.keys() print print "Catalyst aborting...." sys.exit(2) - if find_binary(hash_map[i][1]) == None: + if find_binary(hash_map.hash_map[i].cmd) == None: print - print "digest="+i - print "\tThe "+hash_map[i][1]+\ + print "digest=" + i + print "\tThe " + hash_map.hash_map[i].cmd + \ " binary was not found. It needs to be in your system path" print print "Catalyst aborting...." sys.exit(2) if "hash_function" in conf_values: - if conf_values["hash_function"] not in hash_map: + if conf_values["hash_function"] not in HASH_DEFINITIONS: print print conf_values["hash_function"]+\ " is not a valid hash_function entry" print "Valid hash_function entries:" - print hash_map.keys() + print HASH_DEFINITIONS.keys() print print "Catalyst aborting...." sys.exit(2) - if find_binary(hash_map[conf_values["hash_function"]][1]) == None: + if find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd) == None: print print "hash_function="+conf_values["hash_function"] - print "\tThe "+hash_map[conf_values["hash_function"]][1]+\ + print "\tThe "+hash_map.hash_map[conf_values["hash_function"]].cmd + \ " binary was not found. It needs to be in your system path" print print "Catalyst aborting...." diff --git a/catalyst/support.py b/catalyst/support.py index 5e7ce929..308d9c02 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -114,68 +114,6 @@ contents_map={ "isoinfo-f":[calc_contents,"isoinfo -f -i %(file)s"], } -def generate_hash(file,hash_function="crc32",verbose=False): - try: - return hash_map[hash_function][0](file,hash_map[hash_function][1],hash_map[hash_function][2],\ - hash_map[hash_function][3],verbose) - except: - raise CatalystError,"Error generating hash, is appropriate utility installed on your system?" - -def calc_hash(file,cmd,cmd_args,id_string="MD5",verbose=False): - a=os.popen(cmd+" "+cmd_args+" "+file) - mylines=a.readlines() - a.close() - mylines=mylines[0].split() - result=mylines[0] - if verbose: - print id_string+" (%s) = %s" % (file, result) - return result - -def calc_hash2(file,cmd,cmd_args,id_string="MD5",verbose=False): - a=os.popen(cmd+" "+cmd_args+" "+file) - header=a.readline() - mylines=a.readline().split() - hash=mylines[0] - short_file=os.path.split(mylines[1])[1] - a.close() - result=header+hash+" "+short_file+"\n" - if verbose: - print header+" (%s) = %s" % (short_file, result) - return result - -# This has map must be defined after the function calc_hash -# It is possible to call different functions from this but they must be defined -# before hash_map -# Key,function,cmd,cmd_args,Print string -hash_map={ - "adler32":[calc_hash2,"shash","-a ADLER32","ADLER32"],\ - "crc32":[calc_hash2,"shash","-a CRC32","CRC32"],\ - "crc32b":[calc_hash2,"shash","-a CRC32B","CRC32B"],\ - "gost":[calc_hash2,"shash","-a GOST","GOST"],\ - "haval128":[calc_hash2,"shash","-a HAVAL128","HAVAL128"],\ - "haval160":[calc_hash2,"shash","-a HAVAL160","HAVAL160"],\ - "haval192":[calc_hash2,"shash","-a HAVAL192","HAVAL192"],\ - "haval224":[calc_hash2,"shash","-a HAVAL224","HAVAL224"],\ - "haval256":[calc_hash2,"shash","-a HAVAL256","HAVAL256"],\ - "md2":[calc_hash2,"shash","-a MD2","MD2"],\ - "md4":[calc_hash2,"shash","-a MD4","MD4"],\ - "md5":[calc_hash2,"shash","-a MD5","MD5"],\ - "ripemd128":[calc_hash2,"shash","-a RIPEMD128","RIPEMD128"],\ - "ripemd160":[calc_hash2,"shash","-a RIPEMD160","RIPEMD160"],\ - "ripemd256":[calc_hash2,"shash","-a RIPEMD256","RIPEMD256"],\ - "ripemd320":[calc_hash2,"shash","-a RIPEMD320","RIPEMD320"],\ - "sha1":[calc_hash2,"shash","-a SHA1","SHA1"],\ - "sha224":[calc_hash2,"shash","-a SHA224","SHA224"],\ - "sha256":[calc_hash2,"shash","-a SHA256","SHA256"],\ - "sha384":[calc_hash2,"shash","-a SHA384","SHA384"],\ - "sha512":[calc_hash2,"shash","-a SHA512","SHA512"],\ - "snefru128":[calc_hash2,"shash","-a SNEFRU128","SNEFRU128"],\ - "snefru256":[calc_hash2,"shash","-a SNEFRU256","SNEFRU256"],\ - "tiger":[calc_hash2,"shash","-a TIGER","TIGER"],\ - "tiger128":[calc_hash2,"shash","-a TIGER128","TIGER128"],\ - "tiger160":[calc_hash2,"shash","-a TIGER160","TIGER160"],\ - "whirlpool":[calc_hash2,"shash","-a WHIRLPOOL","WHIRLPOOL"],\ - } def read_from_clst(file): line = '' diff --git a/catalyst/targets/generic_stage_target.py b/catalyst/targets/generic_stage_target.py index eaf2c1f7..b6a62001 100644 --- a/catalyst/targets/generic_stage_target.py +++ b/catalyst/targets/generic_stage_target.py @@ -428,10 +428,11 @@ class generic_stage_target(generic_target): if os.path.isfile(self.settings["source_path"]): # XXX: Is this even necessary if the previous check passes? if os.path.exists(self.settings["source_path"]): - self.settings["source_path_hash"]=\ - generate_hash(self.settings["source_path"],\ - hash_function=self.settings["hash_function"],\ - verbose=False) + self.settings["source_path_hash"] = \ + self.settings["hash_map"].generate_hash( + self.settings["source_path"], + hash_ = self.settings["hash_function"], + verbose = False) print "Source path set to "+self.settings["source_path"] if os.path.isdir(self.settings["source_path"]): print "\tIf this is not desired, remove this directory or turn off" @@ -457,18 +458,22 @@ class generic_stage_target(generic_target): self.settings["snapshot"] + ".tar.xz") if os.path.exists(self.settings["snapshot_path"]): - self.settings["snapshot_path_hash"]=\ - generate_hash(self.settings["snapshot_path"],\ - hash_function=self.settings["hash_function"],verbose=False) + self.settings["snapshot_path_hash"] = \ + self.settings["hash_map"].generate_hash( + self.settings["snapshot_path"], + hash_ = self.settings["hash_function"], + verbose = False) else: self.settings["snapshot_path"]=normpath(self.settings["storedir"]+\ "/snapshots/" + self.settings["snapshot_name"] + self.settings["snapshot"] + ".tar.bz2") if os.path.exists(self.settings["snapshot_path"]): - self.settings["snapshot_path_hash"]=\ - generate_hash(self.settings["snapshot_path"],\ - hash_function=self.settings["hash_function"],verbose=False) + self.settings["snapshot_path_hash"] = \ + self.settings["hash_map"].generate_hash( + self.settings["snapshot_path"], + hash_ = self.settings["hash_function"], + verbose = False) def set_snapcache_path(self): if "SNAPCACHE" in self.settings: @@ -1716,6 +1721,7 @@ class generic_stage_target(generic_target): if os.path.exists(file+".DIGESTS"): os.remove(file+".DIGESTS") if "digests" in self.settings: + hash_map = self.settings["hash_map"] if os.path.exists(file): myf=open(file+".DIGESTS","w") keys={} @@ -1726,14 +1732,14 @@ class generic_stage_target(generic_target): for f in [file, file+'.CONTENTS']: if os.path.exists(f): if "all" in array: - for k in hash_map.keys(): - hash=generate_hash(f,hash_function=k,verbose=\ - "VERBOSE" in self.settings) + for k in list(hash_map.hash_map): + hash = hash_map.generate_hash(f, hash_ = k, + verbose = "VERBOSE" in self.settings) myf.write(hash) else: for j in array: - hash=generate_hash(f,hash_function=j,verbose=\ - "VERBOSE" in self.settings) + hash = hash_map.generate_hash(f, hash_ = j, + verbose = "VERBOSE" in self.settings) myf.write(hash) myf.close() diff --git a/catalyst/targets/livecd_stage2_target.py b/catalyst/targets/livecd_stage2_target.py index 1bfd8201..e7848441 100644 --- a/catalyst/targets/livecd_stage2_target.py +++ b/catalyst/targets/livecd_stage2_target.py @@ -35,7 +35,9 @@ class livecd_stage2_target(generic_stage_target): def set_source_path(self): self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2") if os.path.isfile(self.settings["source_path"]): - self.settings["source_path_hash"]=generate_hash(self.settings["source_path"]) + self.settings["source_path_hash"] = \ + self.settings["hash_map"].generate_hash( + self.settings["source_path"]) else: self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/") if not os.path.exists(self.settings["source_path"]): diff --git a/catalyst/targets/stage2_target.py b/catalyst/targets/stage2_target.py index 15acdeec..6377f5da 100644 --- a/catalyst/targets/stage2_target.py +++ b/catalyst/targets/stage2_target.py @@ -23,8 +23,11 @@ class stage2_target(generic_stage_target): if os.path.isfile(self.settings["source_path"]): if os.path.exists(self.settings["source_path"]): # XXX: Is this even necessary if the previous check passes? - self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\ - hash_function=self.settings["hash_function"],verbose=False) + self.settings["source_path_hash"] = \ + self.settings["hash_map"].generate_hash( + self.settings["source_path"],\ + hash_=self.settings["hash_function"], + verbose=False) print "Source path set to "+self.settings["source_path"] if os.path.isdir(self.settings["source_path"]): print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf" -- 2.26.2