if not fetchonly:
fetched=2
else:
- # Check md5sum's at each fetch for fetchonly.
+ # Verify checksums at each fetch for fetchonly.
verified_ok,reason = portage_checksum.verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile])
if not verified_ok:
writemsg("!!! Previously fetched file: "+str(myfile)+"\n!!! Reason: "+reason+"\nRefetching...\n\n")
try:
mystat=os.stat(mysettings["DISTDIR"]+"/"+myfile)
# no exception? file exists. let digestcheck() report
- # an appropriately for size or md5 errors
+ # an appropriately for size or checksum errors
if (mystat[stat.ST_SIZE]<mydigests[myfile]["size"]):
# Fetch failed... Try the next one... Kill 404 files though.
if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
fetched=2
break
else:
- # File is the correct size--check the MD5 sum for the fetched
+ # File is the correct size--check the checksums for the fetched
# file NOW, for those users who don't have a stable/continuous
# net connection. This way we have a chance to try to download
# from another mirror...
print "!!! Given file does not appear to be readable. Does it exist?"
print "!!! File:",myfile
return None
- mydigests[x] = portage_checksum.perform_all(myfile)
+ mydigests[x] = portage_checksum.perform_multiple_checksums(myfile, hashes=portage_const.MANIFEST1_HASH_FUNCTIONS)
mysize = os.stat(myfile)[stat.ST_SIZE]
else:
if x in oldDigest:
myline += " "+mysum
myline += " "+myarchive
myline += " "+str(mysize)
- if sumName != "MD5":
- # XXXXXXXXXXXXXXXX This cannot be used!
- # Older portage make very dumb assumptions about the formats.
- # We need a lead-in period before we break everything.
- continue
mylines.append(myline)
return mylines
def digestParseFile(myfilename):
"""(filename) -- Parses a given file for entries matching:
- MD5 MD5_STRING_OF_HEX_CHARS FILE_NAME FILE_SIZE
- Ignores lines that do not begin with 'MD5' and returns a
- dict with the filenames as keys and [md5,size] as the values."""
+ <checksumkey> <checksum_hex_string> <filename> <filesize>
+ Ignores lines that don't start with a valid checksum identifier
+ and returns a dict with the filenames as keys and {checksumkey:checksum}
+ as the values."""
if not os.path.exists(myfilename):
return None
def digestCheckFiles(myfiles, mydigests, basedir, note="", strict=0):
"""(fileslist, digestdict, basedir) -- Takes a list of files and a dict
of their digests and checks the digests against the indicated files in
- the basedir given. Returns 1 only if all files exist and match the md5s.
+ the basedir given. Returns 1 only if all files exist and match the checksums.
"""
for x in myfiles:
if not mydigests.has_key(x):
print
return 0
else:
- print ">>> md5 "+note+" ;-)",x
+ print ">>> checksums "+note+" ;-)",x
return 1
def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
- """Checks md5sums. Assumes all files have been downloaded."""
+ """Verifies checksums. Assumes all files have been downloaded."""
# archive files
basedir=mysettings["DISTDIR"]+"/"
digestfn=mysettings["FILESDIR"]+"/digest-"+mysettings["PF"]
# Check the portage-related files here.
mymfiles=listdir(pbasedir,recursive=1,filesonly=1,ignorecvs=1,EmptyOnError=1)
manifest_files = mymdigests.keys()
+ # Files unrelated to the build process are ignored for verification by default
for x in ["Manifest", "ChangeLog", "metadata.xml"]:
while x in mymfiles:
mymfiles.remove(x)
def getfetchsizes(self,mypkg,useflags=None,debug=0):
# returns a filename:size dictionnary of remaining downloads
mydigest=self.finddigest(mypkg)
- mymd5s=digestParseFile(mydigest)
- if not mymd5s:
+ checksums=digestParseFile(mydigest)
+ if not checksums:
if debug: print "[empty/missing/bad digest]: "+mypkg
return None
filesdict={}
else:
myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags)
#XXX: maybe this should be improved: take partial downloads
- # into account? check md5sums?
+ # into account? check checksums?
for myfile in myfiles:
- if debug and myfile not in mymd5s.keys():
+ if debug and myfile not in checksums.keys():
print "[bad digest]: missing",myfile,"for",mypkg
- elif myfile in mymd5s.keys():
+ elif myfile in checksums.keys():
distfile=settings["DISTDIR"]+"/"+myfile
if not os.access(distfile, os.R_OK):
- filesdict[myfile]=int(mymd5s[myfile]["size"])
+ filesdict[myfile]=int(checksums[myfile]["size"])
return filesdict
def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False):
# $Id: /var/cvsroot/gentoo-src/portage/pym/portage_checksum.py,v 1.10.2.2 2005/08/10 05:42:03 ferringb Exp $
-from portage_const import PRIVATE_PATH,PRELINK_BINARY
+from portage_const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE
import os
import shutil
import stat
import commands
import sha
+
+# actual hash functions first
+
+#dict of all available hash functions
+hashfunc_map = {}
+
+# We _try_ to load this module. If it fails we do the slightly slower fallback.
+try:
+ import fchksum
+
+ def md5hash(filename):
+ return fchksum.fmd5t(filename)
+
+except ImportError:
+ import md5
+ def md5hash(filename):
+ return pyhash(filename, md5)
+hashfunc_map["MD5"] = md5hash
+
+def sha1hash(filename):
+ return pyhash(filename, sha)
+hashfunc_map["SHA1"] = sha1hash
+
+# Keep pycrypto optional for now, there are no internal fallbacks for these
+try:
+ import Crypto.Hash.SHA256
+
+ def sha256hash(filename):
+ return pyhash(filename, Crypto.Hash.SHA256)
+ hashfunc_map["SHA256"] = sha256hash
+except ImportError:
+ pass
+
+try:
+ import Crypto.Hash.RIPEMD
+
+ def rmd160hash(filename):
+ return pyhash(filename, Crypto.Hash.RIPEMD)
+ hashfunc_map["RMD160"] = rmd160hash
+except ImportError:
+ pass
+
+# end actual hash functions
+
prelink_capable = False
if os.path.exists(PRELINK_BINARY):
results = commands.getstatusoutput(PRELINK_BINARY+" --version > /dev/null 2>&1")
def perform_md5(x, calc_prelink=0):
return perform_checksum(x, md5hash, calc_prelink)[0]
-def perform_sha1(x, calc_prelink=0):
- return perform_checksum(x, sha1hash, calc_prelink)[0]
-
def perform_all(x, calc_prelink=0):
mydict = {}
- mydict["SHA1"] = perform_sha1(x, calc_prelink)
- mydict["MD5"] = perform_md5(x, calc_prelink)
+ for k in hashfunc_map.keys():
+ mydict[k] = perform_checksum(x, hashfunc_map[k], calc_prelink)[0]
return mydict
def get_valid_checksum_keys():
- return ["SHA1", "MD5"]
+ return hashfunc_map.keys()
def verify_all(filename, mydict, calc_prelink=0, strict=0):
# Dict relates to single file only.
for x in mydict.keys():
if x == "size":
continue
- elif x == "SHA1":
- if mydict[x] != perform_sha1(filename, calc_prelink=calc_prelink):
- if strict:
- raise portage_exception.DigestException, "Failed to verify '$(file)s' on checksum type '%(type)s'" % {"file":filename, "type":x}
- else:
- file_is_ok = False
- reason = "Failed on %s verification" % (x,)
- break
- elif x == "MD5":
- if mydict[x] != perform_md5(filename, calc_prelink=calc_prelink):
+ elif x in hashfunc_map.keys():
+ if mydict[x] != perform_checksum(filename, hashfunc_map[x], calc_prelink=calc_prelink)[0]:
if strict:
raise portage_exception.DigestException, "Failed to verify '$(file)s' on checksum type '%(type)s'" % {"file":filename, "type":x}
else:
break
return file_is_ok,reason
-# We _try_ to load this module. If it fails we do the slow fallback.
-try:
- import fchksum
-
- def md5hash(filename):
- return fchksum.fmd5t(filename)
-
-except ImportError:
- import md5
- def md5hash(filename):
- f = open(filename, 'rb')
- blocksize=32768
- data = f.read(blocksize)
- size = 0L
- sum = md5.new()
- while data:
- sum.update(data)
- size = size + len(data)
- data = f.read(blocksize)
- f.close()
-
- return (sum.hexdigest(),size)
-
-def sha1hash(filename):
+def pyhash(filename, hashobject):
f = open(filename, 'rb')
- blocksize=32768
+ blocksize = HASHING_BLOCKSIZE
data = f.read(blocksize)
size = 0L
- sum = sha.new()
+ sum = hashobject.new()
while data:
sum.update(data)
size = size + len(data)
data = f.read(blocksize)
f.close()
- return (sum.hexdigest(),size)
+ return (sum.hexdigest(), size)
def perform_checksum(filename, hash_function=md5hash, calc_prelink=0):
myfilename = filename[:]
portage_locks.unlockfile(mylock)
return (myhash,mysize)
+
+def perform_multiple_checksums(filename, hashes=["MD5"], calc_prelink=0):
+ rVal = {}
+ for x in hashes:
+ rVal[x] = perform_checksum(filename, hashfunc_map[x], calc_prelink)[0]
+ return rVal
projects / portage.git / commitdiff