From: Fabian Groffen Date: Sun, 24 Jun 2007 15:50:52 +0000 (-0000) Subject: Merged from trunk 6971:6981 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=632bc2d2ea19695c59752d9fab1a0852e9de7e59;p=portage.git Merged from trunk 6971:6981 +------------------------------------------------------------------------+ | 6972 | add hashlib support for checksum verification, refactor | | genone | checksum functions to not look like a complete mess | |--------+---------------------------------------------------------------| | 6973 | Allow sync for unpriviledges users if they have write access | | genone | to $PORTDIR | |--------+---------------------------------------------------------------| | 6974 | syntax error | | genone | | |--------+---------------------------------------------------------------| | 6975 | s/!owned/replaced/ in merge output to clarify it | | genone | | |--------+---------------------------------------------------------------| | 6976 | sort the world file before writing it and ensure that it's | | genone | terminated by a newline | |--------+---------------------------------------------------------------| | 6977 | change default diff_command for emacs (bug #147558) | | genone | | |--------+---------------------------------------------------------------| | 6978 | dont break when the binhost url doesn't end with a slash (bug | | genone | #144440) | |--------+---------------------------------------------------------------| | 6979 | highlight world packages in --pretend output (bug #142473) | | genone | | |--------+---------------------------------------------------------------| | 6980 | Add a pointer from FETCHCOMMAND to RESUMECOMMAND (bug | | genone | #141550) | |--------+---------------------------------------------------------------| | 6981 | get ACCEPT_KEYWORDS from the env in gvisible() to properly | | genone | handle negated values (bug #139600) | +------------------------------------------------------------------------+ svn path=/main/branches/prefix/; revision=6994 --- diff --git a/cnf/etc-update.conf b/cnf/etc-update.conf index 16ae9334c..06c46ad74 100644 --- a/cnf/etc-update.conf +++ b/cnf/etc-update.conf @@ -22,7 +22,7 @@ pager="less" #pager="" # For emacs-users -# diff_command="eval emacs -nw --eval=\(ediff\ \"%file1\"\ \"%file2\"\)" +# diff_command="eval emacs -nw --eval=\'\(ediff\ \"%file1\"\ \"%file2\"\)\'" #using_editor=1 # vim-users: you CAN use vimdiff for diff_command. (see NOTE_1) diff --git a/man/make.conf.5 b/man/make.conf.5 index f3ede8262..10fdbf25e 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -291,6 +291,7 @@ Enable the sandbox in the compile phase, when running without root privs (\fIuse This variable contains the command used for fetching package sources from the internet. It must contain the full path to the executable as well as the place\-holders \\${DISTDIR} and \\${URI}. +Also see \fBRESUMECOMMAND\fR. .TP \fBGENTOO_MIRRORS\fR = \fI[URLs]\fR Insert your space\-seperated list of local mirrors here. These diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index d9ea064f0..96b589ca0 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -587,10 +587,12 @@ def clean_world(vardb, cpv): else: #this doesn't match the package we're unmerging; keep it. newworldlist.append(x) - + + newworldlist.sort() + portage.util.ensure_dirs(os.path.join(vardb.root, portage.PRIVATE_PATH), gid=portage.portage_gid, mode=02770) - portage.util.write_atomic(world_filename, "\n".join(newworldlist)) + portage.util.write_atomic(world_filename, "\n".join(newworldlist)+"\n") def genericdict(mylist): mynewdict={} @@ -2477,6 +2479,8 @@ class depgraph(object): # files to fetch list - avoids counting a same file twice # in size display (verbose mode) myfetchlist=[] + worldlist = getlist(self.settings, "world") + for mylist_index in xrange(len(mylist)): x, depth, ordered = mylist[mylist_index] pkg_type = x[0] @@ -2745,17 +2749,23 @@ class depgraph(object): myoldbest=myoldbest[:-3] myoldbest=blue("["+myoldbest+"]") + if xs[0] in worldlist: + pkgprint = bold + else: + def pkgprint(pkg): + return pkg + if x[1]!="/": if myoldbest: myoldbest +=" " if "--columns" in self.myopts: if "--quiet" in self.myopts: - myprint=addl+" "+indent+darkgreen(xs[0]) + myprint=addl+" "+indent+darkgreen(pkgprint(xs[0])) myprint=myprint+darkblue(" "+xs[1]+xs[2])+" " myprint=myprint+myoldbest myprint=myprint+darkgreen("to "+x[1]) else: - myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(xs[0]) + myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(xs[0])) if (newlp-nc_len(myprint)) > 0: myprint=myprint+(" "*(newlp-nc_len(myprint))) myprint=myprint+"["+darkblue(xs[1]+xs[2])+"] " @@ -2768,17 +2778,17 @@ class depgraph(object): myprint = darkblue("[nomerge ] ") else: myprint = "[" + pkg_type + " " + addl + "] " - myprint += indent + darkgreen(pkg_key) + " " + \ + myprint += indent + darkgreen(pkrprint(pkg_key)) + " " + \ myoldbest + darkgreen("to " + myroot) + " " + \ verboseadd else: if "--columns" in self.myopts: if "--quiet" in self.myopts: - myprint=addl+" "+indent+darkgreen(xs[0]) + myprint=addl+" "+indent+darkgreen(pkgprint(xs[0])) myprint=myprint+" "+green(xs[1]+xs[2])+" " myprint=myprint+myoldbest else: - myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(xs[0]) + myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(xs[0])) if (newlp-nc_len(myprint)) > 0: myprint=myprint+(" "*(newlp-nc_len(myprint))) myprint=myprint+green(" ["+xs[1]+xs[2]+"] ") @@ -2787,9 +2797,9 @@ class depgraph(object): myprint=myprint+myoldbest+" "+verboseadd else: if x[-1] == "nomerge" or not ordered: - myprint=darkblue("[nomerge ] "+indent+x[2]+" "+myoldbest+" ")+verboseadd + myprint=darkblue("[nomerge ] "+indent+pkgprint(x[2])+" "+myoldbest+" ")+verboseadd else: - myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(x[2])+" "+myoldbest+" "+verboseadd + myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(x[2]))+" "+myoldbest+" "+verboseadd p.append(myprint) mysplit = portage.pkgsplit(x[2]) @@ -3340,7 +3350,7 @@ class MergeTask(object): ") Updating world file ("+x[pkgindex]+")") portage.write_atomic( os.path.join(myroot, portage.WORLD_FILE), - "\n".join(myfavdict.values())) + "\n".join(sorted(myfavdict.values()))+"\n") if "--pretend" not in self.myopts and \ "--fetchonly" not in self.myopts and \ @@ -5663,12 +5673,12 @@ def emerge_main(): # check if root user is the current user for the actions where emerge needs this if portage.secpass < 2: # We've already allowed "--version" and "--help" above. - if "--pretend" not in myopts and \ - myaction not in ("search","info"): + if "--pretend" not in myopts and myaction not in ("search","info"): need_superuser = not \ ("--fetchonly" in myopts or \ "--fetch-all-uri" in myopts or \ - myaction in ("metadata", "regen")) + myaction in ("metadata", "regen") or \ + (myaction == "sync" and os.access(settings["PORTDIR"], os.W_OK))) if portage.secpass < 1 or \ need_superuser: if need_superuser: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 30be4d0b0..350dba991 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -820,9 +820,8 @@ def autouse(myvartree, use_cache=1, mysettings=None): def check_config_instance(test): if not isinstance(test, config): - # raise TypeError("Invalid type for config object: %s" % test.__class__) - print "WARNING: Invalid type for config object: %s" % test.__class__ - print "WARNING: no instance of: %s" % config.__class__ + # raise TypeError("Invalid type for config object: %s (should be %s)" % (test.__class__, config)) + print "WARNING: Invalid type for config object: %s (should be %s)" % (test.__class__, config) class config(object): """ diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 576d502b2..786d91c02 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -3,9 +3,6 @@ # Distributed under the terms of the GNU General Public License v2 # $Id: portage_checksum.py 3835 2006-07-11 00:59:10Z zmedico $ -if not hasattr(__builtins__, "set"): - from sets import Set as set - from portage.const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE import os import errno @@ -16,50 +13,88 @@ import portage.process import portage.util import portage.locks import commands -import sha - - -# actual hash functions first +import md5, sha #dict of all available hash functions hashfunc_map = {} +hashorigin_map = {} -# We _try_ to load this module. If it fails we do the slightly slower fallback. -try: - import fchksum +def _generate_hash_function(hashtype, hashobject, origin="unknown"): + def pyhash(filename): + """ + Run a checksum against a file. - def md5hash(filename): - return fchksum.fmd5t(filename) + @param filename: File to run the checksum against + @type filename: String + @return: The hash and size of the data + """ + f = open(filename, 'rb') + blocksize = HASHING_BLOCKSIZE + data = f.read(blocksize) + size = 0L + sum = hashobject() + while data: + sum.update(data) + size = size + len(data) + data = f.read(blocksize) + f.close() -except ImportError: - import md5 - def md5hash(filename): - return pyhash(filename, md5) -hashfunc_map["MD5"] = md5hash + return (sum.hexdigest(), size) + hashfunc_map[hashtype] = pyhash + hashorigin_map[hashtype] = origin + return pyhash + +# Define hash functions, try to use the best module available. Later definitions +# override earlier ones + +# Use the internal modules as last fallback +md5hash = _generate_hash_function("MD5", md5.new, origin="internal") +sha1hash = _generate_hash_function("SHA1", sha.new, origin="internal") -def sha1hash(filename): - return pyhash(filename, sha) -hashfunc_map["SHA1"] = sha1hash +# Use pycrypto when available, prefer it over the internal fallbacks +try: + from Crypto.Hash import MD5, SHA, SHA256, RIPEMD -# Keep pycrypto optional for now, there are no internal fallbacks for these + md5hash = _generate_hash_function("MD5", MD5.new, origin="pycrypto") + sha1hash = _generate_hash_function("SHA1", SHA.new, origin="pycrypto") + sha256hash = _generate_hash_function("SHA256", SHA256.new, origin="pycrypto") + rmd160hash = _generate_hash_function("RMD160", RIPEMD.new, origin="pycrypto") +except ImportError, e: + pass + +# Use hashlib from python-2.5 if available and prefer it over pycrypto and internal fallbacks. +# Need special handling for RMD160 as it may not always be provided by hashlib. try: - import Crypto.Hash.SHA256 + import hashlib - def sha256hash(filename): - return pyhash(filename, Crypto.Hash.SHA256) - hashfunc_map["SHA256"] = sha256hash -except ImportError: + md5hash = _generate_hash_function("MD5", hashlib.md5, origin="hashlib") + sha1hash = _generate_hash_function("SHA1", hashlib.sha1, origin="hashlib") + sha256hash = _generate_hash_function("SHA256", hashlib.sha256, origin="hashlib") + try: + hashlib.new('ripemd160') + except ValueError: + pass + else: + def rmd160(): + return hashlib.new('ripemd160') + rmd160hash = _generate_hash_function("RMD160", rmd160, origin="hashlib") +except ImportError, e: pass + +# Use python-fchksum if available, prefer it over all other MD5 implementations try: - import Crypto.Hash.RIPEMD + import fchksum - def rmd160hash(filename): - return pyhash(filename, Crypto.Hash.RIPEMD) - hashfunc_map["RMD160"] = rmd160hash + def md5hash(filename): + return fchksum.fmd5t(filename) + hashfunc_map["MD5"] = md5hash + hashorigin_map["MD5"] = "python-fchksum" + except ImportError: pass +# There is only one implementation for size def getsize(filename): size = os.stat(filename).st_size return (size, size) @@ -86,6 +121,11 @@ def perform_all(x, calc_prelink=0): def get_valid_checksum_keys(): return hashfunc_map.keys() +def get_hash_origin(hashtype): + if not hashtype in hashfunc_map.keys(): + raise KeyError(hashtype) + return hashorigin_map.get(hashtype, "unknown") + def verify_all(filename, mydict, calc_prelink=0, strict=0): """ Verify all checksums against a file. @@ -145,29 +185,6 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0): break return file_is_ok,reason -def pyhash(filename, hashobject): - """ - Run a checksum against a file. - - @param filename: File to run the checksum against - @type filename: String - @param hashname: The hash object that will execute the checksum on the file - @type hashname: Object - @return: The hash and size of the data - """ - f = open(filename, 'rb') - blocksize = HASHING_BLOCKSIZE - data = f.read(blocksize) - size = 0L - sum = hashobject.new() - while data: - sum.update(data) - size = size + len(data) - data = f.read(blocksize) - f.close() - - return (sum.hexdigest(), size) - def perform_checksum(filename, hashname="MD5", calc_prelink=0): """ Run a specific checksum against a file. @@ -237,6 +254,6 @@ def perform_multiple_checksums(filename, hashes=["MD5"], calc_prelink=0): rVal = {} for x in hashes: if x not in hashfunc_map: - raise portage.exception.DigestException, x+" hash function not available (needs dev-python/pycrypto)" + raise portage.exception.DigestException, x+" hash function not available (needs dev-python/pycrypto or >=dev-lang/python-2.5)" rVal[x] = perform_checksum(filename, x, calc_prelink)[0] return rVal diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 524a36fa2..56c711d13 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -649,6 +649,12 @@ class portdbapi(dbapi): accept_keywords = self.mysettings["ACCEPT_KEYWORDS"].split() pkgdict = self.mysettings.pkeywordsdict aux_keys = ["KEYWORDS", "LICENSE", "EAPI", "SLOT"] + + # Hack: Need to check the env directly here as otherwise stacking + # doesn't work properly as negative values are lost in the config + # object (bug #139600) + egroups = os.environ.get("ACCEPT_KEYWORDS", "").split() + for mycpv in mylist: try: keys, licenses, eapi, slot = self.aux_get(mycpv, aux_keys) @@ -670,6 +676,7 @@ class portdbapi(dbapi): matches = match_to_list(cpv_slot, pkgdict[cp].keys()) for atom in matches: pgroups.extend(pkgdict[cp][atom]) + pgroups.extend(egroups) if matches: # normalize pgroups with incrementals logic so it # matches ACCEPT_KEYWORDS behavior diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index ca7b9f723..c42ec9549 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1188,7 +1188,7 @@ class dblink(object): if is_owned: # A new instance of this package claims the file, so # don't unmerge it. - writemsg_stdout("--- !owned %s %s\n" % \ + writemsg_stdout("--- replaced %s %s\n" % \ (file_type, obj)) continue # next line includes a tweak to protect modules from being unmerged, diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 760fa103c..e4f6e27e3 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -79,14 +79,20 @@ def create_conn(baseurl,conn=None): """(baseurl,conn) --- Takes a protocol://site:port/address url, and an optional connection. If connection is already active, it is passed on. baseurl is reduced to address and is returned in tuple (conn,address)""" + parts = baseurl.split("://",1) if len(parts) != 2: raise ValueError, "Provided URL does not contain protocol identifier. '%s'" % baseurl protocol,url_parts = parts del parts - host,address = url_parts.split("/",1) + + url_parts = url_parts.split("/") + host = url_parts[0] + if len(url_parts) < 2: + address = "/" + else: + address = "/"+"/".join(url_parts[1:]) del url_parts - address = "/"+address userpass_host = host.split("@",1) if len(userpass_host) == 1: diff --git a/pym/portage/update.py b/pym/portage/update.py index 4886998db..c6484a9aa 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -207,7 +207,8 @@ def update_config_files(config_root, protect, protect_mask, update_iter): sys.stdout.write("p") sys.stdout.flush() - write_atomic(os.path.join(config_root, WORLD_FILE), "\n".join(worldlist)) + worldlist.sort() + write_atomic(os.path.join(config_root, WORLD_FILE), "\n".join(worldlist)+"\n") protect_obj = ConfigProtect( config_root, protect, protect_mask)