From: Fabian Groffen Date: Mon, 25 Jun 2007 12:07:54 +0000 (-0000) Subject: Merged from trunk 6995:7031 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=a681beae27a47f5df6149cc194979dabef4b5d9b;p=portage.git Merged from trunk 6995:7031 +------------------------------------------------------------------------+ | 6996 | Fix more typos from r6988. | | zmedico | | |---------+--------------------------------------------------------------| | 6999 | For bug #180706, add sorting and a trailing newline in the | | zmedico | rest of the cases where the world file is written. | |---------+--------------------------------------------------------------| | 7002 | Fix broken base_uri handling for old-style binhosts in | | zmedico | binarytree.gettbz2(). | |---------+--------------------------------------------------------------| | 7004 | Update the FETCHCOMMAND docs to indicate that the command | | zmedico | should be +written to place the fetched file at | | | ${DISTDIR}/${FILE}. | |---------+--------------------------------------------------------------| | 7005 | Clarify RESUMECOMMAND requirements. | | zmedico | | |---------+--------------------------------------------------------------| | 7008 | Use a set to optimize containment lookups. | | zmedico | | |---------+--------------------------------------------------------------| | 7011 | Remove unused imports and unnecessary dict.keys() call. | | zmedico | | |---------+--------------------------------------------------------------| | 7012 | Fix binarytree.isremote() to be consistent with | | zmedico | binarytree.gettbz2() logic wrt partially downloaded files. | |---------+--------------------------------------------------------------| | 7014 | Clean up file_get() to use shlex, varexpand, and spawn. | | zmedico | | |---------+--------------------------------------------------------------| | 7016 | For bug #175891, support ${FETCHCOMMAND}_${PROTOCOL} in | | zmedico | binarytree.gettbz2(). | |---------+--------------------------------------------------------------| | 7018 | Fix resume logic in binarytree.gettbz2(). | | zmedico | | |---------+--------------------------------------------------------------| | 7019 | Fix typo. | | zmedico | | |---------+--------------------------------------------------------------| | 7023 | For bug #175891, add sftp protocol support via the paramiko | | zmedico | library. | |---------+--------------------------------------------------------------| | 7024 | Fix file_get_lib() so it behaves the same way with sftp as | | zmedico | it does with other protocols. | |---------+--------------------------------------------------------------| | 7025 | Allow SystemExit through as usual. | | zmedico | | |---------+--------------------------------------------------------------| | 7027 | Don't use urljoin() in binarytree.gettbz2() since it doesn't | | zmedico | work correctly wih unrecognized protocols like sftp. | |---------+--------------------------------------------------------------| | 7029 | Fix typo. | | zmedico | | |---------+--------------------------------------------------------------| | 7031 | For bug #142473, add 4 color classes corresponding to | | zmedico | combinations of world/merge/nomerge attributes of each | | | package. | +------------------------------------------------------------------------+ svn path=/main/branches/prefix/; revision=7033 --- diff --git a/bin/emaint b/bin/emaint index 25b0be450..caef62a32 100755 --- a/bin/emaint +++ b/bin/emaint @@ -69,7 +69,8 @@ class WorldHandler(object): self._check_world(onProgress) errors = [] try: - portage.write_atomic(self.world_file, "\n".join(self.okay)) + portage.write_atomic(self.world_file, + "\n".join(sorted(self.okay)) + "\n") except portage.exception.PortageException: errors.append(self.world_file + " could not be opened for writing") return errors diff --git a/bin/regenworld b/bin/regenworld index c93a41185..c35bdf845 100755 --- a/bin/regenworld +++ b/bin/regenworld @@ -92,4 +92,5 @@ for mykey in biglist: print "add to world:",myfavkey worldlist.append(myfavkey) -portage.write_atomic(os.path.join("/", portage.WORLD_FILE), "\n".join(worldlist)) +portage.write_atomic(os.path.join("/", portage.WORLD_FILE), + "\n".join(sorted(worldlist)) + "\n") diff --git a/man/make.conf.5 b/man/make.conf.5 index 10fdbf25e..0f33e53c2 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -290,7 +290,8 @@ Enable the sandbox in the compile phase, when running without root privs (\fIuse .B FETCHCOMMAND This variable contains the command used for fetching package sources from the internet. It must contain the full path to the executable as well as the -place\-holders \\${DISTDIR} and \\${URI}. +place\-holders \\${DISTDIR}, \\${FILE} and \\${URI}. The command should be +written to place the fetched file at \\${DISTDIR}/\\${FILE}. Also see \fBRESUMECOMMAND\fR. .TP \fBGENTOO_MIRRORS\fR = \fI[URLs]\fR @@ -454,7 +455,9 @@ settings from ebuilds. See also \fBebuild\fR(5). .B RESUMECOMMAND This variable contains the command used for resuming package sources that have been partially downloaded. It should be defined using the same format -as \fBFETCHCOMMAND\fR. +as \fBFETCHCOMMAND\fR, and must include any additional option(s) that may +be necessary in order to continue a partially downloaded file located at +\\${DISTDIR}/\\${FILE}. .TP \fBROOT\fR = \fI[path]\fR Use \fBROOT\fR to specify the target root filesystem to be used for merging diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 96b589ca0..af25e81e6 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -2479,7 +2479,7 @@ class depgraph(object): # files to fetch list - avoids counting a same file twice # in size display (verbose mode) myfetchlist=[] - worldlist = getlist(self.settings, "world") + worldlist = set(getlist(self.settings, "world")) for mylist_index in xrange(len(mylist)): x, depth, ordered = mylist[mylist_index] @@ -2516,6 +2516,7 @@ class depgraph(object): else: mydbapi = self.trees[myroot][self.pkg_tree_map[pkg_type]].dbapi pkg_status = x[3] + pkg_merge = ordered and pkg_status != "nomerge" binary_package = True if "ebuild" == pkg_type: if "merge" == x[3] or \ @@ -2689,7 +2690,7 @@ class depgraph(object): if verbosity == 3: # size verbose mysize=0 - if x[0] == "ebuild" and ordered and x[-1] != "nomerge": + if pkg_type == "ebuild" and pkg_merge: try: myfilesdict = portdb.getfetchsizes(pkg_key, useflags=self.useFlags[myroot][pkg_key], @@ -2749,23 +2750,32 @@ class depgraph(object): myoldbest=myoldbest[:-3] myoldbest=blue("["+myoldbest+"]") - if xs[0] in worldlist: - pkgprint = bold - else: - def pkgprint(pkg): - return pkg + pkg_cp = xs[0] + pkg_world = pkg_cp in worldlist + + def pkgprint(pkg): + if pkg_merge: + if pkg_world: + return colorize("PKG_MERGE_WORLD", pkg) + else: + return colorize("PKG_MERGE", pkg) + else: + if pkg_world: + return colorize("PKG_NOMERGE_WORLD", pkg) + else: + return colorize("PKG_NOMERGE", pkg) if x[1]!="/": if myoldbest: myoldbest +=" " if "--columns" in self.myopts: if "--quiet" in self.myopts: - myprint=addl+" "+indent+darkgreen(pkgprint(xs[0])) + myprint=addl+" "+indent+pkgprint(pkg_cp) myprint=myprint+darkblue(" "+xs[1]+xs[2])+" " myprint=myprint+myoldbest myprint=myprint+darkgreen("to "+x[1]) else: - myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(xs[0])) + myprint="["+pkgprint(pkg_type)+" "+addl+"] "+indent+pkgprint(pkg_cp) if (newlp-nc_len(myprint)) > 0: myprint=myprint+(" "*(newlp-nc_len(myprint))) myprint=myprint+"["+darkblue(xs[1]+xs[2])+"] " @@ -2774,21 +2784,21 @@ class depgraph(object): myprint=myprint+myoldbest myprint=myprint+darkgreen("to "+x[1])+" "+verboseadd else: - if x[-1] == "nomerge" or not ordered: - myprint = darkblue("[nomerge ] ") + if not pkg_merge: + myprint = "[%s ] " % pkgprint("nomerge") else: myprint = "[" + pkg_type + " " + addl + "] " - myprint += indent + darkgreen(pkrprint(pkg_key)) + " " + \ + myprint += indent + pkgprint(pkg_key) + " " + \ myoldbest + darkgreen("to " + myroot) + " " + \ verboseadd else: if "--columns" in self.myopts: if "--quiet" in self.myopts: - myprint=addl+" "+indent+darkgreen(pkgprint(xs[0])) + myprint=addl+" "+indent+pkgprint(pkg_cp) myprint=myprint+" "+green(xs[1]+xs[2])+" " myprint=myprint+myoldbest else: - myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(xs[0])) + myprint="["+pkgprint(pkg_type)+" "+addl+"] "+indent+pkgprint(pkg_cp) if (newlp-nc_len(myprint)) > 0: myprint=myprint+(" "*(newlp-nc_len(myprint))) myprint=myprint+green(" ["+xs[1]+xs[2]+"] ") @@ -2796,10 +2806,10 @@ class depgraph(object): myprint=myprint+(" "*(oldlp-nc_len(myprint))) myprint=myprint+myoldbest+" "+verboseadd else: - if x[-1] == "nomerge" or not ordered: - myprint=darkblue("[nomerge ] "+indent+pkgprint(x[2])+" "+myoldbest+" ")+verboseadd + if not pkg_merge: + myprint="["+pkgprint("nomerge")+" ] "+indent+pkgprint(pkg_key)+" "+myoldbest+" "+verboseadd else: - myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(x[2]))+" "+myoldbest+" "+verboseadd + myprint="["+pkgprint(pkg_type)+" "+addl+"] "+indent+pkgprint(pkg_key)+" "+myoldbest+" "+verboseadd p.append(myprint) mysplit = portage.pkgsplit(x[2]) @@ -3073,7 +3083,7 @@ class MergeTask(object): "--pretend" in self.myopts): portage.write_atomic( os.path.join(self.target_root, portage.WORLD_FILE), - "\n".join(myfavdict.values())) + "\n".join(sorted(myfavdict.values())) + "\n") mtimedb["resume"]["mergelist"]=mymergelist[:] mtimedb.commit() diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 786d91c02..4b3e82712 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -6,11 +6,9 @@ from portage.const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE import os import errno -import shutil import stat import portage.exception import portage.process -import portage.util import portage.locks import commands import md5, sha @@ -122,7 +120,7 @@ def get_valid_checksum_keys(): return hashfunc_map.keys() def get_hash_origin(hashtype): - if not hashtype in hashfunc_map.keys(): + if hashtype not in hashfunc_map: raise KeyError(hashtype) return hashorigin_map.get(hashtype, "unknown") diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 1930bc71d..9898639e1 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -829,10 +829,15 @@ class binarytree(object): return os.path.join(self.pkgdir, mypath) def isremote(self, pkgname): - "Returns true if the package is kept remotely." - remote = pkgname in self._remotepkgs and \ - not os.path.exists(self.getname(pkgname)) - return remote + """Returns true if the package is kept remotely and it has not been + downloaded (or it is only partially downloaded).""" + if pkgname not in self._remotepkgs: + return False + pkg_path = self.getname(pkgname) + if os.path.exists(pkg_path) and \ + os.path.basename(pkg_path) not in self.invalids: + return False + return True def get_use(self, pkgname): writemsg("deprecated use of binarytree.get_use()," + \ @@ -840,34 +845,42 @@ class binarytree(object): return self.dbapi.aux_get(pkgname, ["USE"])[0].split() def gettbz2(self, pkgname): - "fetches the package from a remote site, if necessary." + """Fetches the package from a remote site, if necessary. Attempts to + resume if the file appears to be partially downloaded.""" print "Fetching '"+str(pkgname)+"'" - mysplit = pkgname.split("/") - tbz2name = mysplit[1]+".tbz2" - if not self.isremote(pkgname): + tbz2_path = self.getname(pkgname) + tbz2name = os.path.basename(tbz2_path) + resume = False + if os.path.exists(tbz2_path): if (tbz2name not in self.invalids): return else: + resume = True writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n", noiselevel=-1) - tbz2_path = self.getname(pkgname) + mydest = os.path.dirname(self.getname(pkgname)) try: os.makedirs(mydest, 0775) except (OSError, IOError): pass - from urlparse import urljoin - base_url = self._remote_base_uri - fcmd = self.settings["RESUMECOMMAND"] + from urlparse import urlparse + # urljoin doesn't work correctly with unrecognized protocols like sftp if self._remote_has_index: rel_url = self._remotepkgs[pkgname].get("PATH") if not rel_url: rel_url = pkgname+".tbz2" - url = urljoin(base_url, rel_url) - success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd) + url = self._remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/") else: - url = urljoin(base_url, tbz2name) - success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd) + url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name + protocol = urlparse(url)[0] + fcmd_prefix = "FETCHCOMMAND" + if resume: + fcmd_prefix = "RESUMECOMMAND" + fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper()) + if not fcmd: + fcmd = self.settings.get(fcmd_prefix) + success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd) if success and "strict" in self.settings.features: metadata = self._remotepkgs[pkgname] digests = {} diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index d72081c7d..9cc264cc4 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1277,14 +1277,14 @@ class dblink(object): os.unlink(obj) except (OSError, IOError), e: pass - show_unmerge("<<<", "fif", file_type, obj) + show_unmerge("<<<", "", file_type, obj) elif pkgfiles[objkey][0] == "fif": if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]): show_unmerge("---", "!fif", file_type, obj) continue - show_unmerge("---", "fif", file_type, obj) + show_unmerge("---", "", file_type, obj) elif pkgfiles[objkey][0] == "dev": - show_unmerge("---", "dev", file_type, obj) + show_unmerge("---", "", file_type, obj) mydirs.sort() mydirs.reverse() @@ -1292,9 +1292,9 @@ class dblink(object): for obj in mydirs: try: os.rmdir(obj) - show_unmerge("<<<", "dir", file_type, obj) + show_unmerge("<<<", "", "dir", obj) except EnvironmentError: - show_unmerge("---", "!empty", file_type, obj) + show_unmerge("---", "!empty", "dir", obj) #remove self from vartree database so that our own virtual gets zapped if we're the last node self.vartree.zap(self.mycpv) diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index e4f6e27e3..a1e349bd9 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -142,6 +142,15 @@ def create_conn(baseurl,conn=None): conn.login(username) conn.set_pasv(passive) conn.set_debuglevel(0) + elif protocol == "sftp": + try: + import paramiko + except ImportError: + raise NotImplementedError( + "paramiko must be installed for sftp support") + t = paramiko.Transport(host) + t.connect(username=username, password=password) + conn = paramiko.SFTPClient.from_transport(t) else: raise NotImplementedError, "%s is not a supported protocol." % protocol @@ -308,6 +317,8 @@ def dir_get_list(baseurl,conn=None): del olddir else: listing = conn.nlst(address) + elif protocol == "sftp": + listing = conn.listdir(address) else: raise TypeError, "Unknown protocol. '%s'" % protocol @@ -333,6 +344,13 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000): data,rc,msg = make_http_request(conn, address, params, headers) elif protocol in ["ftp"]: data,rc,msg = make_ftp_request(conn, address, -chunk_size) + elif protocol == "sftp": + f = conn.open(address) + try: + f.seek(-chunk_size, 2) + data = f.read() + finally: + f.close() else: raise TypeError, "Unknown protocol. '%s'" % protocol @@ -367,28 +385,19 @@ def file_get(baseurl,dest,conn=None,fcmd=None): if not fcmd: return file_get_lib(baseurl,dest,conn) - fcmd = fcmd.replace("${DISTDIR}",dest) - fcmd = fcmd.replace("${URI}", baseurl) - fcmd = fcmd.replace("${FILE}", os.path.basename(baseurl)) - mysplit = fcmd.split() - mycmd = mysplit[0] - myargs = [os.path.basename(mycmd)]+mysplit[1:] - mypid=os.fork() - if mypid == 0: - try: - os.execv(mycmd,myargs) - except OSError: - pass - sys.stderr.write("!!! Failed to spawn fetcher.\n") - sys.stderr.flush() - os._exit(1) - retval=os.waitpid(mypid,0)[1] - if (retval & 0xff) == 0: - retval = retval >> 8 - else: - sys.stderr.write("Spawned processes caught a signal.\n") - sys.exit(1) - if retval != 0: + variables = { + "DISTDIR": dest, + "URI": baseurl, + "FILE": os.path.basename(baseurl) + } + import shlex, StringIO + from portage.util import varexpand + from portage.process import spawn + lexer = shlex.shlex(StringIO.StringIO(fcmd), posix=True) + lexer.whitespace_split = True + myfetch = [varexpand(x, mydict=variables) for x in lexer] + retval = spawn(myfetch, env=os.environ.copy()) + if retval != os.EX_OK: sys.stderr.write("Fetcher exited with a failure condition.\n") return 0 return 1 @@ -410,6 +419,25 @@ def file_get_lib(baseurl,dest,conn=None): data,rc,msg = make_http_request(conn, address, params, headers, dest=dest) elif protocol in ["ftp"]: data,rc,msg = make_ftp_request(conn, address, dest=dest) + elif protocol == "sftp": + rc = 0 + try: + f = conn.open(address) + except SystemExit: + raise + except Exception: + rc = 1 + else: + try: + if dest: + bufsize = 8192 + while True: + data = f.read(bufsize) + if not data: + break + dest.write(data) + finally: + f.close() else: raise TypeError, "Unknown protocol. '%s'" % protocol diff --git a/pym/portage/output.py b/pym/portage/output.py index 33223d5b4..008a10fc7 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -129,6 +129,10 @@ codes["INFORM"] = codes["darkgreen"] codes["UNMERGE_WARN"] = codes["red"] codes["SECURITY_WARN"] = codes["red"] codes["MERGE_LIST_PROGRESS"] = codes["yellow"] +codes["PKG_MERGE"] = codes["darkgreen"] +codes["PKG_MERGE_WORLD"] = codes["green"] +codes["PKG_NOMERGE"] = codes["darkblue"] +codes["PKG_NOMERGE_WORLD"] = codes["blue"] def parse_color_map(): myfile = COLOR_MAP_FILE