self._check_world(onProgress)
errors = []
try:
- portage.write_atomic(self.world_file, "\n".join(self.okay))
+ portage.write_atomic(self.world_file,
+ "\n".join(sorted(self.okay)) + "\n")
except portage.exception.PortageException:
errors.append(self.world_file + " could not be opened for writing")
return errors
print "add to world:",myfavkey
worldlist.append(myfavkey)
-portage.write_atomic(os.path.join("/", portage.WORLD_FILE), "\n".join(worldlist))
+portage.write_atomic(os.path.join("/", portage.WORLD_FILE),
+ "\n".join(sorted(worldlist)) + "\n")
.B FETCHCOMMAND
This variable contains the command used for fetching package sources from
the internet. It must contain the full path to the executable as well as the
-place\-holders \\${DISTDIR} and \\${URI}.
+place\-holders \\${DISTDIR}, \\${FILE} and \\${URI}. The command should be
+written to place the fetched file at \\${DISTDIR}/\\${FILE}.
Also see \fBRESUMECOMMAND\fR.
.TP
\fBGENTOO_MIRRORS\fR = \fI[URLs]\fR
.B RESUMECOMMAND
This variable contains the command used for resuming package sources that
have been partially downloaded. It should be defined using the same format
-as \fBFETCHCOMMAND\fR.
+as \fBFETCHCOMMAND\fR, and must include any additional option(s) that may
+be necessary in order to continue a partially downloaded file located at
+\\${DISTDIR}/\\${FILE}.
.TP
\fBROOT\fR = \fI[path]\fR
Use \fBROOT\fR to specify the target root filesystem to be used for merging
# files to fetch list - avoids counting a same file twice
# in size display (verbose mode)
myfetchlist=[]
- worldlist = getlist(self.settings, "world")
+ worldlist = set(getlist(self.settings, "world"))
for mylist_index in xrange(len(mylist)):
x, depth, ordered = mylist[mylist_index]
else:
mydbapi = self.trees[myroot][self.pkg_tree_map[pkg_type]].dbapi
pkg_status = x[3]
+ pkg_merge = ordered and pkg_status != "nomerge"
binary_package = True
if "ebuild" == pkg_type:
if "merge" == x[3] or \
if verbosity == 3:
# size verbose
mysize=0
- if x[0] == "ebuild" and ordered and x[-1] != "nomerge":
+ if pkg_type == "ebuild" and pkg_merge:
try:
myfilesdict = portdb.getfetchsizes(pkg_key,
useflags=self.useFlags[myroot][pkg_key],
myoldbest=myoldbest[:-3]
myoldbest=blue("["+myoldbest+"]")
- if xs[0] in worldlist:
- pkgprint = bold
- else:
- def pkgprint(pkg):
- return pkg
+ pkg_cp = xs[0]
+ pkg_world = pkg_cp in worldlist
+
+ def pkgprint(pkg):
+ if pkg_merge:
+ if pkg_world:
+ return colorize("PKG_MERGE_WORLD", pkg)
+ else:
+ return colorize("PKG_MERGE", pkg)
+ else:
+ if pkg_world:
+ return colorize("PKG_NOMERGE_WORLD", pkg)
+ else:
+ return colorize("PKG_NOMERGE", pkg)
if x[1]!="/":
if myoldbest:
myoldbest +=" "
if "--columns" in self.myopts:
if "--quiet" in self.myopts:
- myprint=addl+" "+indent+darkgreen(pkgprint(xs[0]))
+ myprint=addl+" "+indent+pkgprint(pkg_cp)
myprint=myprint+darkblue(" "+xs[1]+xs[2])+" "
myprint=myprint+myoldbest
myprint=myprint+darkgreen("to "+x[1])
else:
- myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(xs[0]))
+ myprint="["+pkgprint(pkg_type)+" "+addl+"] "+indent+pkgprint(pkg_cp)
if (newlp-nc_len(myprint)) > 0:
myprint=myprint+(" "*(newlp-nc_len(myprint)))
myprint=myprint+"["+darkblue(xs[1]+xs[2])+"] "
myprint=myprint+myoldbest
myprint=myprint+darkgreen("to "+x[1])+" "+verboseadd
else:
- if x[-1] == "nomerge" or not ordered:
- myprint = darkblue("[nomerge ] ")
+ if not pkg_merge:
+ myprint = "[%s ] " % pkgprint("nomerge")
else:
myprint = "[" + pkg_type + " " + addl + "] "
- myprint += indent + darkgreen(pkrprint(pkg_key)) + " " + \
+ myprint += indent + pkgprint(pkg_key) + " " + \
myoldbest + darkgreen("to " + myroot) + " " + \
verboseadd
else:
if "--columns" in self.myopts:
if "--quiet" in self.myopts:
- myprint=addl+" "+indent+darkgreen(pkgprint(xs[0]))
+ myprint=addl+" "+indent+pkgprint(pkg_cp)
myprint=myprint+" "+green(xs[1]+xs[2])+" "
myprint=myprint+myoldbest
else:
- myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(xs[0]))
+ myprint="["+pkgprint(pkg_type)+" "+addl+"] "+indent+pkgprint(pkg_cp)
if (newlp-nc_len(myprint)) > 0:
myprint=myprint+(" "*(newlp-nc_len(myprint)))
myprint=myprint+green(" ["+xs[1]+xs[2]+"] ")
myprint=myprint+(" "*(oldlp-nc_len(myprint)))
myprint=myprint+myoldbest+" "+verboseadd
else:
- if x[-1] == "nomerge" or not ordered:
- myprint=darkblue("[nomerge ] "+indent+pkgprint(x[2])+" "+myoldbest+" ")+verboseadd
+ if not pkg_merge:
+ myprint="["+pkgprint("nomerge")+" ] "+indent+pkgprint(pkg_key)+" "+myoldbest+" "+verboseadd
else:
- myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(pkgprint(x[2]))+" "+myoldbest+" "+verboseadd
+ myprint="["+pkgprint(pkg_type)+" "+addl+"] "+indent+pkgprint(pkg_key)+" "+myoldbest+" "+verboseadd
p.append(myprint)
mysplit = portage.pkgsplit(x[2])
"--pretend" in self.myopts):
portage.write_atomic(
os.path.join(self.target_root, portage.WORLD_FILE),
- "\n".join(myfavdict.values()))
+ "\n".join(sorted(myfavdict.values())) + "\n")
mtimedb["resume"]["mergelist"]=mymergelist[:]
mtimedb.commit()
from portage.const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE
import os
import errno
-import shutil
import stat
import portage.exception
import portage.process
-import portage.util
import portage.locks
import commands
import md5, sha
return hashfunc_map.keys()
def get_hash_origin(hashtype):
- if not hashtype in hashfunc_map.keys():
+ if hashtype not in hashfunc_map:
raise KeyError(hashtype)
return hashorigin_map.get(hashtype, "unknown")
return os.path.join(self.pkgdir, mypath)
def isremote(self, pkgname):
- "Returns true if the package is kept remotely."
- remote = pkgname in self._remotepkgs and \
- not os.path.exists(self.getname(pkgname))
- return remote
+ """Returns true if the package is kept remotely and it has not been
+ downloaded (or it is only partially downloaded)."""
+ if pkgname not in self._remotepkgs:
+ return False
+ pkg_path = self.getname(pkgname)
+ if os.path.exists(pkg_path) and \
+ os.path.basename(pkg_path) not in self.invalids:
+ return False
+ return True
def get_use(self, pkgname):
writemsg("deprecated use of binarytree.get_use()," + \
return self.dbapi.aux_get(pkgname, ["USE"])[0].split()
def gettbz2(self, pkgname):
- "fetches the package from a remote site, if necessary."
+ """Fetches the package from a remote site, if necessary. Attempts to
+ resume if the file appears to be partially downloaded."""
print "Fetching '"+str(pkgname)+"'"
- mysplit = pkgname.split("/")
- tbz2name = mysplit[1]+".tbz2"
- if not self.isremote(pkgname):
+ tbz2_path = self.getname(pkgname)
+ tbz2name = os.path.basename(tbz2_path)
+ resume = False
+ if os.path.exists(tbz2_path):
if (tbz2name not in self.invalids):
return
else:
+ resume = True
writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n",
noiselevel=-1)
- tbz2_path = self.getname(pkgname)
+
mydest = os.path.dirname(self.getname(pkgname))
try:
os.makedirs(mydest, 0775)
except (OSError, IOError):
pass
- from urlparse import urljoin
- base_url = self._remote_base_uri
- fcmd = self.settings["RESUMECOMMAND"]
+ from urlparse import urlparse
+ # urljoin doesn't work correctly with unrecognized protocols like sftp
if self._remote_has_index:
rel_url = self._remotepkgs[pkgname].get("PATH")
if not rel_url:
rel_url = pkgname+".tbz2"
- url = urljoin(base_url, rel_url)
- success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
+ url = self._remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
else:
- url = urljoin(base_url, tbz2name)
- success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
+ url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name
+ protocol = urlparse(url)[0]
+ fcmd_prefix = "FETCHCOMMAND"
+ if resume:
+ fcmd_prefix = "RESUMECOMMAND"
+ fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper())
+ if not fcmd:
+ fcmd = self.settings.get(fcmd_prefix)
+ success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
if success and "strict" in self.settings.features:
metadata = self._remotepkgs[pkgname]
digests = {}
os.unlink(obj)
except (OSError, IOError), e:
pass
- show_unmerge("<<<", "fif", file_type, obj)
+ show_unmerge("<<<", "", file_type, obj)
elif pkgfiles[objkey][0] == "fif":
if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]):
show_unmerge("---", "!fif", file_type, obj)
continue
- show_unmerge("---", "fif", file_type, obj)
+ show_unmerge("---", "", file_type, obj)
elif pkgfiles[objkey][0] == "dev":
- show_unmerge("---", "dev", file_type, obj)
+ show_unmerge("---", "", file_type, obj)
mydirs.sort()
mydirs.reverse()
for obj in mydirs:
try:
os.rmdir(obj)
- show_unmerge("<<<", "dir", file_type, obj)
+ show_unmerge("<<<", "", "dir", obj)
except EnvironmentError:
- show_unmerge("---", "!empty", file_type, obj)
+ show_unmerge("---", "!empty", "dir", obj)
#remove self from vartree database so that our own virtual gets zapped if we're the last node
self.vartree.zap(self.mycpv)
conn.login(username)
conn.set_pasv(passive)
conn.set_debuglevel(0)
+ elif protocol == "sftp":
+ try:
+ import paramiko
+ except ImportError:
+ raise NotImplementedError(
+ "paramiko must be installed for sftp support")
+ t = paramiko.Transport(host)
+ t.connect(username=username, password=password)
+ conn = paramiko.SFTPClient.from_transport(t)
else:
raise NotImplementedError, "%s is not a supported protocol." % protocol
del olddir
else:
listing = conn.nlst(address)
+ elif protocol == "sftp":
+ listing = conn.listdir(address)
else:
raise TypeError, "Unknown protocol. '%s'" % protocol
data,rc,msg = make_http_request(conn, address, params, headers)
elif protocol in ["ftp"]:
data,rc,msg = make_ftp_request(conn, address, -chunk_size)
+ elif protocol == "sftp":
+ f = conn.open(address)
+ try:
+ f.seek(-chunk_size, 2)
+ data = f.read()
+ finally:
+ f.close()
else:
raise TypeError, "Unknown protocol. '%s'" % protocol
if not fcmd:
return file_get_lib(baseurl,dest,conn)
- fcmd = fcmd.replace("${DISTDIR}",dest)
- fcmd = fcmd.replace("${URI}", baseurl)
- fcmd = fcmd.replace("${FILE}", os.path.basename(baseurl))
- mysplit = fcmd.split()
- mycmd = mysplit[0]
- myargs = [os.path.basename(mycmd)]+mysplit[1:]
- mypid=os.fork()
- if mypid == 0:
- try:
- os.execv(mycmd,myargs)
- except OSError:
- pass
- sys.stderr.write("!!! Failed to spawn fetcher.\n")
- sys.stderr.flush()
- os._exit(1)
- retval=os.waitpid(mypid,0)[1]
- if (retval & 0xff) == 0:
- retval = retval >> 8
- else:
- sys.stderr.write("Spawned processes caught a signal.\n")
- sys.exit(1)
- if retval != 0:
+ variables = {
+ "DISTDIR": dest,
+ "URI": baseurl,
+ "FILE": os.path.basename(baseurl)
+ }
+ import shlex, StringIO
+ from portage.util import varexpand
+ from portage.process import spawn
+ lexer = shlex.shlex(StringIO.StringIO(fcmd), posix=True)
+ lexer.whitespace_split = True
+ myfetch = [varexpand(x, mydict=variables) for x in lexer]
+ retval = spawn(myfetch, env=os.environ.copy())
+ if retval != os.EX_OK:
sys.stderr.write("Fetcher exited with a failure condition.\n")
return 0
return 1
data,rc,msg = make_http_request(conn, address, params, headers, dest=dest)
elif protocol in ["ftp"]:
data,rc,msg = make_ftp_request(conn, address, dest=dest)
+ elif protocol == "sftp":
+ rc = 0
+ try:
+ f = conn.open(address)
+ except SystemExit:
+ raise
+ except Exception:
+ rc = 1
+ else:
+ try:
+ if dest:
+ bufsize = 8192
+ while True:
+ data = f.read(bufsize)
+ if not data:
+ break
+ dest.write(data)
+ finally:
+ f.close()
else:
raise TypeError, "Unknown protocol. '%s'" % protocol
codes["UNMERGE_WARN"] = codes["red"]
codes["SECURITY_WARN"] = codes["red"]
codes["MERGE_LIST_PROGRESS"] = codes["yellow"]
+codes["PKG_MERGE"] = codes["darkgreen"]
+codes["PKG_MERGE_WORLD"] = codes["green"]
+codes["PKG_NOMERGE"] = codes["darkblue"]
+codes["PKG_NOMERGE_WORLD"] = codes["blue"]
def parse_color_map():
myfile = COLOR_MAP_FILE
projects / portage.git / commitdiff