return flags
return None
- def create(self, mybigkey, myparent=None, addme=1, myuse=None,
+ def create(self, mybigkey, myparent=None, addme=1, metadata=None,
priority=DepPriority(), rev_dep=False, arg=None):
"""
Fills the digraph with nodes comprised of packages to merge.
# unused parameters
rev_dep = False
- myuse = None
mytype, myroot, mykey = mybigkey
# directive, otherwise we add a "merge" directive.
mydbapi = self.trees[myroot][self.pkg_tree_map[mytype]].dbapi
- metadata = dict(izip(self._mydbapi_keys,
- mydbapi.aux_get(mykey, self._mydbapi_keys)))
- if mytype == "ebuild":
- pkgsettings.setcpv(mykey, mydb=portdb)
- metadata["USE"] = pkgsettings["USE"]
- myuse = pkgsettings["USE"].split()
- else:
- # The myuse parameter to this method is deprecated, so get it
- # directly from the metadata here.
- myuse = metadata["USE"].split()
+ if metadata is None:
+ metadata = dict(izip(self._mydbapi_keys,
+ mydbapi.aux_get(mykey, self._mydbapi_keys)))
+ if mytype == "ebuild":
+ pkgsettings.setcpv(mykey, mydb=portdb)
+ metadata["USE"] = pkgsettings["USE"]
+ myuse = metadata["USE"].split()
if not arg and myroot == self.target_root:
try:
if "empty" not in self.myparams and vardbapi.cpv_exists(mykey):
merging=0
+ merge_node = (mytype, myroot, mykey, "merge")
+ if self.digraph.contains(merge_node):
+ merging = 1
+
""" If we aren't merging, perform the --newuse check.
If the package has new iuse flags or different use flags then if
--newuse is specified, we need to merge the package. """
print colorize("BAD", "\n*** You need to adjust PKGDIR to emerge this package.\n")
return 0, myfavorites
if not self.create(["binary", myroot, mykey],
- None, "--onlydeps" not in self.myopts,
- myuse=mytbz2.getelements("USE"), arg=x):
+ addme=("--onlydeps" not in self.myopts), arg=x):
return (0,myfavorites)
arg_atoms.append((x, "="+mykey))
elif ext==".ebuild":
for x in mymerge:
selected_pkg = None
if x[0]=="!":
- selected_pkg = ["blocks", myroot, x[1:], None]
+ selected_pkg = (["blocks", myroot, x[1:]], None)
else:
#We are not processing a blocker but a normal dependency
if myparent:
matched_packages = []
myeb_matches = portdb.xmatch("match-visible", x)
myeb = None
- if "--usepkgonly" not in self.myopts:
+ myeb_pkg = None
+ metadata = None
+ existing_node = None
+ if myeb_matches:
myeb = portage.best(myeb_matches)
+ # For best performance, try to reuse an exising node
+ # and it's cached metadata. The portdbapi caches SLOT
+ # metadata in memory so it's really only pulled once.
+ slot_atom = "%s:%s" % (portage.dep_getkey(myeb),
+ portdb.aux_get(myeb, ["SLOT"])[0])
+ existing_node = self._slot_node_map[myroot].get(slot_atom)
+ if existing_node:
+ e_type, myroot, e_cpv, e_status = existing_node
+ metadata = dict(izip(self._mydbapi_keys,
+ self.mydbapi[myroot].aux_get(e_cpv, self._mydbapi_keys)))
+ cpv_slot = "%s:%s" % (e_cpv, metadata["SLOT"])
+ if portage.match_from_list(x, [cpv_slot]):
+ matched_packages.append(
+ ([e_type, myroot, e_cpv], metadata))
+ else:
+ existing_node = None
- myeb_pkg=None
- if "--usepkg" in self.myopts:
+ if not existing_node and \
+ "--usepkg" in self.myopts:
# The next line assumes the binarytree has been populated.
# XXX: Need to work out how we use the binary tree with roots.
myeb_pkg_matches = bindb.match(x)
not portdb.cpv_exists(pkg)]
if myeb_pkg_matches:
myeb_pkg = portage.best(myeb_pkg_matches)
-
- if myeb_pkg and \
+ # For best performance, try to reuse an exising node
+ # and it's cached metadata. The bindbapi caches SLOT
+ # metadata in memory so it's really only pulled once.
+ slot_atom = "%s:%s" % (portage.dep_getkey(myeb_pkg),
+ bindb.aux_get(myeb_pkg, ["SLOT"])[0])
+ existing_node = self._slot_node_map[myroot].get(slot_atom)
+ if existing_node:
+ e_type, myroot, e_cpv, e_status = existing_node
+ metadata = dict(izip(self._mydbapi_keys,
+ self.mydbapi[myroot].aux_get(e_cpv, self._mydbapi_keys)))
+ cpv_slot = "%s:%s" % (e_cpv, metadata["SLOT"])
+ if portage.match_from_list(x, [cpv_slot]):
+ myeb_pkg = None
+ matched_packages.append(
+ ([e_type, myroot, e_cpv], metadata))
+ else:
+ existing_node = None
+ if not existing_node:
+ # For best performance, avoid pulling
+ # metadata whenever possible.
+ metadata = dict(izip(self._mydbapi_keys,
+ bindb.aux_get(myeb_pkg, self._mydbapi_keys)))
+
+ if not existing_node and \
+ myeb_pkg and \
("--newuse" in self.myopts or \
"--reinstall" in self.myopts):
- iuses = set(filter_iuse_defaults(
- bindb.aux_get(myeb_pkg, ["IUSE"])[0].split()))
- old_use = bindb.aux_get(myeb_pkg, ["USE"])[0].split()
+ iuses = set(filter_iuse_defaults(metadata["IUSE"].split()))
+ old_use = metadata["USE"].split()
mydb = None
if "--usepkgonly" not in self.myopts and myeb:
mydb = portdb
forced_flags, old_use, iuses, now_use, cur_iuse):
myeb_pkg = None
if myeb_pkg:
- binpkguseflags = \
- self.trees[myroot]["bintree"].dbapi.aux_get(
- myeb_pkg, ["USE"])[0].split()
matched_packages.append(
- ["binary", myroot, myeb_pkg, binpkguseflags])
-
- if "--usepkgonly" not in self.myopts and myeb_matches:
+ (["binary", myroot, myeb_pkg], metadata))
+
+ if not existing_node and \
+ myeb and \
+ "--usepkgonly" not in self.myopts:
+ metadata = dict(izip(self._mydbapi_keys,
+ portdb.aux_get(myeb, self._mydbapi_keys)))
+ pkgsettings.setcpv(myeb, mydb=portdb)
+ metadata["USE"] = pkgsettings["USE"]
matched_packages.append(
- ["ebuild", myroot, myeb, None])
+ (["ebuild", myroot, myeb], metadata))
if not matched_packages and \
not (arg and "selective" not in self.myparams):
if myeb_inst_matches:
myeb_inst = portage.best(myeb_inst_matches)
if myeb_inst:
- binpkguseflags = vardb.aux_get(
- myeb_inst, ["USE"])[0].split()
+ metadata = dict(izip(self._mydbapi_keys,
+ vardb.aux_get(myeb_inst, self._mydbapi_keys)))
matched_packages.append(
- ["installed", myroot, myeb_inst, binpkguseflags])
+ (["installed", myroot, myeb_inst], metadata))
if not matched_packages:
if raise_on_missing:
return 0
if "--debug" in self.myopts:
- for pkg in matched_packages:
+ for pkg, metadata in matched_packages:
print (pkg[0] + ":").rjust(10), pkg[2]
if len(matched_packages) > 1:
bestmatch = portage.best(
- [pkg[2] for pkg in matched_packages])
+ [pkg[2] for pkg, metadata in matched_packages])
matched_packages = [pkg for pkg in matched_packages \
- if pkg[2] == bestmatch]
+ if pkg[0][2] == bestmatch]
# ordered by type preference ("ebuild" type is the last resort)
selected_pkg = matched_packages[0]
- pkgtype, myroot, mycpv, myuse = selected_pkg
- mydbapi = self.trees[myroot][self.pkg_tree_map[pkgtype]].dbapi
- slot_atom = "%s:%s" % (portage.dep_getkey(mycpv),
- mydbapi.aux_get(mycpv, ["SLOT"])[0])
- existing_node = self._slot_node_map[myroot].get(
- slot_atom, None)
- if existing_node:
- e_type, myroot, e_cpv, e_status = existing_node
- if portage.match_from_list(x, [e_cpv]):
- # The existing node can be reused.
- # Just pass in None for myuse since
- # self.create() doesn't use it anymore.
- selected_pkg = [e_type, myroot, e_cpv, None]
if myparent:
#we are a dependency, so we want to be unconditionally added
mypriority = priority.copy()
if vardb.match(x):
mypriority.satisfied = True
- if not self.create(selected_pkg[0:3], myparent,
- myuse=selected_pkg[-1], priority=mypriority,
+ if not self.create(selected_pkg[0], myparent=myparent,
+ metadata=selected_pkg[1], priority=mypriority,
rev_dep=rev_deps, arg=arg):
return 0
else:
#if mysource is not set, then we are a command-line dependency and should not be added
#if --onlydeps is specified.
- if not self.create(selected_pkg[0:3], myparent,
+ if not self.create(selected_pkg[0], myparent=myparent,
addme=("--onlydeps" not in self.myopts),
- myuse=selected_pkg[-1], rev_dep=rev_deps, arg=arg):
+ metadata=selected_pkg[1], rev_dep=rev_deps, arg=arg):
return 0
if "--debug" in self.myopts:
# Load the most current variables from ${ROOT}/etc/profile.env
settings.unlock()
+ settings.reload()
settings.regenerate()
settings.lock()
print header_title.rjust(int(header_width/2 + len(header_title)/2))
print header_width * "="
print "System uname: "+unameout
- gentoo_release = portage.grabfile(os.path.join(
- settings["PORTAGE_CONFIGROOT"] + EPREFIX, "etc", "gentoo-release"))
- if gentoo_release:
- print gentoo_release[0]
- else:
- print "Unknown Host Operating System"
lastSync = portage.grabfile(os.path.join(
settings["PORTDIR"], "metadata", "timestamp.chk"))
print "Timestamp of tree:",
writemsg("--- 'profiles/arch.list' is empty or not available. Empty portage tree?\n")
else:
for group in groups:
- if group not in archlist and group[0] != '-':
+ if group not in archlist and \
+ not (group.startswith("-") and group[1:] in archlist) and \
+ group not in ("*", "~*", "**"):
writemsg("!!! INVALID ACCEPT_KEYWORDS: %s\n" % str(group),
noiselevel=-1)
if modified:
self.virtuals = self.__getvirtuals_compile()
+ def reload(self):
+ """Reload things like /etc/profile.env that can change during runtime."""
+ env_d_filename = os.path.join(self["ROOT"], EPREFIX_LSTRIP, "etc", "profile.env")
+ self.configdict["env.d"].clear()
+ env_d = getconfig(env_d_filename, expand=False)
+ if env_d:
+ # env_d will be None if profile.env doesn't exist.
+ self.configdict["env.d"].update(env_d)
def regenerate(self,useonly=0,use_cache=1):
"""
else:
self.already_in_regenerate = 1
- # We grab the latest profile.env here since it changes frequently.
- self.configdict["env.d"].clear()
- env_d = getconfig(
- os.path.join(self["ROOT"], EPREFIX_LSTRIP, "etc", "profile.env"), expand=False)
- if env_d:
- # env_d will be None if profile.env doesn't exist.
- self.configdict["env.d"].update(env_d)
-
if useonly:
myincrementals=["USE"]
else:
"ebuild's files must be downloaded"
print "!!! manually. See the comments in" + \
" the ebuild for more information.\n"
+ mysettings["EBUILD_PHASE"] = "unpack"
spawn(EBUILD_SH_BINARY + " nofetch", mysettings)
elif listonly:
continue
phase_retval = spawn(actionmap[mydo]["cmd"] % mydo, mysettings, debug=debug, logfile=logfile, **kwargs)
mysettings["EBUILD_PHASE"] = ""
- if not kwargs["droppriv"] and secpass >= 2:
+ if "userpriv" in mysettings.features and \
+ not kwargs["droppriv"] and secpass >= 2:
""" Privileged phases may have left files that need to be made
writable to a less privileged user."""
apply_recursive_permissions(mysettings["T"],
detects a package-specific change in config. For the ebuild
environment, a reset call is forced in order to ensure that the
latest env.d variables are used."""
+ mysettings.reload()
mysettings.reset(use_cache=use_cache)
mysettings.setcpv(mycpv, use_cache=use_cache, mydb=mydbapi)
makepickle = CACHE_PATH+"/metadata.idx.most_recent"
conn,protocol,address,params,headers = create_conn(baseurl, conn)
-
+ out = sys.stdout
try:
metadatafile = open(CACHE_PATH+"/remote_metadata.pickle")
metadata = cPickle.load(metadatafile)
- sys.stderr.write("Loaded metadata pickle.\n")
+ out.write("Loaded metadata pickle.\n")
+ out.flush()
metadatafile.close()
except (cPickle.UnpicklingError, OSError, IOError, EOFError):
metadata = {}
sys.stderr.write("--- "+str(e)+"\n")
if trynum < 3:
sys.stderr.write("Retrying...\n")
+ sys.stderr.flush()
mytempfile.close()
continue
if match_in_array([mfile],suffix=".gz"):
- sys.stderr.write("gzip'd\n")
+ out.write("gzip'd\n")
+ out.flush()
try:
import gzip
mytempfile.seek(0)
except Exception, e:
mytempfile.close()
sys.stderr.write("!!! Failed to use gzip: "+str(e)+"\n")
+ sys.stderr.flush()
mytempfile.close()
try:
metadata[baseurl]["data"] = cPickle.loads(data)
metadata[baseurl]["indexname"] = mfile
metadata[baseurl]["timestamp"] = int(time.time())
metadata[baseurl]["modified"] = 0 # It's not, right after download.
- sys.stderr.write("Pickle loaded.\n")
+ out.write("Pickle loaded.\n")
+ out.flush()
break
except SystemExit, e:
raise
except Exception, e:
sys.stderr.write("!!! Failed to read data from index: "+str(mfile)+"\n")
sys.stderr.write("!!! "+str(e)+"\n")
+ sys.stderr.flush()
try:
metadatafile = open(CACHE_PATH+"/remote_metadata.pickle", "w+")
cPickle.dump(metadata,metadatafile)
except Exception, e:
sys.stderr.write("!!! Failed to write binary metadata to disk!\n")
sys.stderr.write("!!! "+str(e)+"\n")
+ sys.stderr.flush()
break
# We may have metadata... now we run through the tbz2 list and check.
- sys.stderr.write(yellow("cache miss: 'x'")+" --- "+green("cache hit: 'o'")+"\n")
+
+ class CacheStats(object):
+ from time import time
+ def __init__(self, out):
+ self.misses = 0
+ self.hits = 0
+ self.last_update = 0
+ self.out = out
+ self.min_display_latency = 0.2
+ def update(self):
+ cur_time = self.time()
+ if cur_time - self.last_update >= self.min_display_latency:
+ self.last_update = cur_time
+ self.display()
+ def display(self):
+ self.out.write("\r"+yellow("cache miss: '"+str(self.misses)+"'")+\
+ " --- "+green("cache hit: '"+str(self.hits)+"'"))
+ self.out.flush()
+
+ cache_stats = CacheStats(out)
+ have_tty = out.isatty()
+ if have_tty:
+ cache_stats.display()
binpkg_filenames = set()
for x in tbz2list:
x = os.path.basename(x)
binpkg_filenames.add(x)
if x not in metadata[baseurl]["data"]:
- sys.stderr.write(yellow("x"))
+ cache_stats.misses += 1
+ if have_tty:
+ cache_stats.update()
metadata[baseurl]["modified"] = 1
myid = None
for retry in xrange(3):
metadata[baseurl]["data"][x] = make_metadata_dict(myid)
elif verbose:
sys.stderr.write(red("!!! Failed to retrieve metadata on: ")+str(x)+"\n")
+ sys.stderr.flush()
else:
- sys.stderr.write(green("o"))
+ cache_stats.hits += 1
+ if have_tty:
+ cache_stats.update()
+ cache_stats.display()
# Cleanse stale cache for files that don't exist on the server anymore.
stale_cache = set(metadata[baseurl]["data"]).difference(binpkg_filenames)
if stale_cache:
metadata[baseurl]["modified"] = 1
del stale_cache
del binpkg_filenames
- sys.stderr.write("\n")
-
+ out.write("\n")
+ out.flush()
+
try:
if metadata[baseurl].has_key("modified") and metadata[baseurl]["modified"]:
metadata[baseurl]["timestamp"] = int(time.time())
except Exception, e:
sys.stderr.write("!!! Failed to write binary metadata to disk!\n")
sys.stderr.write("!!! "+str(e)+"\n")
+ sys.stderr.flush()
if not keepconnection:
conn.close()
projects / portage.git / commitdiff