LAST_E_CMD LAST_E_LEN LD_PRELOAD MISC_FUNCTIONS_ARGS MOPREFIX \
NORMAL PKGDIR PKGUSE PKG_LOGDIR PKG_TMPDIR \
PORTAGE_ACTUAL_DISTDIR PORTAGE_ARCHLIST PORTAGE_BASHRC \
- PORTAGE_BINPKG_FILE \
PORTAGE_BINPKG_TAR_OPTS PORTAGE_BINPKG_TMPFILE PORTAGE_BUILDDIR \
PORTAGE_COLORMAP PORTAGE_CONFIGROOT PORTAGE_DEBUG \
PORTAGE_DEPCACHEDIR PORTAGE_GID PORTAGE_INST_GID \
it will increment it. For more information about nice levels and what
are acceptable ranges, see \fBnice\fR(1).
.TP
+\fBPORTAGE_RO_DISTDIRS\fR = \fI[space delimited list of directories]\fR
+When a given file does not exist in \fBDISTDIR\fR, search for the file
+in this list of directories. Search order is from left to right. Note
+that the current implementation works by creating a symlink inside
+\fBDISTDIR\fR, but that may change in the future.
+.TP
\fBPORTAGE_RSYNC_INITIAL_TIMEOUT\fR = \fIinteger\fR
Used by \fBemerge \-\-sync\fR as a timeout for the initial connection to an
rsync server.
1) the set of installed packages (including COUNTER) has changed
2) the old-style virtuals have changed
"""
+
+ # Number of uncached packages to trigger cache update, since
+ # it's wasteful to update it for every vdb change.
+ _cache_threshold = 5
+
class BlockerData(object):
__slots__ = ("__weakref__", "atoms", "counter")
portage.CACHE_PATH.lstrip(os.path.sep), "vdb_blockers.pickle")
self._cache_version = "1"
self._cache_data = None
- self._modified = False
+ self._modified = 0
self._load()
def _load(self):
self._cache_data = {"version":self._cache_version}
self._cache_data["blockers"] = {}
self._cache_data["virtuals"] = self._virtuals
- self._modified = False
+ self._modified = 0
def flush(self):
"""If the current user has permission and the internal blocker cache
"virtuals" : vardb.settings.getvirtuals()
}
"""
- if self._modified and \
+ if self._modified >= self._cache_threshold and \
secpass >= 2:
try:
f = portage.util.atomic_ofstream(self._cache_filename)
self._cache_filename, gid=portage.portage_gid, mode=0644)
except (IOError, OSError), e:
pass
- self._modified = False
+ self._modified = 0
def __setitem__(self, cpv, blocker_data):
"""
"""
self._cache_data["blockers"][cpv] = \
(blocker_data.counter, tuple(str(x) for x in blocker_data.atoms))
- self._modified = True
+ self._modified += 1
def __iter__(self):
return iter(self._cache_data["blockers"])
def __delitem__(self, cpv):
del self._cache_data["blockers"][cpv]
- self._modified = True
def __getitem__(self, cpv):
"""
for pkg in vardb:
cpv = pkg.cpv
stale_cache.discard(cpv)
+ pkg_in_graph = self.digraph.contains(pkg)
# Check for masked installed packages. For keyword
# mask there are a couple of common cases that are
# TODO: Share visibility code to fix this inconsistency.
if pkg in final_db:
- if not visible(pkgsettings, pkg):
+ if pkg_in_graph and not visible(pkgsettings, pkg):
self._masked_installed.add(pkg)
elif graph_complete_for_root and \
pkgsettings.getMissingKeywords(
pkg.cpv, pkg.metadata) and \
pkg.metadata["KEYWORDS"].split() and \
- not self.digraph.contains(pkg):
+ not pkg_in_graph:
self._masked_installed.add(pkg)
blocker_atoms = None
blockers = None
- if self.digraph.contains(pkg):
+ if pkg_in_graph:
blockers = []
try:
blockers.extend(
"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
"PORTAGE_GPG_DIR",
"PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
+ "PORTAGE_RO_DISTDIRS",
"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
"PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
"QUICKPKG_DEFAULT_OPTS",
os.rename(filename, temp_filename)
return temp_filename
+def _check_digests(filename, digests):
+ """
+ Check digests and displey a message if an error occurs.
+ @return True if all digests match, False otherwise.
+ """
+ verified_ok, reason = portage.checksum.verify_all(filename, digests)
+ if not verified_ok:
+ writemsg("!!! Previously fetched" + \
+ " file: '%s'\n" % filename, noiselevel=-1)
+ writemsg("!!! Reason: %s\n" % reason[0],
+ noiselevel=-1)
+ writemsg(("!!! Got: %s\n" + \
+ "!!! Expected: %s\n") % \
+ (reason[1], reason[2]), noiselevel=-1)
+ return False
+ return True
+
+def _check_distfile(filename, digests, eout):
+ """
+ @return a tuple of (match, stat_obj) where match is True if filename
+ matches all given digests (if any) and stat_obj is a stat result, or
+ None if the file does not exist.
+ """
+ if digests is None:
+ digests = {}
+ size = digests.get("size")
+ if size is not None and len(digests) == 1:
+ digests = None
+
+ try:
+ st = os.stat(filename)
+ except OSError:
+ return (False, None)
+ if size is not None and size != st.st_size:
+ return (False, st)
+ if not digests:
+ if size is not None:
+ eout.ebegin("%s %s ;-)" % (os.path.basename(filename), "size"))
+ eout.eend(0)
+ else:
+ if _check_digests(filename, digests):
+ eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
+ " ".join(sorted(digests))))
+ eout.eend(0)
+ else:
+ return (False, st)
+ return (True, st)
+
_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
_size_suffix_map = {
# no digests because fetch was not called for a specific package
mydigests = {}
+ import shlex
+ ro_distdirs = [x for x in \
+ shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
+ if os.path.isdir(x)]
+
fsmirrors = []
for x in range(len(mymirrors)-1,-1,-1):
if mymirrors[x] and mymirrors[x][0]=='/':
1 partially downloaded
2 completely downloaded
"""
+ fetched = 0
+
+ orig_digests = mydigests.get(myfile, {})
+ size = orig_digests.get("size")
+ pruned_digests = orig_digests
+ if parallel_fetchonly:
+ pruned_digests = {}
+ if size is not None:
+ pruned_digests["size"] = size
+
myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
- fetched=0
has_space = True
file_lock = None
if listonly:
waiting_msg=waiting_msg)
try:
if not listonly:
+
+ eout = portage.output.EOutput()
+ eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
+ match, mystat = _check_distfile(
+ myfile_path, pruned_digests, eout)
+ if match:
+ if can_fetch and not fetch_to_ro:
+ try:
+ apply_secpass_permissions(myfile_path,
+ gid=portage_gid, mode=0664, mask=02,
+ stat_cached=mystat)
+ except portage.exception.PortageException, e:
+ if not os.access(myfile_path, os.R_OK):
+ writemsg("!!! Failed to adjust permissions:" + \
+ " %s\n" % str(e), noiselevel=-1)
+ del e
+ continue
+
+ if can_fetch and mystat is None:
+ # Remove broken symlinks if necessary.
+ try:
+ os.unlink(myfile_path)
+ except OSError:
+ pass
+
+ if mystat is not None:
+ if mystat.st_size == 0:
+ if can_fetch:
+ try:
+ os.unlink(myfile_path)
+ except OSError:
+ pass
+ elif can_fetch:
+ if mystat.st_size < fetch_resume_size and \
+ mystat.st_size < size:
+ writemsg((">>> Deleting distfile with size " + \
+ "%d (smaller than " "PORTAGE_FETCH_RESU" + \
+ "ME_MIN_SIZE)\n") % mystat.st_size)
+ try:
+ os.unlink(myfile_path)
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ raise
+ del e
+ elif mystat.st_size >= size:
+ temp_filename = \
+ _checksum_failure_temp_file(
+ mysettings["DISTDIR"], myfile)
+ writemsg_stdout("Refetching... " + \
+ "File renamed to '%s'\n\n" % \
+ temp_filename, noiselevel=-1)
+
+ if can_fetch and ro_distdirs:
+ readonly_file = None
+ for x in ro_distdirs:
+ filename = os.path.join(x, myfile)
+ match, mystat = _check_distfile(
+ filename, pruned_digests, eout)
+ if match:
+ readonly_file = filename
+ break
+ if readonly_file is not None:
+ try:
+ os.unlink(myfile_path)
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ raise
+ del e
+ os.symlink(readonly_file, myfile_path)
+ continue
+
if fsmirrors and not os.path.exists(myfile_path) and has_space:
for mydir in fsmirrors:
mirror_file = os.path.join(mydir, myfile)
_excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \
"|".join(_excluded_dirs) + r')$')
+ # Number of uncached packages to trigger cache update, since
+ # it's wasteful to update it for every vdb change.
+ _aux_cache_threshold = 5
+
_aux_cache_keys_re = re.compile(r'^NEEDED\..*$')
_aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$')
return list(self._iter_match(mydep,
self.cp_list(mydep.cp, use_cache=use_cache)))
try:
- curmtime = os.stat(self.root+VDB_PATH+"/"+mycat)[stat.ST_MTIME]
+ curmtime = os.stat(self.root+VDB_PATH+"/"+mycat).st_mtime
except (IOError, OSError):
curmtime=0
users have read access and benefit from faster metadata lookups (as
long as at least part of the cache is still valid)."""
if self._aux_cache is not None and \
- self._aux_cache["modified"] and \
+ self._aux_cache["modified"] >= self._aux_cache_threshold and \
secpass >= 2:
valid_nodes = set(self.cpv_all())
for cpv in self._aux_cache["packages"].keys():
self._aux_cache_filename, gid=portage_gid, mode=0644)
except (IOError, OSError), e:
pass
- self._aux_cache["modified"] = False
+ self._aux_cache["modified"] = 0
def aux_get(self, mycpv, wants):
"""This automatically caches selected keys that are frequently needed
not self._aux_cache.get("packages"):
self._aux_cache = {"version": self._aux_cache_version}
self._aux_cache["packages"] = {}
- self._aux_cache["modified"] = False
+ self._aux_cache["modified"] = 0
mydir = self.getpath(mycpv)
mydir_stat = None
try:
for aux_key in cache_these:
cache_data[aux_key] = mydata[aux_key]
self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data)
- self._aux_cache["modified"] = True
+ self._aux_cache["modified"] += 1
return [mydata[x] for x in wants]
def _aux_get(self, mycpv, wants):
projects / portage.git / commitdiff