From 664bc1909118aaebf2d1c228025b847e5b2ca3c8 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Tue, 3 Jun 2008 15:46:05 +0000 Subject: [PATCH] Merged from trunk 10545:10553 | 10546 | Don't unset PORTAGE_BINPKG_FILE because it's readonly. | | zmedico | | | 10547 | Add support for a PORTAGE_RO_DISTDIRS variable. When a given | | zmedico | file does not exist in DISTDIR, search for the file in this | | | list of directories. Search order is from left to right. | | | Note that the current implementation works by creating a | | | symlink inside DISTDIR, but that may change in the future. | | 10548 | Fix vardbapi.flush_cache() so that it only updates the cache | | zmedico | when the number of uncached packages reaches a certain | | | threshold (currently 5). The cache file can be several | | | megabytes in size, so updating it for every vdb change is | | | wasteful. | | 10549 | Add modification threshold for BlockerCache.flush(), to | | zmedico | avoid wasteful IO. | | 10550 | Use float mtime instead of int for finer granularity in | | zmedico | vardbapi.match() cache staleness checks. | | 10551 | When pruning the digests to optimize parallel-fetch, use a | | zmedico | separate dict so that the original digests are still | | | available if a file needs to be downloaded. | | 10552 | Fix typo. | | zmedico | | | 10553 | Tweak the installed masked code from bug #223447 so that it | | zmedico | less noisy by suppressing masking messages for packages that | | | haven't been pulled into the current dependency graph. | svn path=/main/branches/prefix/; revision=10556 --- bin/isolated-functions.sh | 1 - man/make.conf.5 | 6 ++ pym/_emerge/__init__.py | 23 +++--- pym/portage/__init__.py | 136 ++++++++++++++++++++++++++++++++++- pym/portage/dbapi/vartree.py | 14 ++-- 5 files changed, 164 insertions(+), 16 deletions(-) diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index d12fb64a7..ddb8e040e 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -487,7 +487,6 @@ save_ebuild_env() { LAST_E_CMD LAST_E_LEN LD_PRELOAD MISC_FUNCTIONS_ARGS MOPREFIX \ NORMAL PKGDIR PKGUSE PKG_LOGDIR PKG_TMPDIR \ PORTAGE_ACTUAL_DISTDIR PORTAGE_ARCHLIST PORTAGE_BASHRC \ - PORTAGE_BINPKG_FILE \ PORTAGE_BINPKG_TAR_OPTS PORTAGE_BINPKG_TMPFILE PORTAGE_BUILDDIR \ PORTAGE_COLORMAP PORTAGE_CONFIGROOT PORTAGE_DEBUG \ PORTAGE_DEPCACHEDIR PORTAGE_GID PORTAGE_INST_GID \ diff --git a/man/make.conf.5 b/man/make.conf.5 index ed33ff3a7..f854a2538 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -451,6 +451,12 @@ emerge is running at. In other words, this will not set the nice level, it will increment it. For more information about nice levels and what are acceptable ranges, see \fBnice\fR(1). .TP +\fBPORTAGE_RO_DISTDIRS\fR = \fI[space delimited list of directories]\fR +When a given file does not exist in \fBDISTDIR\fR, search for the file +in this list of directories. Search order is from left to right. Note +that the current implementation works by creating a symlink inside +\fBDISTDIR\fR, but that may change in the future. +.TP \fBPORTAGE_RSYNC_INITIAL_TIMEOUT\fR = \fIinteger\fR Used by \fBemerge \-\-sync\fR as a timeout for the initial connection to an rsync server. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index d1bd2029d..bb3bab358 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1477,6 +1477,11 @@ class BlockerCache(DictMixin): 1) the set of installed packages (including COUNTER) has changed 2) the old-style virtuals have changed """ + + # Number of uncached packages to trigger cache update, since + # it's wasteful to update it for every vdb change. + _cache_threshold = 5 + class BlockerData(object): __slots__ = ("__weakref__", "atoms", "counter") @@ -1492,7 +1497,7 @@ class BlockerCache(DictMixin): portage.CACHE_PATH.lstrip(os.path.sep), "vdb_blockers.pickle") self._cache_version = "1" self._cache_data = None - self._modified = False + self._modified = 0 self._load() def _load(self): @@ -1562,7 +1567,7 @@ class BlockerCache(DictMixin): self._cache_data = {"version":self._cache_version} self._cache_data["blockers"] = {} self._cache_data["virtuals"] = self._virtuals - self._modified = False + self._modified = 0 def flush(self): """If the current user has permission and the internal blocker cache @@ -1580,7 +1585,7 @@ class BlockerCache(DictMixin): "virtuals" : vardb.settings.getvirtuals() } """ - if self._modified and \ + if self._modified >= self._cache_threshold and \ secpass >= 2: try: f = portage.util.atomic_ofstream(self._cache_filename) @@ -1590,7 +1595,7 @@ class BlockerCache(DictMixin): self._cache_filename, gid=portage.portage_gid, mode=0644) except (IOError, OSError), e: pass - self._modified = False + self._modified = 0 def __setitem__(self, cpv, blocker_data): """ @@ -1604,14 +1609,13 @@ class BlockerCache(DictMixin): """ self._cache_data["blockers"][cpv] = \ (blocker_data.counter, tuple(str(x) for x in blocker_data.atoms)) - self._modified = True + self._modified += 1 def __iter__(self): return iter(self._cache_data["blockers"]) def __delitem__(self, cpv): del self._cache_data["blockers"][cpv] - self._modified = True def __getitem__(self, cpv): """ @@ -3481,6 +3485,7 @@ class depgraph(object): for pkg in vardb: cpv = pkg.cpv stale_cache.discard(cpv) + pkg_in_graph = self.digraph.contains(pkg) # Check for masked installed packages. For keyword # mask there are a couple of common cases that are @@ -3506,18 +3511,18 @@ class depgraph(object): # TODO: Share visibility code to fix this inconsistency. if pkg in final_db: - if not visible(pkgsettings, pkg): + if pkg_in_graph and not visible(pkgsettings, pkg): self._masked_installed.add(pkg) elif graph_complete_for_root and \ pkgsettings.getMissingKeywords( pkg.cpv, pkg.metadata) and \ pkg.metadata["KEYWORDS"].split() and \ - not self.digraph.contains(pkg): + not pkg_in_graph: self._masked_installed.add(pkg) blocker_atoms = None blockers = None - if self.digraph.contains(pkg): + if pkg_in_graph: blockers = [] try: blockers.extend( diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index e75178bec..fd9990bf1 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -996,6 +996,7 @@ class config(object): "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE", "PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT", + "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR", "QUICKPKG_DEFAULT_OPTS", @@ -3116,6 +3117,54 @@ def _checksum_failure_temp_file(distdir, basename): os.rename(filename, temp_filename) return temp_filename +def _check_digests(filename, digests): + """ + Check digests and displey a message if an error occurs. + @return True if all digests match, False otherwise. + """ + verified_ok, reason = portage.checksum.verify_all(filename, digests) + if not verified_ok: + writemsg("!!! Previously fetched" + \ + " file: '%s'\n" % filename, noiselevel=-1) + writemsg("!!! Reason: %s\n" % reason[0], + noiselevel=-1) + writemsg(("!!! Got: %s\n" + \ + "!!! Expected: %s\n") % \ + (reason[1], reason[2]), noiselevel=-1) + return False + return True + +def _check_distfile(filename, digests, eout): + """ + @return a tuple of (match, stat_obj) where match is True if filename + matches all given digests (if any) and stat_obj is a stat result, or + None if the file does not exist. + """ + if digests is None: + digests = {} + size = digests.get("size") + if size is not None and len(digests) == 1: + digests = None + + try: + st = os.stat(filename) + except OSError: + return (False, None) + if size is not None and size != st.st_size: + return (False, st) + if not digests: + if size is not None: + eout.ebegin("%s %s ;-)" % (os.path.basename(filename), "size")) + eout.eend(0) + else: + if _check_digests(filename, digests): + eout.ebegin("%s %s ;-)" % (os.path.basename(filename), + " ".join(sorted(digests)))) + eout.eend(0) + else: + return (False, st) + return (True, st) + _fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)') _size_suffix_map = { @@ -3249,6 +3298,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", # no digests because fetch was not called for a specific package mydigests = {} + import shlex + ro_distdirs = [x for x in \ + shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \ + if os.path.isdir(x)] + fsmirrors = [] for x in range(len(mymirrors)-1,-1,-1): if mymirrors[x] and mymirrors[x][0]=='/': @@ -3376,8 +3430,17 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", 1 partially downloaded 2 completely downloaded """ + fetched = 0 + + orig_digests = mydigests.get(myfile, {}) + size = orig_digests.get("size") + pruned_digests = orig_digests + if parallel_fetchonly: + pruned_digests = {} + if size is not None: + pruned_digests["size"] = size + myfile_path = os.path.join(mysettings["DISTDIR"], myfile) - fetched=0 has_space = True file_lock = None if listonly: @@ -3423,6 +3486,77 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", waiting_msg=waiting_msg) try: if not listonly: + + eout = portage.output.EOutput() + eout.quiet = mysettings.get("PORTAGE_QUIET") == "1" + match, mystat = _check_distfile( + myfile_path, pruned_digests, eout) + if match: + if can_fetch and not fetch_to_ro: + try: + apply_secpass_permissions(myfile_path, + gid=portage_gid, mode=0664, mask=02, + stat_cached=mystat) + except portage.exception.PortageException, e: + if not os.access(myfile_path, os.R_OK): + writemsg("!!! Failed to adjust permissions:" + \ + " %s\n" % str(e), noiselevel=-1) + del e + continue + + if can_fetch and mystat is None: + # Remove broken symlinks if necessary. + try: + os.unlink(myfile_path) + except OSError: + pass + + if mystat is not None: + if mystat.st_size == 0: + if can_fetch: + try: + os.unlink(myfile_path) + except OSError: + pass + elif can_fetch: + if mystat.st_size < fetch_resume_size and \ + mystat.st_size < size: + writemsg((">>> Deleting distfile with size " + \ + "%d (smaller than " "PORTAGE_FETCH_RESU" + \ + "ME_MIN_SIZE)\n") % mystat.st_size) + try: + os.unlink(myfile_path) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + elif mystat.st_size >= size: + temp_filename = \ + _checksum_failure_temp_file( + mysettings["DISTDIR"], myfile) + writemsg_stdout("Refetching... " + \ + "File renamed to '%s'\n\n" % \ + temp_filename, noiselevel=-1) + + if can_fetch and ro_distdirs: + readonly_file = None + for x in ro_distdirs: + filename = os.path.join(x, myfile) + match, mystat = _check_distfile( + filename, pruned_digests, eout) + if match: + readonly_file = filename + break + if readonly_file is not None: + try: + os.unlink(myfile_path) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + os.symlink(readonly_file, myfile_path) + continue + if fsmirrors and not os.path.exists(myfile_path) and has_space: for mydir in fsmirrors: mirror_file = os.path.join(mydir, myfile) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 7e1cab2a5..664189833 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -338,6 +338,10 @@ class vardbapi(dbapi): _excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \ "|".join(_excluded_dirs) + r')$') + # Number of uncached packages to trigger cache update, since + # it's wasteful to update it for every vdb change. + _aux_cache_threshold = 5 + _aux_cache_keys_re = re.compile(r'^NEEDED\..*$') _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') @@ -633,7 +637,7 @@ class vardbapi(dbapi): return list(self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))) try: - curmtime = os.stat(self.root+VDB_PATH+"/"+mycat)[stat.ST_MTIME] + curmtime = os.stat(self.root+VDB_PATH+"/"+mycat).st_mtime except (IOError, OSError): curmtime=0 @@ -659,7 +663,7 @@ class vardbapi(dbapi): users have read access and benefit from faster metadata lookups (as long as at least part of the cache is still valid).""" if self._aux_cache is not None and \ - self._aux_cache["modified"] and \ + self._aux_cache["modified"] >= self._aux_cache_threshold and \ secpass >= 2: valid_nodes = set(self.cpv_all()) for cpv in self._aux_cache["packages"].keys(): @@ -674,7 +678,7 @@ class vardbapi(dbapi): self._aux_cache_filename, gid=portage_gid, mode=0644) except (IOError, OSError), e: pass - self._aux_cache["modified"] = False + self._aux_cache["modified"] = 0 def aux_get(self, mycpv, wants): """This automatically caches selected keys that are frequently needed @@ -719,7 +723,7 @@ class vardbapi(dbapi): not self._aux_cache.get("packages"): self._aux_cache = {"version": self._aux_cache_version} self._aux_cache["packages"] = {} - self._aux_cache["modified"] = False + self._aux_cache["modified"] = 0 mydir = self.getpath(mycpv) mydir_stat = None try: @@ -764,7 +768,7 @@ class vardbapi(dbapi): for aux_key in cache_these: cache_data[aux_key] = mydata[aux_key] self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) - self._aux_cache["modified"] = True + self._aux_cache["modified"] += 1 return [mydata[x] for x in wants] def _aux_get(self, mycpv, wants): -- 2.26.2