* Make portdbapi.aux_get() automatically pull pre-generated metadata

  directly from the metadata/cache/ directory when metadata-transfer
  is not enabled in FEATURES. This makes all cache modules behave
  similar to existing metadata_overlay module, except when
  FEATURES="metadata-transfer" has been explicitly enabled.

* Remove metadata-transfer from FEATURES in make.globals so that it's
  no longer enabled by default. Users will be happy about this since
  the "Updating Portage cache" part of emerge --sync can be more time
  consuming than the rsync run itself. It's also nicer when when using
  PORTDIR over nfs or bind mounted in a chroot since there's no need
  to run emerge --metadata.

* Update FEATURES="metadata-transfer" docs to warn users of the rsync
  tree to modify eclases in PORTDIR_OVERLAY if necessary instead of
  doing it directly in the rsync tree.

* Relevant updates to /etc/portage/modules docs.

(trunk r9930)

svn path=/main/branches/2.1.2/; revision=9931

diff --git a/cnf/make.globals b/cnf/make.globals
index 86c2104d3e3196dd9a03fff3b1d53a9d5763b210..63c68d841722e4d9e178ba15f6e62938ca490aa6 100644 (file)
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -38,7 +38,7 @@ FETCHCOMMAND="/usr/bin/wget -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\"
 RESUMECOMMAND="/usr/bin/wget -c -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
 
 # Default user options
-FEATURES="distlocks parallel-fetch metadata-transfer sandbox sfperms strict unmerge-orphans userfetch"
+FEATURES="distlocks parallel-fetch sandbox sfperms strict unmerge-orphans userfetch"
 
 # Default chunksize for binhost comms
 PORTAGE_BINHOST_CHUNKSIZE="3000"

diff --git a/man/make.conf.5 b/man/make.conf.5
index 4d012c060a2ab4f7846951d536c4249af70ab13a..0d0f8549c69455fb42001cfb561e5f421e9a973c 100644 (file)
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -215,6 +215,16 @@ when \fImirror\fR is also in the \fBebuild\fR(5) \fBRESTRICT\fR variable.
 .TP
 .B metadata\-transfer
 Automatically perform a metadata transfer when `emerge \-\-sync` is run.
+In versions of portage greater than 2.1.5, this feature is disabled by
+default. When metadata\-transfer is disabled, metadata cache from the
+${PORTDIR}/metadata/cache/ directory will be used directly (if available)
+and eclasses in ${PORTDIR}/eclass/ must not be modified except by
+`emerge \-\-sync` operations since the cache validation mechanism
+will not recognize eclass modifications. Normally, this issue only
+pertains to users of the rsync tree since the cvs tree does not contain
+a metadata/cache/ directory. Users of the rsync tree who want to modify
+eclasses should use \fBPORTDIR_OVERLAY\fR in order for the cache
+validation mechanism to work correctly.
 .TP
 .B mirror
 Fetch everything in \fBSRC_URI\fR regardless of \fBUSE\fR settings,

diff --git a/man/portage.5 b/man/portage.5
index 6728ebe281f2761d35993b65b4eadbef225e0b9e..5d953db83a476d1b2f3bd9d659f8e5d7d24ac5e3 100644 (file)
--- a/man/portage.5
+++ b/man/portage.5
@@ -387,18 +387,15 @@ override.
 
 .I Example:
 .nf
-portdbapi.auxdbmodule = cache.metadata_overlay.database
-.fi
-
-The metadata_overlay cache module makes it possible to disable
-FEATURES="metadata-transfer" in \fBmake.conf\fR(5).  When the user initially
-enables metadata_overlay in /etc/portage/modules, all of the cache files
-contained in /var/cache/edb/dep/${PORTDIR} must be manually removed in order
-to avoid unecessary cache regeneration.  In addition, users of the
-metadata_overlay module must never modify eclasses in ${PORTDIR} because
-portage will not be able to detect that cache regeneration is necessary.  If
-the user would like to modify eclasses, it is safe to use metadata_overlay
-together with PORTDIR_OVERLAY in \fBmake.conf\fR.
+portdbapi.auxdbmodule = cache.sqlite.database
+.fi
+
+After changing the portdbapi.auxdbmodule setting, it may be necessary to
+transfer or regenerate metadata cache. Users of the rsync tree need to
+run `emerge \-\-metadata` if they have enabled FEATURES="metadata-transfer"
+in \fBmake.conf\fR(5). In order to regenerate metadata for repositories
+listed in \fBPORTDIR_OVERLAY\fR or a cvs tree, run `emerge \-\-regen`
+(see \fBemerge\fR(1)).
 .TP
 .BR package.keywords
 Per\-package KEYWORDS.  Useful for mixing unstable packages in with a normally 

diff --git a/pym/portage.py b/pym/portage.py
index db1ba59fb7a0358624e450e964e704109d965e81..430c7872c8e78405459b56126c1a0d4dc8bb9ce8 100644 (file)
--- a/pym/portage.py
+++ b/pym/portage.py
@@ -7170,7 +7170,8 @@ class portdbapi(dbapi):
                                pass
 
                self.auxdbmodule  = self.mysettings.load_best_module("portdbapi.auxdbmodule")
-               self.auxdb        = {}
+               self.auxdb = {}
+               self._pregen_auxdb = {}
                self._init_cache_dirs()
                # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
                # ~harring
@@ -7189,6 +7190,11 @@ class portdbapi(dbapi):
                                # location, label, auxdbkeys
                                self.auxdb[x] = self.auxdbmodule(
                                        self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid)
+               if "metadata-transfer" not in self.mysettings.features:
+                       for x in self.porttrees:
+                               if os.path.isdir(os.path.join(x, "metadata", "cache")):
+                                       self._pregen_auxdb[x] = self.metadbmodule(
+                                               x, "metadata/cache", filtered_auxdbkeys, readonly=True)
                # Selectively cache metadata in order to optimize dep matching.
                self._aux_cache_keys = set(
                        ["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE",
@@ -7350,26 +7356,41 @@ class portdbapi(dbapi):
                                noiselevel=-1)
                        raise KeyError(mycpv)
 
-               try:
-                       mydata = self.auxdb[mylocation][mycpv]
-                       eapi = mydata.get("EAPI","").strip()
-                       if not eapi:
-                               eapi = "0"
-                       if eapi.startswith("-") and eapi_is_supported(eapi[1:]):
-                               doregen = True
-                       elif emtime != long(mydata.get("_mtime_", 0)):
-                               doregen = True
-                       elif len(mydata.get("_eclasses_", [])) > 0:
-                               doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"])
-                       else:
-                               doregen = False
-                               
-               except KeyError:
-                       doregen = True
-               except CacheError:
-                       doregen = True
-                       try:                            del self.auxdb[mylocation][mycpv]
-                       except KeyError:        pass
+               # Pull pre-generated metadata from the metadata/cache/
+               # directory if it exists and is valid, otherwise fall
+               # back to the normal writable cache.
+               auxdbs = []
+               pregen_auxdb = self._pregen_auxdb.get(mylocation)
+               if pregen_auxdb is not None:
+                       auxdbs.append(pregen_auxdb)
+               auxdbs.append(self.auxdb[mylocation])
+
+               doregen = True
+               for auxdb in auxdbs:
+                       try:
+                               mydata = auxdb[mycpv]
+                               eapi = mydata.get("EAPI","").strip()
+                               if not eapi:
+                                       eapi = "0"
+                               if eapi.startswith("-") and eapi_is_supported(eapi[1:]):
+                                       pass
+                               elif emtime != long(mydata.get("_mtime_", 0)):
+                                       pass
+                               elif len(mydata.get("_eclasses_", [])) > 0:
+                                       if self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"]):
+                                               doregen = False
+                               else:
+                                       doregen = False
+                       except KeyError:
+                               pass
+                       except CacheError:
+                               if auxdb is not pregen_auxdb:
+                                       try:
+                                               del auxdb[mycpv]
+                                       except KeyError:
+                                               pass
+                       if not doregen:
+                               break
 
                writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2)