Portage writes a compressed copy of 'Packages' index file.

This behaviour is enabled by FEATURES="compress-index". The
resulting file is 'Packages.gz' and its modification time will
match that of 'Packages'.

Web-servers use that copy to avoid repeated on-the-fly compression.

In order to re-use 'atomic_ofstream' usage of 'codecs.zlib_codec'
has been considered and discarded, because 'GzipFile' yields
smaller files. (According to Mark's tests 62% smaller.)

Example usage, Nginx:

  location =/Packages {
    gzip_static on;
    default_type text/plain;
  }

Apache httpd (use with caution):

  RewriteRule ^(.*)/Packages$ $1/Packages.gz [T=text/plain,E=GZIP:gzip,L]
  <FilesMatch "Packages\.gz$">
    Header set Content-Encoding gzip
  </FilesMatch>

diff --git a/man/make.conf.5 b/man/make.conf.5
index 876a8a330fee9ddfe0513e1d4679b601a0da0ac8..02cc8098db85df9a58bceab9f17f5f6ea7cd0e28 100644 (file)
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -268,6 +268,13 @@ space.  Make sure you have built both binutils and gdb with USE=zlib
 support for this to work.  See \fBsplitdebug\fR for general split debug
 information (upon which this feature depends).
 .TP
+.B compress\-index
+If set then a compressed copy of 'Packages' index file will be written.
+This feature is intended for Gentoo binhosts using certain webservers
+(such as, but not limited to, Nginx with gzip_static module) to avoid
+redundant on\-the\-fly compression.  The resulting file will be called
+'Packages.gz' and its modification time will match that of 'Packages'.
+.TP
 .B config\-protect\-if\-modified
 This causes the \fBCONFIG_PROTECT\fR behavior to be skipped for files
 that have not been modified since they were installed. This feature is

diff --git a/pym/portage/const.py b/pym/portage/const.py
index ceef5c56b4f350553dfcea547709a37e372f16e2..c2049f8718937d75c9a8bd8556f113d6793cdd40 100644 (file)
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -89,7 +89,7 @@ SUPPORTED_FEATURES       = frozenset([
                            "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
                            "ccache", "chflags", "clean-logs",
                            "collision-protect", "compress-build-logs", "compressdebug",
-                           "config-protect-if-modified",
+                           "compress-index", "config-protect-if-modified",
                            "digest", "distcc", "distcc-pump", "distlocks",
                            "downgrade-backup", "ebuild-locks", "fakeroot",
                            "fail-clean", "force-mirror", "force-prefix", "getbinpkg",

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 03675031e57086051876113104d14e18c0e77fdc..77fc0c4ef947069985584a3c1b7c46f8e74b3317 100644 (file)
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -41,6 +41,7 @@ import sys
 import tempfile
 import textwrap
 import warnings
+from gzip import GzipFile
 from itertools import chain
 try:
        from urllib.parse import urlparse
@@ -1186,13 +1187,21 @@ class binarytree(object):
                        pkgindex.packages.append(d)
 
                        self._update_pkgindex_header(pkgindex.header)
-                       pkgindex_filename = os.path.join(self.pkgdir, "Packages")
-                       f = atomic_ofstream(pkgindex_filename)
-                       pkgindex.write(f)
-                       f.close()
-                       # some seconds might have elapsed since TIMESTAMP
+                       contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO())
+                       pkgindex.write(contents)
+                       contents = contents.getvalue()
                        atime = mtime = long(pkgindex.header["TIMESTAMP"])
-                       os.utime(pkgindex_filename, (atime, mtime))
+
+                       pkgindex_filename = os.path.join(self.pkgdir, "Packages")
+                       output_files = [(atomic_ofstream(pkgindex_filename, mode="wb"), pkgindex_filename)]
+                       if "compress-index" in self.settings.features:
+                               gz_fname = pkgindex_filename + ".gz"
+                               output_files.append((GzipFile(gz_fname, mode="wb"), gz_fname))
+                       for f, fname in output_files:
+                               f.write(contents)
+                               f.close()
+                               # some seconds might have elapsed since TIMESTAMP
+                               os.utime(fname, (atime, mtime))
                finally:
                        if pkgindex_lock:
                                unlockfile(pkgindex_lock)