Optimize `emaint` --fix binhost so that it the Packages file isn't re-read

and re-written for each package that's updated. Instead, hold a lock for
the whole time the command is running and just update it once.

svn path=/main/trunk/; revision=11307

diff --git a/bin/emaint b/bin/emaint
index b9b7a6676c8b23a48b5d8231dff2355418ac39d2..561d267197a5df2e9007a553214a758773582cb6 100755 (executable)
--- a/bin/emaint
+++ b/bin/emaint
@@ -112,8 +112,7 @@ class BinhostHandler(object):
                self._bintree = portage.db[myroot]["bintree"]
                self._bintree.populate()
                self._pkgindex_file = os.path.join(self._bintree.pkgdir, "Packages")
-               from portage import getbinpkg
-               self._pkgindex = getbinpkg.PackageIndex()
+               self._pkgindex = self._bintree._new_pkgindex()
                f = open(self._pkgindex_file, 'r')
                try:
                        self._pkgindex.read(f)
@@ -149,7 +148,7 @@ class BinhostHandler(object):
                cpv_all = self._bintree.dbapi.cpv_all()
                cpv_all.sort()
                missing = []
-               maxval = len(cpv_all)
+               maxval = 0
                if onProgress:
                        onProgress(maxval, 0)
                pkgindex = self._pkgindex
@@ -157,35 +156,61 @@ class BinhostHandler(object):
                metadata = {}
                for d in pkgindex.packages:
                        metadata[d["CPV"]] = d
+
                for i, cpv in enumerate(cpv_all):
                        d = metadata.get(cpv)
                        if not d or "MD5" not in d:
-                               bintree.inject(cpv)
-                       if onProgress:
-                               onProgress(maxval, i+1)
+                               missing.append(cpv)
+
                stale = set(metadata).difference(cpv_all)
-               if stale:
+               if missing or stale:
                        from portage import locks
                        pkgindex_lock = locks.lockfile(
                                self._pkgindex_file, wantnewlockfile=1)
                        try:
-                               from portage import getbinpkg
-                               pkgindex = getbinpkg.PackageIndex()
+                               # Repopulate with lock held.
+                               bintree._populate()
+                               cpv_all = self._bintree.dbapi.cpv_all()
+                               cpv_all.sort()
+
+                               pkgindex = bintree._new_pkgindex()
                                self._pkgindex = pkgindex
                                f = open(self._pkgindex_file, 'r')
                                try:
                                        self._pkgindex.read(f)
                                finally:
                                        f.close()
-                               from portage.dbapi.bintree import binarytree
-                               self._bintree = binarytree(bintree.root, bintree.pkgdir,
-                                       settings=bintree.settings)
-                               del bintree
-                               portage.db[self._bintree.root]["bintree"] = self._bintree
-                               self._bintree._populate()
+
+                               metadata = {}
+                               for d in pkgindex.packages:
+                                       metadata[d["CPV"]] = d
+
+                               # Recount missing packages, with lock held.
+                               del missing[:]
+                               for i, cpv in enumerate(cpv_all):
+                                       d = metadata.get(cpv)
+                                       if not d or "MD5" not in d:
+                                               missing.append(cpv)
+
+                               maxval = len(missing)
+                               for i, cpv in enumerate(missing):
+                                       try:
+                                               metadata[cpv] = bintree._pkgindex_entry(cpv)
+                                       except portage.exception.InvalidDependString:
+                                               writemsg("!!! Invalid binary package: '%s'\n" % \
+                                                       bintree.getname(cpv), noiselevel=-1)
+
+                                       if onProgress:
+                                               onProgress(maxval, i+1)
+
                                for cpv in set(metadata).difference(
                                        self._bintree.dbapi.cpv_all()):
                                        del metadata[cpv]
+
+                               # We've updated the pkgindex, so set it to
+                               # repopulate when necessary.
+                               bintree.populated = False
+
                                del pkgindex.packages[:]
                                pkgindex.packages.extend(metadata.itervalues())
                                from portage.util import atomic_ofstream
@@ -196,6 +221,11 @@ class BinhostHandler(object):
                                        f.close()
                        finally:
                                locks.unlockfile(pkgindex_lock)
+
+               if onProgress:
+                       if maxval == 0:
+                               maxval = 1
+                       onProgress(maxval, maxval)
                return None
 
 class MoveHandler(object):

diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py
index 96e9e553b9a12e0089dd832ea543d01163a7e1a0..112301476fcc55c31fcddce0d48d3d0357ed6ab2 100644 (file)
--- a/pym/portage/cache/mappings.py
+++ b/pym/portage/cache/mappings.py
@@ -262,6 +262,9 @@ def slot_dict_class(keys, prefix="_val_"):
                                        except AttributeError:
                                                pass
 
+                       def __str__(self):
+                               return str(dict(self.iteritems()))
+
                v = SlotDict
                _slot_dict_classes[v.allowed_keys] = v
        return v

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 86b8aba81d693086b198e41285ce2c334ad0b6fd..59835c07345136aa9d4aaa5eac128324172ebe96 100644 (file)
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -764,9 +764,6 @@ class binarytree(object):
                                noiselevel=-1)
                        return
                slot = slot.strip()
-               from portage.checksum import perform_multiple_checksums
-               digests = perform_multiple_checksums(
-                       full_path, hashes=self._pkgindex_hashes)
                self.dbapi.cpv_inject(cpv)
                self.dbapi._aux_cache.pop(cpv, None)
 
@@ -797,26 +794,16 @@ class binarytree(object):
                                        del f
                        if not self._pkgindex_version_supported(pkgindex):
                                pkgindex = self._new_pkgindex()
-                       d = digests
-                       d["CPV"] = cpv
-                       d["SLOT"] = slot
-                       d["MTIME"] = str(long(s.st_mtime))
-                       d["SIZE"] = str(s.st_size)
-                       rel_path = self._pkg_paths[cpv]
-                       # record location if it's non-default
-                       if rel_path != cpv + ".tbz2":
-                               d["PATH"] = rel_path
-                       from itertools import izip
-                       d.update(izip(self._pkgindex_aux_keys,
-                               self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
+
                        try:
-                               self._eval_use_flags(cpv, d)
+                               d = self._pkgindex_entry(cpv)
                        except portage.exception.InvalidDependString:
                                writemsg("!!! Invalid binary package: '%s'\n" % \
                                        self.getname(cpv), noiselevel=-1)
                                self.dbapi.cpv_remove(cpv)
                                del self._pkg_paths[cpv]
                                return
+
                        # If found, remove package(s) with duplicate path.
                        for i in xrange(len(pkgindex.packages) - 1, -1, -1):
                                d2 = pkgindex.packages[i]
@@ -825,6 +812,7 @@ class binarytree(object):
                                if d2.get("PATH", "") == d.get("PATH", ""):
                                        del pkgindex.packages[i]
                        pkgindex.packages.append(d)
+
                        self._update_pkgindex_header(pkgindex.header)
                        from portage.util import atomic_ofstream
                        f = atomic_ofstream(os.path.join(self.pkgdir, "Packages"))
@@ -836,6 +824,36 @@ class binarytree(object):
                        if pkgindex_lock:
                                unlockfile(pkgindex_lock)
 
+       def _pkgindex_entry(self, cpv):
+               """
+               Performs checksums and evaluates USE flag conditionals.
+               Raises InvalidDependString if necessary.
+               @rtype: dict
+               @returns: a dict containing entry for the give cpv.
+               """
+
+               pkg_path = self.getname(cpv)
+               from portage.checksum import perform_multiple_checksums
+
+               d = dict(izip(self._pkgindex_aux_keys,
+                       self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
+
+               d.update(perform_multiple_checksums(
+                       pkg_path, hashes=self._pkgindex_hashes))
+
+               d["CPV"] = cpv
+               st = os.stat(pkg_path)
+               d["MTIME"] = str(long(st.st_mtime))
+               d["SIZE"] = str(st.st_size)
+
+               rel_path = self._pkg_paths[cpv]
+               # record location if it's non-default
+               if rel_path != cpv + ".tbz2":
+                       d["PATH"] = rel_path
+
+               self._eval_use_flags(cpv, d)
+               return d
+
        def _new_pkgindex(self):
                return portage.getbinpkg.PackageIndex(
                        allowed_pkg_keys=self._pkgindex_allowed_pkg_keys,