Misc performance enhancements. Thanks to Marat Radchenko

<marat@slonopotamus.org> for this patch from bug #276813.

svn path=/main/trunk/; revision=14398

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index 918c935f596d5a388fe68fa30dcebc55a094ba14..983055a4b8672775051d3061b49f56b3d9814ac9 100644 (file)
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -9,6 +9,7 @@ from portage.cache import cache_errors
 import errno
 import stat
 import sys
+import os as _os
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
@@ -32,33 +33,33 @@ class database(fs_template.FsBased):
                        self._ensure_dirs()
 
        def _getitem(self, cpv):
-               fp = os.path.join(self.location, cpv)
+               # Don't use os.path.join, for better performance.
+               fp = self.location + _os.sep + cpv
                try:
                        myf = codecs.open(_unicode_encode(fp,
                                encoding=_encodings['fs'], errors='strict'),
                                mode='r', encoding=_encodings['repo.content'],
                                errors='replace')
                        try:
-                               d = self._parse_data(myf.readlines(), cpv)
+                               d = self._parse_data(myf.read().split("\n"), cpv)
                                if '_mtime_' not in d:
                                        # Backward compatibility with old cache
                                        # that uses mtime mangling.
-                                       d['_mtime_'] = long(os.fstat(myf.fileno()).st_mtime)
+                                       d['_mtime_'] = long(_os.fstat(myf.fileno()).st_mtime)
                                return d
                        finally:
                                myf.close()
                except (IOError, OSError) as e:
                        if e.errno != errno.ENOENT:
                                raise cache_errors.CacheCorruption(cpv, e)
-                       raise KeyError(cpv)
+                       raise KeyError(cpv, e)
 
        def _parse_data(self, data, cpv):
                try:
-                       d = dict(map(lambda x:x.rstrip("\n").split("=", 1), data))
+                       return dict( x.split("=", 1) for x in data )
                except ValueError as e:
                        # If a line is missing an "=", the split length is 1 instead of 2.
                        raise cache_errors.CacheCorruption(cpv, e)
-               return d
 
        def _setitem(self, cpv, values):
 #              import pdb;pdb.set_trace()
@@ -101,7 +102,6 @@ class database(fs_template.FsBased):
                        os.remove(fp)
                        raise cache_errors.CacheCorruption(cpv, e)
 
-
        def _delitem(self, cpv):
 #              import pdb;pdb.set_trace()
                try:
@@ -112,11 +112,9 @@ class database(fs_template.FsBased):
                        else:
                                raise cache_errors.CacheCorruption(cpv, e)
 
-
        def __contains__(self, cpv):
                return os.path.exists(os.path.join(self.location, cpv))
 
-
        def __iter__(self):
                """generator for walking the dir struct"""
                dirs = [self.location]
@@ -140,4 +138,3 @@ class database(fs_template.FsBased):
                                        continue
                                yield p[len_base+1:]
                        dirs.pop(0)
-

diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py
index 81b1f17fd42de61e8fd687e4c8b2ac55b12613c0..2704bd5135a3fd14cfae5a4a2c0c853a51868400 100644 (file)
--- a/pym/portage/cache/metadata.py
+++ b/pym/portage/cache/metadata.py
@@ -52,7 +52,7 @@ class database(flat_hash.database):
                                d.clear()
                                try:
                                        for i, key in enumerate(self.auxdbkey_order):
-                                               d[key] = data[i].rstrip("\n")
+                                               d[key] = data[i]
                                except IndexError:
                                        pass
                                break

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index b073e4268c18a1a43cfae370bd2b76930fc29bb0..f68d92fd2e34e8d0cd680fa694735a0ea8349742 100644 (file)
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -379,9 +379,6 @@ class portdbapi(dbapi):
                        "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository",
                        "RESTRICT", "SLOT"])
 
-               # Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict
-               # initialization until the first aux_get call.
-               self._aux_cache_slot_dict = None
                self._aux_cache = {}
                self._broken_ebuilds = set()
 
@@ -546,9 +543,10 @@ class portdbapi(dbapi):
                return metadata
 
        def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
-
                try:
-                       st = os.stat(ebuild_path)
+                       # Don't use unicode-wrapped os module, for better performance.
+                       st = _os.stat(_unicode_encode(ebuild_path,
+                               encoding=_encodings['fs'], errors='strict'))
                        emtime = st[stat.ST_MTIME]
                except OSError:
                        writemsg(_("!!! aux_get(): ebuild for " \
@@ -668,8 +666,7 @@ class portdbapi(dbapi):
                                mydata["_eclasses_"] = {}
 
                # do we have a origin repository name for the current package
-               mydata["repository"] = self._repository_map.get(
-                       os.path.sep.join(myebuild.split(os.path.sep)[:-3]), "")
+               mydata["repository"] = self._repository_map.get(mylocation, "")
 
                mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", []))
                mydata["_mtime_"] = long(st.st_mtime)
@@ -687,10 +684,7 @@ class portdbapi(dbapi):
                returnme = [mydata.get(x, "") for x in mylist]
 
                if cache_me:
-                       if self._aux_cache_slot_dict is None:
-                               self._aux_cache_slot_dict = \
-                                       slot_dict_class(self._aux_cache_keys)
-                       aux_cache = self._aux_cache_slot_dict()
+                       aux_cache = {}
                        for x in self._aux_cache_keys:
                                aux_cache[x] = mydata.get(x, "")
                        self._aux_cache[mycpv] = aux_cache