Add a generic portage.cache.mappings.slot_dict_class() function which
authorZac Medico <zmedico@gentoo.org>
Wed, 25 Jun 2008 22:36:19 +0000 (22:36 -0000)
committerZac Medico <zmedico@gentoo.org>
Wed, 25 Jun 2008 22:36:19 +0000 (22:36 -0000)
generates mapping classes that behave similar to a dict but store
values as object attributes that are allocated via __slots__. Instances
of these objects have a smaller memory footprint than a normal dict object.
These classes are used to reduce the memory footprint of the dbapi.aux_get()
caches and the Package.metadata attribute.

svn path=/main/trunk/; revision=10790

pym/_emerge/__init__.py
pym/portage/cache/mappings.py
pym/portage/dbapi/bintree.py
pym/portage/dbapi/porttree.py

index 2f54b6b650cfaca27bd9f66ddde729901ccc85ed..ca76400a335310797882264d207830252b330416 100644 (file)
@@ -1371,108 +1371,33 @@ class Package(Task):
                        return True
                return False
 
-class _PackageMetadataWrapper(object):
+_all_metadata_keys = set(x for x in portage.auxdbkeys \
+       if not x.startswith("UNUSED_"))
+_all_metadata_keys.discard("CDEPEND")
+_all_metadata_keys.update(Package.metadata_keys)
+
+from portage.cache.mappings import slot_dict_class
+_PackageMetadataWrapperBase = slot_dict_class(_all_metadata_keys)
+
+class _PackageMetadataWrapper(_PackageMetadataWrapperBase):
        """
        Detect metadata updates and synchronize Package attributes.
        """
-       _keys = set(x for x in portage.auxdbkeys \
-               if not x.startswith("UNUSED_"))
-       _keys.discard("CDEPEND")
-       _keys.update(Package.metadata_keys)
-       _keys = tuple(sorted(_keys))
-       __slots__ = ("__weakref__", "_pkg") + tuple("_val_" + k for k in _keys)
+
+       __slots__ = ("_pkg",)
        _wrapped_keys = frozenset(
                ["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"])
 
        def __init__(self, pkg, metadata):
+               _PackageMetadataWrapperBase.__init__(self)
                self._pkg = pkg
                self.update(metadata)
 
-       def __iter__(self):
-               for k, v in self.iteritems():
-                       yield k
-
-       def __len__(self):
-               l = 0
-               for i in self.iteritems():
-                       l += 1
-               return l
-
-       def keys(self):
-               return list(self)
-
-       def iteritems(self):
-               for k in self._keys:
-                       try:
-                               yield (k, getattr(self, "_val_" + k))
-                       except AttributeError:
-                               pass
-
-       def items(self):
-               return list(self.iteritems())
-
-       def itervalues(self):
-               for k, v in self.itervalues():
-                       yield v
-
-       def values(self):
-               return list(self.itervalues())
-
-       def __delitem__(self, k):
-               try:
-                       delattr(self, "_val_" + k)
-               except AttributeError:
-                       raise KeyError(k)
-
        def __setitem__(self, k, v):
-               setattr(self, "_val_" + k, v)
+               _PackageMetadataWrapperBase.__setitem__(self, k, v)
                if k in self._wrapped_keys:
                        getattr(self, "_set_" + k.lower())(k, v)
 
-       def update(self, d):
-               i = getattr(d, "iteritems", None)
-               if i is None:
-                       i = d
-               else:
-                       i = i()
-               for k, v in i:
-                       self[k] = v
-
-       def __getitem__(self, k):
-               try:
-                       return getattr(self, "_val_" + k)
-               except AttributeError:
-                       raise KeyError(k)
-
-       def get(self, key, default=None):
-               try:
-                       return self[key]
-               except KeyError:
-                       return default
-
-       def __contains__(self, k):
-               return hasattr(self, "_val_" + k)
-
-       def pop(self, key, *args):
-               if len(args) > 1:
-                       raise TypeError("pop expected at most 2 arguments, got " + \
-                               repr(1 + len(args)))
-               try:
-                       value = self[key]
-               except KeyError:
-                       if args:
-                               return args[0]
-                       raise
-               del self[key]
-               return value
-
-       def clear(self):
-               for k in self._keys:
-                       try:
-                               delattr(self, "_val_" + k)
-                       except AttributError:
-                               pass
-
        def _set_inherited(self, k, v):
                if isinstance(v, basestring):
                        v = frozenset(v.split())
index 9aa5a21e2a8d18dea3af83d268a0fae51525c9a5..7c347a4f92579b1b6d8ca532c33cdf73bc89e5b3 100644 (file)
@@ -4,6 +4,7 @@
 # $Id$
 
 import UserDict
+import weakref
 
 class ProtectedDict(UserDict.DictMixin):
        """
@@ -101,3 +102,128 @@ class LazyLoad(UserDict.DictMixin):
                        self.pull = None
                return key in self.d
 
+_slot_dict_classes = weakref.WeakValueDictionary()
+
+def slot_dict_class(keys):
+       if isinstance(keys, frozenset):
+               keys_set = keys
+       else:
+               keys_set = frozenset(keys)
+       v = _slot_dict_classes.get(keys_set)
+       if v is None:
+
+               class SlotDict(object):
+
+                       _keys = keys_set
+                       __slots__ = ("__weakref__",) + tuple("_val_" + k for k in _keys)
+
+                       def __iter__(self):
+                               for k, v in self.iteritems():
+                                       yield k
+
+                       def __len__(self):
+                               l = 0
+                               for i in self.iteritems():
+                                       l += 1
+                               return l
+
+                       def keys(self):
+                               return list(self)
+
+                       def iteritems(self):
+                               for k in self._keys:
+                                       try:
+                                               yield (k, getattr(self, "_val_" + k))
+                                       except AttributeError:
+                                               pass
+
+                       def items(self):
+                               return list(self.iteritems())
+
+                       def itervalues(self):
+                               for k, v in self.itervalues():
+                                       yield v
+
+                       def values(self):
+                               return list(self.itervalues())
+
+                       def __delitem__(self, k):
+                               try:
+                                       delattr(self, "_val_" + k)
+                               except AttributeError:
+                                       raise KeyError(k)
+
+                       def __setitem__(self, k, v):
+                               setattr(self, "_val_" + k, v)
+
+                       def setdefault(self, key, default=None):
+                               try:
+                                       return self[key]
+                               except KeyError:
+                                       self[key] = default
+                               return default
+
+                       def update(self, d):
+                               i = getattr(d, "iteritems", None)
+                               if i is None:
+                                       i = d
+                               else:
+                                       i = i()
+                               for k, v in i:
+                                       self[k] = v
+
+                       def __getitem__(self, k):
+                               try:
+                                       return getattr(self, "_val_" + k)
+                               except AttributeError:
+                                       raise KeyError(k)
+
+                       def get(self, key, default=None):
+                               try:
+                                       return self[key]
+                               except KeyError:
+                                       return default
+
+                       def __contains__(self, k):
+                               return hasattr(self, "_val_" + k)
+
+                       def has_key(self, k):
+                               return k in self
+
+                       def pop(self, key, *args):
+                               if len(args) > 1:
+                                       raise TypeError(
+                                               "pop expected at most 2 arguments, got " + \
+                                               repr(1 + len(args)))
+                               try:
+                                       value = self[key]
+                               except KeyError:
+                                       if args:
+                                               return args[0]
+                                       raise
+                               del self[key]
+                               return value
+
+                       def popitem(self):
+                               try:
+                                       k, v = self.iteritems().next()
+                               except StopIteration:
+                                       raise KeyError, 'container is empty'
+                               del self[k]
+                               return (k, v)
+
+                       def copy(self):
+                               c = self.__class__()
+                               c.update(self)
+                               return c
+
+                       def clear(self):
+                               for k in self._keys:
+                                       try:
+                                               delattr(self, "_val_" + k)
+                                       except AttributError:
+                                               pass
+
+               v = SlotDict
+               _slot_dict_classes[keys_set] = v
+               return v
index fe8b372c2120bfe892b899b3ec8f3bfc6347421e..b3c36a23b44e19f37e49b352f4a0dbac8bec7ccd 100644 (file)
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+from portage.cache.mappings import slot_dict_class
 from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list
 from portage.dbapi.virtual import fakedbapi
 from portage.exception import InvalidPackageName, InvalidAtom, \
@@ -33,6 +34,7 @@ class bindbapi(fakedbapi):
                        ["CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
                        "LICENSE", "PDEPEND", "PROVIDE",
                        "RDEPEND", "repository", "RESTRICT", "SLOT", "USE"])
+               self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
                self._aux_cache = {}
 
        def match(self, *pargs, **kwargs):
@@ -75,7 +77,7 @@ class bindbapi(fakedbapi):
                        if not mydata.setdefault("EAPI", "0"):
                                mydata["EAPI"] = "0"
                if cache_me:
-                       aux_cache = {}
+                       aux_cache = self._aux_cache_slot_dict()
                        for x in self._aux_cache_keys:
                                aux_cache[x] = mydata.get(x, "")
                        self._aux_cache[mycpv] = aux_cache
index 3c27c2d95892e492b4c04fea28e67023ed8134c8..b6e39f63b44d4c4c74610eb9b4d35a04d9c3d748 100644 (file)
@@ -3,6 +3,7 @@
 # $Id$
 
 from portage.cache.cache_errors import CacheError
+from portage.cache.mappings import slot_dict_class
 from portage.const import REPO_NAME_LOC
 from portage.data import portage_gid, secpass
 from portage.dbapi import dbapi
@@ -138,6 +139,10 @@ class portdbapi(dbapi):
                        ["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE",
                        "PDEPEND", "PROVIDE", "RDEPEND", "repository",
                        "RESTRICT", "SLOT"])
+
+               # Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict
+               # initialization until the first aux_get call.
+               self._aux_cache_slot_dict = None
                self._aux_cache = {}
                self._broken_ebuilds = set()
 
@@ -386,7 +391,10 @@ class portdbapi(dbapi):
                                returnme.append(mydata.get(x,""))
 
                if cache_me:
-                       aux_cache = {}
+                       if self._aux_cache_slot_dict is None:
+                               self._aux_cache_slot_dict = \
+                                       slot_dict_class(self._aux_cache_keys)
+                       aux_cache = self._aux_cache_slot_dict()
                        for x in self._aux_cache_keys:
                                aux_cache[x] = mydata.get(x, "")
                        self._aux_cache[mycpv] = aux_cache
@@ -866,4 +874,3 @@ class portagetree(object):
                except Exception, e:
                        pass
                return myslot
-