From ebf51d6cc85ffa5f6e493f104afab6039f8e41bb Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 27 Jun 2008 14:47:19 +0000 Subject: [PATCH] Merged from trunk 10788:10801 | 10789 | Swap out the whole Package.root_config attribute inside | | zmedico | depgraph.break_refs(). | | 10790 | Add a generic portage.cache.mappings.slot_dict_class() | | zmedico | function which generates mapping classes that behave similar | | | to a dict but store values as object attributes that are | | | allocated via __slots__. Instances of these objects have a | | | smaller memory footprint than a normal dict object. These | | | classes are used to reduce the memory footprint of the | | | dbapi.aux_get() caches and the Package.metadata attribute. | | 10791 | Fix indentation. | | zmedico | | | 10792 | Fix KeyError constructor style. | | zmedico | | | 10793 | Fix typo. | | zmedico | | | 10794 | Add docstring to slot_dict_class(). | | zmedico | | | 10795 | Fix typo. | | zmedico | | | 10796 | Use SlotDict where appropriate in binarytree._populate(). | | zmedico | | | 10797 | Make PackageIndex use SlotDict for package metadata storage. | | zmedico | The set of allowed keys is passed into the PackageIndex | | | constructor (normal dict instances will be used if the set | | | of keys is not passed in for some reason). A | | | SlotDict.allowed_keys attribute now provides access to a | | | frozenset of allowed keys. | | 10798 | Implement lazy initialization of global "portdb", "settings" | | zmedico | and other variables that pollute the portage module. This | | | works by initializing the global variables with dummy | | | "proxy" objects that serve as a means to trigger lazy | | | initialization. As soon as the first attribute access or | | | method call occurs on one of the proxy objects, it causes | | | all the proxy objects to be replaced with the real ones. | | | It's possible for an unsupported attribute access or method | | | call on a proxy object to trigger an error, leading to | | | breakage. However, hopefully these such corner cases will | | | negligible (only time will tell). | | 10799 | Use a separate proxy type to trigger portage.portdb | | zmedico | initialization separately from the rest of the legacy global | | | variables. This can be useful since sometimes the other | | | variables are needed while the portdb is not. | | 10800 | Also use a separate proxy type for portage.mtimedb since | | zmedico | it's independent from the portdb and settings. | | 10801 | change default name of EverythingSet to @all-installed (bug | | genone | #229467) | svn path=/main/branches/prefix/; revision=10814 --- cnf/sets.conf | 2 +- pym/_emerge/__init__.py | 120 ++++++----------------------- pym/portage/__init__.py | 73 ++++++++++++++++-- pym/portage/cache/mappings.py | 138 ++++++++++++++++++++++++++++++++++ pym/portage/dbapi/bintree.py | 21 +++++- pym/portage/dbapi/porttree.py | 11 ++- pym/portage/getbinpkg.py | 33 ++++++-- pym/portage/util.py | 61 +++++++++++++++ 8 files changed, 342 insertions(+), 117 deletions(-) diff --git a/cnf/sets.conf b/cnf/sets.conf index 7d852cae5..997e13d69 100644 --- a/cnf/sets.conf +++ b/cnf/sets.conf @@ -25,7 +25,7 @@ class = portage.sets.security.NewAffectedSet world-candidate = False # Again, not much to change here, though people might prefer a different name -[everything] +[all-installed] class = portage.sets.dbapi.EverythingSet world-candidate = False diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 93fa11246..a3d7bdb69 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1398,108 +1398,33 @@ class Package(Task): return True return False -class _PackageMetadataWrapper(object): +_all_metadata_keys = set(x for x in portage.auxdbkeys \ + if not x.startswith("UNUSED_")) +_all_metadata_keys.discard("CDEPEND") +_all_metadata_keys.update(Package.metadata_keys) + +from portage.cache.mappings import slot_dict_class +_PackageMetadataWrapperBase = slot_dict_class(_all_metadata_keys) + +class _PackageMetadataWrapper(_PackageMetadataWrapperBase): """ Detect metadata updates and synchronize Package attributes. """ - _keys = set(x for x in portage.auxdbkeys \ - if not x.startswith("UNUSED_")) - _keys.discard("CDEPEND") - _keys.update(Package.metadata_keys) - _keys = tuple(sorted(_keys)) - __slots__ = ("__weakref__", "_pkg") + tuple("_val_" + k for k in _keys) + + __slots__ = ("_pkg",) _wrapped_keys = frozenset( ["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"]) def __init__(self, pkg, metadata): + _PackageMetadataWrapperBase.__init__(self) self._pkg = pkg self.update(metadata) - def __iter__(self): - for k, v in self.iteritems(): - yield k - - def __len__(self): - l = 0 - for i in self.iteritems(): - l += 1 - return l - - def keys(self): - return list(self) - - def iteritems(self): - for k in self._keys: - try: - yield (k, getattr(self, "_val_" + k)) - except AttributeError: - pass - - def items(self): - return list(self.iteritems()) - - def itervalues(self): - for k, v in self.itervalues(): - yield v - - def values(self): - return list(self.itervalues()) - - def __delitem__(self, k): - try: - delattr(self, "_val_" + k) - except AttributeError: - raise KeyError(k) - def __setitem__(self, k, v): - setattr(self, "_val_" + k, v) + _PackageMetadataWrapperBase.__setitem__(self, k, v) if k in self._wrapped_keys: getattr(self, "_set_" + k.lower())(k, v) - def update(self, d): - i = getattr(d, "iteritems", None) - if i is None: - i = d - else: - i = i() - for k, v in i: - self[k] = v - - def __getitem__(self, k): - try: - return getattr(self, "_val_" + k) - except AttributeError: - raise KeyError(k) - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def __contains__(self, k): - return hasattr(self, "_val_" + k) - - def pop(self, key, *args): - if len(args) > 1: - raise TypeError("pop expected at most 2 arguments, got " + \ - repr(1 + len(args))) - try: - value = self[key] - except KeyError: - if args: - return args[0] - raise - del self[key] - return value - - def clear(self): - for k in self._keys: - try: - delattr(self, "_val_" + k) - except AttributError: - pass - def _set_inherited(self, k, v): if isinstance(v, basestring): v = frozenset(v.split()) @@ -3977,17 +3902,14 @@ class depgraph(object): if not isinstance(node, Package): continue - # The visible packages cache has fullfilled it's purpose - # and it's no longer needed, so free the memory. - node.root_config.visible_pkgs.clear() - - if isinstance(node.root_config.trees["vartree"], FakeVartree): - # The FakeVartree references the _package_cache which - # references the depgraph. So that Package instances don't - # hold the depgraph and FakeVartree on the heap, replace - # the FakeVartree reference with the real vartree. - node.root_config.trees["vartree"] = \ - self._trees_orig[node.root]["vartree"] + # The FakeVartree references the _package_cache which + # references the depgraph. So that Package instances don't + # hold the depgraph and FakeVartree on the heap, replace + # the RootConfig that references the FakeVartree with the + # original RootConfig instance which references the actual + # vartree. + node.root_config = \ + self._trees_orig[node.root]["root_config"] def _resolve_conflicts(self): if not self._complete_graph(): diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 879e65b67..e6ce9b591 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -6993,6 +6993,54 @@ def create_trees(config_root=None, target_root=None, trees=None): binarytree, myroot, mysettings["PKGDIR"], settings=mysettings) return trees +class _LegacyGlobalProxy(portage.util.ObjectProxy): + """ + Instances of these serve as proxies to global variables + that are initialized on demand. + """ + def __init__(self, name): + portage.util.ObjectProxy.__init__(self) + object.__setattr__(self, '_name', name) + + def _get_target(self): + init_legacy_globals() + name = object.__getattribute__(self, '_name') + return globals()[name] + +class _PortdbProxy(portage.util.ObjectProxy): + """ + The portdb is initialized separately from the rest + of the variables, since sometimes the other variables + are needed while the portdb is not. + """ + + def _get_target(self): + init_legacy_globals() + global db, portdb, root, _portdb_initialized + if not _portdb_initialized: + portdb = db[root]["porttree"].dbapi + _portdb_initialized = True + return portdb + +class _MtimedbProxy(portage.util.ObjectProxy): + """ + The mtimedb is independent from the portdb and other globals. + """ + + def __init__(self, name): + portage.util.ObjectProxy.__init__(self) + object.__setattr__(self, '_name', name) + + def _get_target(self): + global mtimedb, mtimedbfile, _mtimedb_initialized + if not _mtimedb_initialized: + mtimedbfile = os.path.join("/", + CACHE_PATH.lstrip(os.path.sep), "mtimedb") + mtimedb = MtimeDB(mtimedbfile) + _mtimedb_initialized = True + name = object.__getattribute__(self, '_name') + return globals()[name] + # Initialization of legacy globals. No functions/classes below this point # please! When the above functions and classes become independent of the # below global variables, it will be possible to make the below code @@ -7002,6 +7050,11 @@ def create_trees(config_root=None, target_root=None, trees=None): # overhead (and other issues!) of initializing the legacy globals. def init_legacy_globals(): + global _globals_initialized + if _globals_initialized: + return + _globals_initialized = True + global db, settings, root, portdb, selinux_enabled, mtimedbfile, mtimedb, \ archlist, features, groups, pkglines, thirdpartymirrors, usedefaults, \ profiledir, flushmtimedb @@ -7019,18 +7072,14 @@ def init_legacy_globals(): del _initializing_globals settings = db["/"]["vartree"].settings - portdb = db["/"]["porttree"].dbapi for myroot in db: if myroot != "/": settings = db[myroot]["vartree"].settings - portdb = db[myroot]["porttree"].dbapi break root = settings["ROOT"] - mtimedbfile = os.path.join("/", CACHE_PATH.lstrip(os.path.sep), "mtimedb") - mtimedb = MtimeDB(mtimedbfile) # ======================================================================== # COMPATIBILITY @@ -7060,7 +7109,21 @@ def init_legacy_globals(): # use within Portage. External use of this variable is unsupported because # it is experimental and it's behavior is likely to change. if "PORTAGE_LEGACY_GLOBALS" not in os.environ: - init_legacy_globals() + + _mtimedb_initialized = False + mtimedb = _MtimedbProxy("mtimedb") + mtimedbfile = _MtimedbProxy("mtimedbfile") + + _portdb_initialized = False + portdb = _PortdbProxy() + + _globals_initialized = False + + for k in ("db", "settings", "root", "selinux_enabled", + "archlist", "features", "groups", + "pkglines", "thirdpartymirrors", "usedefaults", "profiledir", + "flushmtimedb"): + globals()[k] = _LegacyGlobalProxy(k) # Clear the cache dircache={} diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py index 9aa5a21e2..2cddd8147 100644 --- a/pym/portage/cache/mappings.py +++ b/pym/portage/cache/mappings.py @@ -4,6 +4,7 @@ # $Id$ import UserDict +import weakref class ProtectedDict(UserDict.DictMixin): """ @@ -101,3 +102,140 @@ class LazyLoad(UserDict.DictMixin): self.pull = None return key in self.d +_slot_dict_classes = weakref.WeakValueDictionary() + +def slot_dict_class(keys): + """ + Generates mapping classes that behave similar to a dict but store values + as object attributes that are allocated via __slots__. Instances of these + objects have a smaller memory footprint than a normal dict object. + + @param keys: Fixed set of allowed keys + @type keys: iterable + @rtype: SlotDict + @returns: A class that constructs SlotDict instances + having the specified keys. + """ + if isinstance(keys, frozenset): + keys_set = keys + else: + keys_set = frozenset(keys) + v = _slot_dict_classes.get(keys_set) + if v is None: + + class SlotDict(object): + + allowed_keys = keys_set + __slots__ = ("__weakref__",) + \ + tuple("_val_" + k for k in allowed_keys) + + def __iter__(self): + for k, v in self.iteritems(): + yield k + + def __len__(self): + l = 0 + for i in self.iteritems(): + l += 1 + return l + + def keys(self): + return list(self) + + def iteritems(self): + for k in self.allowed_keys: + try: + yield (k, getattr(self, "_val_" + k)) + except AttributeError: + pass + + def items(self): + return list(self.iteritems()) + + def itervalues(self): + for k, v in self.iteritems(): + yield v + + def values(self): + return list(self.itervalues()) + + def __delitem__(self, k): + try: + delattr(self, "_val_" + k) + except AttributeError: + raise KeyError(k) + + def __setitem__(self, k, v): + setattr(self, "_val_" + k, v) + + def setdefault(self, key, default=None): + try: + return self[key] + except KeyError: + self[key] = default + return default + + def update(self, d): + i = getattr(d, "iteritems", None) + if i is None: + i = d + else: + i = i() + for k, v in i: + self[k] = v + + def __getitem__(self, k): + try: + return getattr(self, "_val_" + k) + except AttributeError: + raise KeyError(k) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __contains__(self, k): + return hasattr(self, "_val_" + k) + + def has_key(self, k): + return k in self + + def pop(self, key, *args): + if len(args) > 1: + raise TypeError( + "pop expected at most 2 arguments, got " + \ + repr(1 + len(args))) + try: + value = self[key] + except KeyError: + if args: + return args[0] + raise + del self[key] + return value + + def popitem(self): + try: + k, v = self.iteritems().next() + except StopIteration: + raise KeyError('container is empty') + del self[k] + return (k, v) + + def copy(self): + c = self.__class__() + c.update(self) + return c + + def clear(self): + for k in self.allowed_keys: + try: + delattr(self, "_val_" + k) + except AttributeError: + pass + + v = SlotDict + _slot_dict_classes[v.allowed_keys] = v + return v diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index f795d9866..2fe21b4f2 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +from portage.cache.mappings import slot_dict_class from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list from portage.dbapi.virtual import fakedbapi from portage.exception import InvalidPackageName, InvalidAtom, \ @@ -18,7 +19,7 @@ import portage.xpak, portage.getbinpkg import os, errno, stat import re -from itertools import izip +from itertools import chain, izip class bindbapi(fakedbapi): _known_keys = frozenset(list(fakedbapi._known_keys) + \ @@ -35,6 +36,7 @@ class bindbapi(fakedbapi): "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "EPREFIX"]) + self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) self._aux_cache = {} def match(self, *pargs, **kwargs): @@ -77,7 +79,7 @@ class bindbapi(fakedbapi): if not mydata.setdefault("EAPI", "0"): mydata["EAPI"] = "0" if cache_me: - aux_cache = {} + aux_cache = self._aux_cache_slot_dict() for x in self._aux_cache_keys: aux_cache[x] = mydata.get(x, "") self._aux_cache[mycpv] = aux_cache @@ -187,6 +189,16 @@ class binarytree(object): ("repository" , "REPO"), ) + self._pkgindex_allowed_pkg_keys = set(chain( + self._pkgindex_keys, + self._pkgindex_aux_keys, + self._pkgindex_hashes, + self._pkgindex_default_pkg_data, + self._pkgindex_inherited_keys, + self._pkgindex_default_header_data, + chain(*self._pkgindex_translated_keys) + )) + def move_ent(self, mylist): if not self.populated: self.populate() @@ -483,7 +495,7 @@ class binarytree(object): update_pkgindex = True self.dbapi.cpv_inject(mycpv) if not self.dbapi._aux_cache_keys.difference(d): - aux_cache = {} + aux_cache = self.dbapi._aux_cache_slot_dict() for k in self.dbapi._aux_cache_keys: aux_cache[k] = d[k] self.dbapi._aux_cache[mycpv] = aux_cache @@ -580,7 +592,7 @@ class binarytree(object): d.pop("PATH", None) metadata[mycpv] = d if not self.dbapi._aux_cache_keys.difference(d): - aux_cache = {} + aux_cache = self.dbapi._aux_cache_slot_dict() for k in self.dbapi._aux_cache_keys: aux_cache[k] = d[k] self.dbapi._aux_cache[mycpv] = aux_cache @@ -835,6 +847,7 @@ class binarytree(object): def _new_pkgindex(self): return portage.getbinpkg.PackageIndex( + allowed_pkg_keys=self._pkgindex_allowed_pkg_keys, default_header_data=self._pkgindex_default_header_data, default_pkg_data=self._pkgindex_default_pkg_data, inherited_keys=self._pkgindex_inherited_keys, diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 3c27c2d95..b6e39f63b 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -3,6 +3,7 @@ # $Id$ from portage.cache.cache_errors import CacheError +from portage.cache.mappings import slot_dict_class from portage.const import REPO_NAME_LOC from portage.data import portage_gid, secpass from portage.dbapi import dbapi @@ -138,6 +139,10 @@ class portdbapi(dbapi): ["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "repository", "RESTRICT", "SLOT"]) + + # Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict + # initialization until the first aux_get call. + self._aux_cache_slot_dict = None self._aux_cache = {} self._broken_ebuilds = set() @@ -386,7 +391,10 @@ class portdbapi(dbapi): returnme.append(mydata.get(x,"")) if cache_me: - aux_cache = {} + if self._aux_cache_slot_dict is None: + self._aux_cache_slot_dict = \ + slot_dict_class(self._aux_cache_keys) + aux_cache = self._aux_cache_slot_dict() for x in self._aux_cache_keys: aux_cache[x] = mydata.get(x, "") self._aux_cache[mycpv] = aux_cache @@ -866,4 +874,3 @@ class portagetree(object): except Exception, e: pass return myslot - diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 574da21f8..34ad243be 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -4,6 +4,7 @@ # $Id$ from portage.output import red, yellow, green +from portage.cache.mappings import slot_dict_class import portage.xpak import HTMLParser import sys @@ -681,8 +682,17 @@ def _cmp_cpv(d1, d2): class PackageIndex(object): - def __init__(self, default_header_data=None, default_pkg_data=None, - inherited_keys=None, translated_keys=None): + def __init__(self, + allowed_pkg_keys=None, + default_header_data=None, + default_pkg_data=None, + inherited_keys=None, + translated_keys=None): + + self._pkg_slot_dict = None + if allowed_pkg_keys is not None: + self._pkg_slot_dict = slot_dict_class(allowed_pkg_keys) + self._default_header_data = default_header_data self._default_pkg_data = default_pkg_data self._inherited_keys = inherited_keys @@ -697,8 +707,15 @@ class PackageIndex(object): self.packages = [] self.modified = True - def _readpkgindex(self, pkgfile): - d = {} + def _readpkgindex(self, pkgfile, pkg_entry=True): + + allowed_keys = None + if self._pkg_slot_dict is None or not pkg_entry: + d = {} + else: + d = self._pkg_slot_dict() + allowed_keys = d.allowed_keys + for line in pkgfile: line = line.rstrip("\n") if not line: @@ -709,7 +726,11 @@ class PackageIndex(object): k, v = line if v: v = v[1:] - d[self._read_translation_map.get(k, k)] = v + k = self._read_translation_map.get(k, k) + if allowed_keys is not None and \ + k not in allowed_keys: + continue + d[k] = v return d def _writepkgindex(self, pkgfile, items): @@ -723,7 +744,7 @@ class PackageIndex(object): self.readBody(pkgfile) def readHeader(self, pkgfile): - self.header.update(self._readpkgindex(pkgfile)) + self.header.update(self._readpkgindex(pkgfile, pkg_entry=False)) def readBody(self, pkgfile): while True: diff --git a/pym/portage/util.py b/pym/portage/util.py index f3951b52c..6012b0f26 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -913,6 +913,67 @@ def ensure_dirs(dir_path, *args, **kwargs): perms_modified = apply_permissions(dir_path, *args, **kwargs) return created_dir or perms_modified +class ObjectProxy(object): + + """ + Object that acts as a proxy to another object, forwarding + attribute accesses and method calls. This can be useful + for implementing lazy initialization. + """ + + def _get_target(self): + raise NotImplementedError(self) + + def __getattribute__(self, attr): + result = object.__getattribute__(self, '_get_target')() + return getattr(result, attr) + + def __setattr__(self, attr, value): + result = object.__getattribute__(self, '_get_target')() + setattr(result, attr, value) + + def __call__(self, *args, **kwargs): + result = object.__getattribute__(self, '_get_target')() + return result(*args, **kwargs) + + def __setitem__(self, key, value): + object.__getattribute__(self, '_get_target')()[key] = value + + def __getitem__(self, key): + return object.__getattribute__(self, '_get_target')()[key] + + def __delitem__(self, key): + del object.__getattribute__(self, '_get_target')()[key] + + def __contains__(self, key): + return key in object.__getattribute__(self, '_get_target')() + + def __iter__(self): + return iter(object.__getattribute__(self, '_get_target')()) + + def __len__(self): + return len(object.__getattribute__(self, '_get_target')()) + + def __repr__(self): + return repr(object.__getattribute__(self, '_get_target')()) + + def __str__(self): + return str(object.__getattribute__(self, '_get_target')()) + + def __hash__(self): + return hash(object.__getattribute__(self, '_get_target')()) + + def __eq__(self, other): + return object.__getattribute__(self, '_get_target')() == other + + def __ne__(self, other): + return object.__getattribute__(self, '_get_target')() != other + + def __nonzero__(self): + if object.__getattribute__(self, '_get_target')(): + return True + return False + class LazyItemsDict(dict): """A mapping object that behaves like a standard dict except that it allows for lazy initialization of values via callable objects. Lazy items can be -- 2.26.2