From: Fabian Groffen Date: Mon, 23 Feb 2009 19:29:16 +0000 (-0000) Subject: Merged from trunk -r12671:12686 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=6570b4215d402acd68c33d7455d9f9708ef44744;p=portage.git Merged from trunk -r12671:12686 | 12672 | In LazyItemsDict, when a singleton is instantiated, replace | | zmedico | the wrapper with the singleton since the wrapper is no | | | longer needed at this point. | | 12673 | In config.setcpv(), use LazyItemsDict to implement lazy | | zmedico | evaluation of PORTAGE_IUSE. The PORTAGE_IUSE value is lazily | | | evaluated since re.escape() is slow and the value is only | | | used when an ebuild phase needs to be executed (it's used | | | only to generate QA notices). Thanks to Marat Radchenko | | | for identifying this | | | performance issue and submitting the initial patch which | | | used a memoization approach instead of lazy evaluation. | | 12674 | Make the UserDict and LazyItemsDict constructors use an | | zmedico | optional positional argument instead of a keyword argument. | | 12675 | Make lazy_import() skip proxy creation for modules that | | zmedico | already exist in sys.modules. | | 12676 | Reduce bloat in portage.util by splitting ObjectProxy and | | zmedico | lazyimport into a new portage.proxy package. | | 12677 | Fix typo in __all__. | | zmedico | | | 12678 | Make lazyimport proxies trigger automatic destruction of | | zmedico | other proxies for modules that have been imported. This way, | | | destruction of a single proxy can trigger destruction of all | | | the rest. | | 12679 | Fix _LazyImportFrom to use the correct module name when | | zmedico | registering and unregistering. | | 12680 | Use lazyimport to avoid importing the checksum, locks, and | | zmedico | util modules when portage is initially imported. | | 12681 | Use lazyimport to avoid importing the dep, output, update, | | zmedico | and versions modules when portage is initially imported. | | 12682 | Use lazyimport to avoid importing the portage.elog module | | zmedico | when portage is initially imported. | | 12683 | Use a lock to ensure that threaded code doesn't cause | | zmedico | problems with proxy registration and unregistration. | | 12684 | The python that's inside stage 1 or 2 is built with a | | zmedico | minimal configuration which does not include the | | | /usr/lib/pythonX.Y/encodings directory. This results in | | | error like the following: LookupError: no codec search | | | functions registered: can't find encoding In order to solve | | | this problem, detect it early and manually register a search | | | function for the ascii codec. Starting with python-3.0 this | | | problem is more noticeable because of stricter handling of | | | encoding and decoding between strings of characters and | | | bytes. | | 12685 | Avoid spurious CCACHE_DIR permissions adjustments when | | zmedico | fetching with a temporary PORTAGE_TMPDIR setting (for | | | fetchonly). Thanks to Marat Radchenko | | | for reporting. | | 12686 | In EbuildFetchonly.execute(), skip builddir creation when | | zmedico | the ebuild does not set RESTRICT=fetch (since it's not | | | possible for pkg_nofetch to be executed in this case). | svn path=/main/branches/prefix/; revision=12698 --- diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index d34fbe0e1..8fc5ef24d 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1710,6 +1710,32 @@ class EbuildFetchonly(SlotObject): __slots__ = ("fetch_all", "pkg", "pretend", "settings") def execute(self): + settings = self.settings + pkg = self.pkg + portdb = pkg.root_config.trees["porttree"].dbapi + ebuild_path = portdb.findname(pkg.cpv) + settings.setcpv(pkg) + debug = settings.get("PORTAGE_DEBUG") == "1" + use_cache = 1 # always true + portage.doebuild_environment(ebuild_path, "fetch", + settings["ROOT"], settings, debug, use_cache, portdb) + restrict_fetch = 'fetch' in settings['PORTAGE_RESTRICT'].split() + + if restrict_fetch: + rval = self._execute_with_builddir() + else: + rval = portage.doebuild(ebuild_path, "fetch", + settings["ROOT"], settings, debug=debug, + listonly=self.pretend, fetchonly=1, fetchall=self.fetch_all, + mydbapi=portdb, tree="porttree") + + if rval != os.EX_OK: + msg = "Fetch failed for '%s'" % (pkg.cpv,) + eerror(msg, phase="unpack", key=pkg.cpv) + + return rval + + def _execute_with_builddir(self): # To spawn pkg_nofetch requires PORTAGE_BUILDDIR for # ensuring sane $PWD (bug #239560) and storing elog # messages. Use a private temp directory, in order @@ -1739,11 +1765,7 @@ class EbuildFetchonly(SlotObject): root_config = pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(pkg.cpv) - settings.setcpv(pkg) debug = settings.get("PORTAGE_DEBUG") == "1" - use_cache = 1 # always true - portage.doebuild_environment(ebuild_path, "fetch", - root_config.root, settings, debug, use_cache, portdb) portage.prepare_build_dirs(self.pkg.root, self.settings, 0) retval = portage.doebuild(ebuild_path, "fetch", diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index addb16b63..802525815 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1,5 +1,5 @@ # portage.py -- core Portage functionality -# Copyright 1998-2004 Gentoo Foundation +# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ @@ -73,8 +73,9 @@ if platform.system() in ["FreeBSD"]: try: from portage.cache.cache_errors import CacheError - import portage.util as util - util.lazy_import(globals(), + import portage.proxy.lazyimport + import portage.proxy as proxy + proxy.lazyimport.lazyimport(globals(), 'portage.checksum', 'portage.checksum:perform_checksum,perform_md5,prelink_capable', 'portage.cvstree', @@ -93,6 +94,15 @@ try: 'portage.update:dep_transform,fixdbentries,grab_updates,' + \ 'parse_updates,update_config_files,update_dbentries,' + \ 'update_dbentry', + 'portage.util', + 'portage.util:atomic_ofstream,apply_secpass_permissions,' + \ + 'apply_recursive_permissions,dump_traceback,getconfig,' + \ + 'grabdict,grabdict_package,grabfile,grabfile_package,' + \ + 'map_dictlist_vals,new_protect_filename,normalize_path,' + \ + 'pickle_read,pickle_write,stack_dictlist,stack_dicts,' + \ + 'stack_lists,unique_array,varexpand,writedict,writemsg,' + \ + 'writemsg_stdout,write_atomic', + 'portage.versions', 'portage.versions:best,catpkgsplit,catsplit,endversion_keys,' + \ 'suffix_value@endversion,pkgcmp,pkgsplit,vercmp,ververify', 'portage.xpak', @@ -112,12 +122,6 @@ try: from portage.data import ostype, lchown, userland, secpass, uid, wheelgid, \ portage_uid, portage_gid, userpriv_groups from portage.manifest import Manifest - - from portage.util import atomic_ofstream, apply_secpass_permissions, apply_recursive_permissions, \ - dump_traceback, getconfig, grabdict, grabdict_package, grabfile, grabfile_package, \ - map_dictlist_vals, new_protect_filename, normalize_path, \ - pickle_read, pickle_write, stack_dictlist, stack_dicts, stack_lists, \ - unique_array, varexpand, writedict, writemsg, writemsg_stdout, write_atomic import portage.exception from portage.localization import _ @@ -607,7 +611,9 @@ class digraph(object): #parse /etc/env.d and generate /etc/profile.env def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, - env=None, writemsg_level=portage.util.writemsg_level): + env=None, writemsg_level=None): + if writemsg_level is None: + writemsg_level = portage.util.writemsg_level if target_root is None: global settings target_root = settings["ROOT"] @@ -959,6 +965,19 @@ def check_config_instance(test): if not isinstance(test, config): raise TypeError("Invalid type for config object: %s (should be %s)" % (test.__class__, config)) +def _lazy_iuse_regex(iuse_implicit): + """ + The PORTAGE_IUSE value is lazily evaluated since re.escape() is slow + and the value is only used when an ebuild phase needs to be executed + (it's used only to generate QA notices). + """ + # Escape anything except ".*" which is supposed to pass through from + # _get_implicit_iuse(). + regex = sorted(re.escape(x) for x in iuse_implicit) + regex = "^(%s)$" % "|".join(regex) + regex = regex.replace("\\.\\*", ".*") + return regex + class config(object): """ This class encompasses the main portage configuration. Data is pulled from @@ -1491,7 +1510,7 @@ class config(object): self.configlist.append(self.mygcfg) self.configdict["conf"]=self.configlist[-1] - self.configlist.append({}) + self.configlist.append(util.LazyItemsDict()) self.configdict["pkg"]=self.configlist[-1] #auto-use: @@ -2092,12 +2111,9 @@ class config(object): iuse_implicit = self._get_implicit_iuse() iuse_implicit.update(x.lstrip("+-") for x in iuse.split()) - # Escape anything except ".*" which is supposed - # to pass through from _get_implicit_iuse() - regex = sorted(re.escape(x) for x in iuse_implicit) - regex = "^(%s)$" % "|".join(regex) - regex = regex.replace("\\.\\*", ".*") - self.configdict["pkg"]["PORTAGE_IUSE"] = regex + # PORTAGE_IUSE is not always needed so it's lazily evaluated. + self.configdict["pkg"].addLazySingleton( + "PORTAGE_IUSE", _lazy_iuse_regex, iuse_implicit) ebuild_force_test = self.get("EBUILD_FORCE_TEST") == "1" if ebuild_force_test and \ @@ -5153,7 +5169,10 @@ def prepare_build_dirs(myroot, mysettings, cleanup): return 1 _prepare_workdir(mysettings) - _prepare_features_dirs(mysettings) + if mysettings.get('EBUILD_PHASE') != 'fetch': + # Avoid spurious permissions adjustments when fetching with + # a temporary PORTAGE_TMPDIR setting (for fetchonly). + _prepare_features_dirs(mysettings) def _adjust_perms_msg(settings, msg): @@ -7529,6 +7548,62 @@ def portageexit(): atexit_register(portageexit) +def _ensure_default_encoding(): + """ + The python that's inside stage 1 or 2 is built with a minimal + configuration which does not include the /usr/lib/pythonX.Y/encodings + directory. This results in error like the following: + + LookupError: no codec search functions registered: can't find encoding + + In order to solve this problem, detect it early and manually register + a search function for the ascii codec. Starting with python-3.0 this + problem is more noticeable because of stricter handling of encoding + and decoding between strings of characters and bytes. + """ + + import codecs + try: + codecs.lookup(sys.getdefaultencoding()) + except LookupError: + pass + else: + return + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codecs.ascii_encode(input, self.errors)[0] + + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return codecs.ascii_decode(input, self.errors)[0] + + class StreamWriter(codecs.StreamWriter): + encode = codecs.ascii_encode + + class StreamReader(codecs.StreamReader): + decode = codecs.ascii_decode + + # The sys.setdefaultencoding() function doesn't necessarily exist, + # so just setup the ascii codec to correspond to whatever name + # happens to be returned by sys.getdefaultencoding(). + encoding = sys.getdefaultencoding() + + def search_function(name): + if name != encoding: + return None + return codecs.CodecInfo( + name=encoding, + encode=codecs.ascii_encode, + decode=codecs.ascii_decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamwriter=StreamWriter, + streamreader=StreamReader, + ) + + codecs.register(search_function) + def _global_updates(trees, prev_mtimes): """ Perform new global updates if they exist in $PORTDIR/profiles/updates/. @@ -7764,7 +7839,7 @@ def create_trees(config_root=None, target_root=None, trees=None): myroots.append((settings["ROOT"], settings)) for myroot, mysettings in myroots: - trees[myroot] = portage.util.LazyItemsDict(trees.get(myroot, None)) + trees[myroot] = portage.util.LazyItemsDict(trees.get(myroot, {})) trees[myroot].addLazySingleton("virtuals", mysettings.getvirtuals, myroot) trees[myroot].addLazySingleton( "vartree", vartree, myroot, categories=mysettings.categories, @@ -7775,13 +7850,13 @@ def create_trees(config_root=None, target_root=None, trees=None): binarytree, myroot, mysettings["PKGDIR"], settings=mysettings) return trees -class _LegacyGlobalProxy(portage.util.ObjectProxy): +class _LegacyGlobalProxy(proxy.objectproxy.ObjectProxy): """ Instances of these serve as proxies to global variables that are initialized on demand. """ def __init__(self, name): - portage.util.ObjectProxy.__init__(self) + proxy.objectproxy.ObjectProxy.__init__(self) object.__setattr__(self, '_name', name) def _get_target(self): @@ -7789,7 +7864,7 @@ class _LegacyGlobalProxy(portage.util.ObjectProxy): name = object.__getattribute__(self, '_name') return globals()[name] -class _PortdbProxy(portage.util.ObjectProxy): +class _PortdbProxy(proxy.objectproxy.ObjectProxy): """ The portdb is initialized separately from the rest of the variables, since sometimes the other variables @@ -7804,13 +7879,13 @@ class _PortdbProxy(portage.util.ObjectProxy): _portdb_initialized = True return portdb -class _MtimedbProxy(portage.util.ObjectProxy): +class _MtimedbProxy(proxy.objectproxy.ObjectProxy): """ The mtimedb is independent from the portdb and other globals. """ def __init__(self, name): - portage.util.ObjectProxy.__init__(self) + proxy.objectproxy.ObjectProxy.__init__(self) object.__setattr__(self, '_name', name) def _get_target(self): @@ -7919,6 +7994,8 @@ if True: "flushmtimedb"): globals()[k] = _LegacyGlobalProxy(k) + _ensure_default_encoding() + # Clear the cache dircache={} diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py index a632ce61d..1117b855b 100644 --- a/pym/portage/cache/mappings.py +++ b/pym/portage/cache/mappings.py @@ -139,10 +139,16 @@ class UserDict(MutableMapping): http://bugs.python.org/issue2876 """ - def __init__(self, dict=None, **kwargs): - self.data = {} - if dict is not None: - self.update(dict) + def __init__(self, *args, **kwargs): + + if len(args) > 1: + raise TypeError( + "expected at most 1 positional argument, got " + \ + repr(len(args))) + + if args: + self.update(args[0]) + if kwargs: self.update(kwargs) diff --git a/pym/portage/data.py b/pym/portage/data.py index c78255d8c..0f8eff474 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -1,14 +1,16 @@ # data.py -- Calculated/Discovered Data Values -# Copyright 1998-2004 Gentoo Foundation +# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ import os, sys, pwd, grp, platform -from portage.util import writemsg from portage.const import rootuid, portageuser, portagegroup, EPREFIX -from portage.output import colorize -from portage.output import create_color_func -bad = create_color_func("BAD") + +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.output:colorize', + 'portage.util:writemsg', +) ostype=platform.system() userland = None @@ -38,7 +40,7 @@ if not lchown: lchown() def portage_group_warning(): - warn_prefix = bad("*** WARNING *** ") + warn_prefix = colorize("BAD", "*** WARNING *** ") mylines = [ "For security reasons, only system administrators should be", "allowed in the portage group. Untrusted users or processes", diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index dae7ad014..6bea1e5d3 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -1,4 +1,4 @@ -# Copyright 1998-2007 Gentoo Foundation +# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ @@ -6,13 +6,17 @@ __all__ = ["dbapi"] import os import re -from portage.dep import match_from_list -from portage.locks import unlockfile -from portage.output import colorize -from portage.util import writemsg, cmp_sort_key -from portage import auxdbkeys, dep_expand -from portage.versions import catpkgsplit, pkgcmp +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.dep:match_from_list', + 'portage.locks:unlockfile', + 'portage.output:colorize', + 'portage.util:cmp_sort_key,writemsg', + 'portage.versions:catpkgsplit,pkgcmp', +) + +from portage import auxdbkeys, dep_expand class dbapi(object): _category_re = re.compile(r'^\w[-.+\w]*$') diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 1cca34ac7..c7a05cecc 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -1,24 +1,26 @@ -# Copyright 1998-2007 Gentoo Foundation +# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ __all__ = ["bindbapi", "binarytree"] +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list', + 'portage.output:EOutput,colorize', + 'portage.update:update_dbentries', + 'portage.util:ensure_dirs,normalize_path,writemsg,writemsg_stdout', + 'portage.versions:best,catpkgsplit,catsplit', +) + from portage.cache.mappings import slot_dict_class -from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list from portage.dbapi.virtual import fakedbapi from portage.exception import InvalidPackageName, \ PermissionDenied, PortageException -from portage.output import EOutput, colorize -from portage.util import ensure_dirs, normalize_path, writemsg, writemsg_stdout -from portage.versions import best, catpkgsplit, catsplit -from portage.update import update_dbentries from portage.const import EAPI, EAPIPREFIX from portage import dep_expand, listdir, _check_distfile, _movefile -import portage - import os, errno, stat import re from itertools import chain, izip diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index d342c22d9..ab5e9ba13 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -1,22 +1,25 @@ -# Copyright 1998-2007 Gentoo Foundation +# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ __all__ = ["portdbapi", "close_portdbapi_caches", "portagetree"] +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.checksum', + 'portage.dep:dep_getkey,match_from_list,paren_reduce,use_reduce', + 'portage.util:ensure_dirs,writemsg', + 'portage.versions:best,catpkgsplit,pkgsplit,ver_regexp', +) + from portage.cache.cache_errors import CacheError from portage.cache.mappings import slot_dict_class from portage.const import REPO_NAME_LOC from portage.data import portage_gid, secpass from portage.dbapi import dbapi -from portage.dep import use_reduce, paren_reduce, dep_getkey, match_from_list from portage.exception import PortageException, \ FileNotFound, InvalidDependString, InvalidPackageName from portage.manifest import Manifest -from portage.util import ensure_dirs, writemsg -from portage.versions import pkgsplit, catpkgsplit, best, ver_regexp - -import portage.checksum from portage import eclass_cache, auxdbkeys, doebuild, flatten, \ listdir, dep_expand, eapi_is_supported, key_expand, dep_check, \ diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index d9916e693..af7b1af47 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2007 Gentoo Foundation +# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ @@ -6,30 +6,34 @@ __all__ = ["PreservedLibsRegistry", "LinkageMap", "LinkageMapMachO", "vardbapi", "vartree", "dblink"] + \ ["write_contents", "tar_contents"] -from portage.checksum import perform_md5 +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.checksum:perform_md5', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \ + 'use_reduce,paren_reduce', + 'portage.elog:elog_process', + 'portage.elog.filtering:filter_mergephases,filter_unmergephases', + 'portage.locks:lockdir,unlockdir', + 'portage.output:bold,colorize', + 'portage.update:fixdbentries', + 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ + 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ + 'grabfile,grabdict,normalize_path,new_protect_filename,getlibpaths', + 'portage.versions:best,catpkgsplit,catsplit,pkgcmp,pkgsplit', +) + from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH, EPREFIX, EPREFIX_LSTRIP from portage.data import portage_gid, portage_uid, secpass, ostype from portage.dbapi import dbapi -from portage.dep import use_reduce, paren_reduce, isvalidatom, \ - isjustname, dep_getkey, match_from_list from portage.exception import CommandNotFound, \ InvalidData, InvalidPackageName, \ FileNotFound, PermissionDenied, UnsupportedAPIException -from portage.locks import lockdir, unlockdir -from portage.output import bold, colorize -from portage.update import fixdbentries -from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, \ - writemsg, writemsg_level, write_atomic, atomic_ofstream, writedict, \ - grabfile, grabdict, normalize_path, new_protect_filename, getlibpaths -from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp from portage import listdir, dep_expand, digraph, flatten, key_expand, \ doebuild_environment, doebuild, env_update, prepare_build_dirs, \ abssymlink, movefile, _movefile, bsd_chflags, cpv_getkey -from portage.elog import elog_process -from portage.elog.filtering import filter_mergephases, filter_unmergephases from portage.cache.mappings import slot_dict_class import os, re, shutil, stat, errno, copy, subprocess diff --git a/pym/portage/elog/__init__.py b/pym/portage/elog/__init__.py index ad5cc873c..5e8b11c8b 100644 --- a/pym/portage/elog/__init__.py +++ b/pym/portage/elog/__init__.py @@ -1,13 +1,16 @@ # elog/__init__.py - elog core functions -# Copyright 2006-2007 Gentoo Foundation +# Copyright 2006-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:writemsg', +) + from portage.const import EBUILD_PHASES from portage.exception import PortageException from portage.process import atexit_register -from portage.util import writemsg - from portage.elog.messages import collect_ebuild_messages, collect_messages from portage.elog.filtering import filter_loglevels diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py index a09b73229..4fa73ba5f 100644 --- a/pym/portage/elog/messages.py +++ b/pym/portage/elog/messages.py @@ -1,11 +1,15 @@ # elog/messages.py - elog core functions -# Copyright 2006-2007 Gentoo Foundation +# Copyright 2006-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ -from portage.output import colorize +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.output:colorize', + 'portage.util:writemsg', +) + from portage.const import EBUILD_PHASES -from portage.util import writemsg import os import sys diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 9329c889b..1244e4232 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -1,13 +1,16 @@ -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Header: $ import errno, os -import portage.versions, portage.const -from portage.checksum import * +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.util:write_atomic', +) + from portage.exception import * -from portage.util import write_atomic class FileNotInManifestException(PortageException): pass diff --git a/pym/portage/output.py b/pym/portage/output.py index db22084df..1e59f172e 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -1,4 +1,4 @@ -# Copyright 1998-2004 Gentoo Foundation +# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ @@ -11,8 +11,13 @@ import os import re import shlex import sys + +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:writemsg', +) + from portage.const import COLOR_MAP_FILE, EPREFIX -from portage.util import writemsg from portage.exception import CommandNotFound, FileNotFound, \ ParseError, PermissionDenied, PortageException diff --git a/pym/portage/process.py b/pym/portage/process.py index 3bc6bd1a8..7c0fb342f 100644 --- a/pym/portage/process.py +++ b/pym/portage/process.py @@ -1,5 +1,5 @@ # portage.py -- core Portage functionality -# Copyright 1998-2004 Gentoo Foundation +# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ @@ -9,7 +9,11 @@ import atexit import signal import sys -from portage.util import dump_traceback +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:dump_traceback', +) + from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY from portage.exception import CommandNotFound diff --git a/pym/portage/proxy/__init__.py b/pym/portage/proxy/__init__.py new file mode 100644 index 000000000..95cbda1ed --- /dev/null +++ b/pym/portage/proxy/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2009 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ diff --git a/pym/portage/proxy/lazyimport.py b/pym/portage/proxy/lazyimport.py new file mode 100644 index 000000000..1a4a246ef --- /dev/null +++ b/pym/portage/proxy/lazyimport.py @@ -0,0 +1,175 @@ +# Copyright 2009 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +__all__ = ['lazyimport'] + +import sys +import types + +try: + import threading +except ImportError: + import dummy_threading as threading + +from portage.proxy.objectproxy import ObjectProxy + +_module_proxies = {} +_module_proxies_lock = threading.RLock() + +def _register_module_proxy(name, proxy): + _module_proxies_lock.acquire() + try: + proxy_list = _module_proxies.get(name) + if proxy_list is None: + proxy_list = [] + _module_proxies[name] = proxy_list + proxy_list.append(proxy) + finally: + _module_proxies_lock.release() + +def _unregister_module_proxy(name): + """ + Destroy all proxies that reference the give module name. Also, check + for other proxies referenced by modules that have been imported and + destroy those proxies too. This way, destruction of a single proxy + can trigger destruction of all the rest. + """ + _module_proxies_lock.acquire() + try: + proxy_list = _module_proxies.get(name) + if proxy_list is not None: + # First delete this name from the dict so that + # if this same thread reenters below, it won't + # enter this path again. + del _module_proxies[name] + for proxy in proxy_list: + object.__getattribute__(proxy, '_get_target')() + + modules = sys.modules + for name, proxy_list in list(_module_proxies.iteritems()): + if name not in modules: + continue + # First delete this name from the dict so that + # if this same thread reenters below, it won't + # enter this path again. + del _module_proxies[name] + for proxy in proxy_list: + object.__getattribute__(proxy, '_get_target')() + finally: + _module_proxies_lock.release() + +class _LazyImport(ObjectProxy): + + __slots__ = ('_scope', '_alias', '_name', '_target') + + def __init__(self, scope, alias, name): + ObjectProxy.__init__(self) + object.__setattr__(self, '_scope', scope) + object.__setattr__(self, '_alias', alias) + object.__setattr__(self, '_name', name) + _register_module_proxy(name, self) + + def _get_target(self): + try: + return object.__getattribute__(self, '_target') + except AttributeError: + pass + name = object.__getattribute__(self, '_name') + __import__(name) + target = sys.modules[name] + object.__setattr__(self, '_target', target) + object.__getattribute__(self, '_scope')[ + object.__getattribute__(self, '_alias')] = target + _unregister_module_proxy(name) + return target + +class _LazyImportFrom(_LazyImport): + + __slots__ = ('_attr_name',) + + def __init__(self, scope, name, attr_name, alias): + object.__setattr__(self, '_attr_name', attr_name) + _LazyImport.__init__(self, scope, alias, name) + + def _get_target(self): + try: + return object.__getattribute__(self, '_target') + except AttributeError: + pass + name = object.__getattribute__(self, '_name') + attr_name = object.__getattribute__(self, '_attr_name') + __import__(name) + target = getattr(sys.modules[name], attr_name) + object.__setattr__(self, '_target', target) + object.__getattribute__(self, '_scope')[ + object.__getattribute__(self, '_alias')] = target + _unregister_module_proxy(name) + return target + +def lazyimport(scope, *args): + """ + Create a proxy in the given scope in order to performa a lazy import. + + Syntax Result + foo import foo + foo:bar,baz from foo import bar, baz + foo:bar@baz from foo import bar as baz + + @param scope: the scope in which to place the import, typically globals() + @type myfilename: dict + @param args: module names to import + @type args: strings + """ + + modules = sys.modules + + for s in args: + parts = s.split(':', 1) + if len(parts) == 1: + name = s + + if not name or not isinstance(name, basestring): + raise ValueError(name) + + components = name.split('.') + parent_scope = scope + for i in xrange(len(components)): + alias = components[i] + if i < len(components) - 1: + parent_name = ".".join(components[:i+1]) + __import__(parent_name) + mod = modules.get(parent_name) + if not isinstance(mod, types.ModuleType): + # raise an exception + __import__(name) + parent_scope[alias] = mod + parent_scope = mod.__dict__ + continue + + already_imported = modules.get(name) + if already_imported is not None: + parent_scope[alias] = already_imported + else: + parent_scope[alias] = \ + _LazyImport(parent_scope, alias, name) + + else: + name, fromlist = parts + already_imported = modules.get(name) + fromlist = fromlist.split(',') + for s in fromlist: + alias = s.split('@', 1) + if len(alias) == 1: + alias = alias[0] + attr_name = alias + else: + attr_name, alias = alias + if already_imported is not None: + try: + scope[alias] = getattr(already_imported, attr_name) + except AttributeError: + raise ImportError('cannot import name %s' % attr_name) + else: + scope[alias] = \ + _LazyImportFrom(scope, name, attr_name, alias) diff --git a/pym/portage/proxy/objectproxy.py b/pym/portage/proxy/objectproxy.py new file mode 100644 index 000000000..00d3f6157 --- /dev/null +++ b/pym/portage/proxy/objectproxy.py @@ -0,0 +1,66 @@ +# Copyright 2008-2009 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +__all__ = ['ObjectProxy'] + +class ObjectProxy(object): + + """ + Object that acts as a proxy to another object, forwarding + attribute accesses and method calls. This can be useful + for implementing lazy initialization. + """ + + __slots__ = () + + def _get_target(self): + raise NotImplementedError(self) + + def __getattribute__(self, attr): + result = object.__getattribute__(self, '_get_target')() + return getattr(result, attr) + + def __setattr__(self, attr, value): + result = object.__getattribute__(self, '_get_target')() + setattr(result, attr, value) + + def __call__(self, *args, **kwargs): + result = object.__getattribute__(self, '_get_target')() + return result(*args, **kwargs) + + def __setitem__(self, key, value): + object.__getattribute__(self, '_get_target')()[key] = value + + def __getitem__(self, key): + return object.__getattribute__(self, '_get_target')()[key] + + def __delitem__(self, key): + del object.__getattribute__(self, '_get_target')()[key] + + def __contains__(self, key): + return key in object.__getattribute__(self, '_get_target')() + + def __iter__(self): + return iter(object.__getattribute__(self, '_get_target')()) + + def __len__(self): + return len(object.__getattribute__(self, '_get_target')()) + + def __repr__(self): + return repr(object.__getattribute__(self, '_get_target')()) + + def __str__(self): + return str(object.__getattribute__(self, '_get_target')()) + + def __hash__(self): + return hash(object.__getattribute__(self, '_get_target')()) + + def __eq__(self, other): + return object.__getattribute__(self, '_get_target')() == other + + def __ne__(self, other): + return object.__getattribute__(self, '_get_target')() != other + + def __nonzero__(self): + return bool(object.__getattribute__(self, '_get_target')()) diff --git a/pym/portage/update.py b/pym/portage/update.py index 85d07f5a3..fffdc26eb 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -1,15 +1,18 @@ -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Header: $ import errno, os, re, sys -from portage.util import ConfigProtect, grabfile, new_protect_filename, \ - normalize_path, write_atomic, writemsg +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.dep:dep_getkey,get_operator,isvalidatom,isjustname,remove_slot', + 'portage.util:ConfigProtect,grabfile,new_protect_filename,' + \ + 'normalize_path,write_atomic,writemsg', + 'portage.versions:ververify' +) + from portage.exception import DirectoryNotFound, PortageException -from portage.versions import ververify -from portage.dep import dep_getkey, get_operator, isvalidatom, isjustname, \ - remove_slot from portage.const import USER_CONFIG_PATH, WORLD_FILE ignored_dbentries = ("CONTENTS", "environment.bz2") diff --git a/pym/portage/util.py b/pym/portage/util.py index 8039818df..42139c656 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -1,7 +1,16 @@ -# Copyright 2004 Gentoo Foundation +# Copyright 2004-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ +__all__ = ['apply_permissions', 'apply_recursive_permissions', + 'apply_secpass_permissions', 'apply_stat_permissions', 'atomic_ofstream', + 'cmp_sort_key', 'ConfigProtect', 'dump_traceback', 'ensure_dirs', + 'getconfig', 'getlibpaths', 'grabdict', 'grabdict_package', 'grabfile', + 'grabfile_package', 'grablines', 'initialize_logger', 'LazyItemsDict', + 'map_dictlist_vals', 'new_protect_filename', 'normalize_path', + 'pickle_read', 'stack_dictlist', 'stack_dicts', 'stack_lists', + 'unique_array', 'varexpand', 'write_atomic', 'writedict', 'writemsg', + 'writemsg_level', 'writemsg_stdout'] import os import errno @@ -16,6 +25,7 @@ from portage.exception import PortageException, FileNotFound, \ OperationNotPermitted, PermissionDenied, ReadOnlyFileSystem import portage.exception from portage.dep import isvalidatom +from portage.proxy.objectproxy import ObjectProxy from portage.const import EPREFIX, EPREFIX_LSTRIP try: @@ -335,165 +345,6 @@ def writedict(mydict,myfilename,writekey=True): return 0 return 1 -class ObjectProxy(object): - - """ - Object that acts as a proxy to another object, forwarding - attribute accesses and method calls. This can be useful - for implementing lazy initialization. - """ - - __slots__ = () - - def _get_target(self): - raise NotImplementedError(self) - - def __getattribute__(self, attr): - result = object.__getattribute__(self, '_get_target')() - return getattr(result, attr) - - def __setattr__(self, attr, value): - result = object.__getattribute__(self, '_get_target')() - setattr(result, attr, value) - - def __call__(self, *args, **kwargs): - result = object.__getattribute__(self, '_get_target')() - return result(*args, **kwargs) - - def __setitem__(self, key, value): - object.__getattribute__(self, '_get_target')()[key] = value - - def __getitem__(self, key): - return object.__getattribute__(self, '_get_target')()[key] - - def __delitem__(self, key): - del object.__getattribute__(self, '_get_target')()[key] - - def __contains__(self, key): - return key in object.__getattribute__(self, '_get_target')() - - def __iter__(self): - return iter(object.__getattribute__(self, '_get_target')()) - - def __len__(self): - return len(object.__getattribute__(self, '_get_target')()) - - def __repr__(self): - return repr(object.__getattribute__(self, '_get_target')()) - - def __str__(self): - return str(object.__getattribute__(self, '_get_target')()) - - def __hash__(self): - return hash(object.__getattribute__(self, '_get_target')()) - - def __eq__(self, other): - return object.__getattribute__(self, '_get_target')() == other - - def __ne__(self, other): - return object.__getattribute__(self, '_get_target')() != other - - def __nonzero__(self): - return bool(object.__getattribute__(self, '_get_target')()) - -class _LazyImport(ObjectProxy): - - __slots__ = ('_scope', '_alias', '_name', '_target') - - def __init__(self, scope, alias, name): - ObjectProxy.__init__(self) - object.__setattr__(self, '_scope', scope) - object.__setattr__(self, '_alias', alias) - object.__setattr__(self, '_name', name) - - def _get_target(self): - try: - return object.__getattribute__(self, '_target') - except AttributeError: - pass - name = object.__getattribute__(self, '_name') - __import__(name) - target = sys.modules[name] - object.__setattr__(self, '_target', target) - object.__getattribute__(self, '_scope')[ - object.__getattribute__(self, '_alias')] = target - return target - -class _LazyImportFrom(_LazyImport): - - __slots__ = () - - def _get_target(self): - try: - return object.__getattribute__(self, '_target') - except AttributeError: - pass - name = object.__getattribute__(self, '_name') - components = name.split('.') - parent_name = '.'.join(components[:-1]) - __import__(parent_name) - target = getattr(sys.modules[parent_name], components[-1]) - object.__setattr__(self, '_target', target) - object.__getattribute__(self, '_scope')[ - object.__getattribute__(self, '_alias')] = target - return target - -def lazy_import(scope, *args): - """ - Create a proxy in the given scope in order to performa a lazy import. - - Syntax Result - foo import foo - foo:bar,baz from foo import bar, baz - foo:bar@baz from foo import bar as baz - - @param scope: the scope in which to place the import, typically globals() - @type myfilename: dict - @param args: module names to import - @type args: strings - """ - - for s in args: - parts = s.split(':', 1) - if len(parts) == 1: - name = s - - if not name or not isinstance(name, basestring): - raise ValueError(name) - - components = name.split('.') - parent_scope = scope - for i in xrange(len(components)): - alias = components[i] - mod = parent_scope.get(alias) - if isinstance(mod, types.ModuleType): - parent_scope = mod.__dict__ - continue - if i < len(components) - 1: - parent_name = ".".join(components[:i+1]) - __import__(parent_name) - mod = sys.modules.get(parent_name) - if not isinstance(mod, types.ModuleType): - # raise an exception - __import__(name) - parent_scope[alias] = mod - parent_scope = mod.__dict__ - continue - parent_scope[alias] = _LazyImport(parent_scope, alias, name) - - else: - name, fromlist = parts - fromlist = fromlist.split(',') - for s in fromlist: - alias = s.split('@', 1) - if len(alias) == 1: - alias = alias[0] - orig = alias - else: - orig, alias = alias - scope[alias] = _LazyImportFrom(scope, alias, - name + '.' + orig) - class _tolerant_shlex(shlex.shlex): def sourcehook(self, newfile): try: @@ -1200,11 +1051,25 @@ class LazyItemsDict(dict): """A mapping object that behaves like a standard dict except that it allows for lazy initialization of values via callable objects. Lazy items can be overwritten and deleted just as normal items.""" - def __init__(self, initial_items=None): + + __slots__ = ('lazy_items',) + + def __init__(self, *args, **kwargs): + + if len(args) > 1: + raise TypeError( + "expected at most 1 positional argument, got " + \ + repr(len(args))) + dict.__init__(self) self.lazy_items = {} - if initial_items is not None: - self.update(initial_items) + + if args: + self.update(args[0]) + + if kwargs: + self.update(kwargs) + def addLazyItem(self, item_key, value_callable, *pargs, **kwargs): """Add a lazy item for the given key. When the item is requested, value_callable will be called with *pargs and **kwargs arguments.""" @@ -1214,18 +1079,9 @@ class LazyItemsDict(dict): def addLazySingleton(self, item_key, value_callable, *pargs, **kwargs): """This is like addLazyItem except value_callable will only be called a maximum of 1 time and the result will be cached for future requests.""" - class SingletonItem(object): - def __init__(self, value_callable, *pargs, **kwargs): - self._callable = value_callable - self._pargs = pargs - self._kwargs = kwargs - self._called = False - def __call__(self): - if not self._called: - self._called = True - self._value = self._callable(*self._pargs, **self._kwargs) - return self._value - self.addLazyItem(item_key, SingletonItem(value_callable, *pargs, **kwargs)) + self.addLazyItem(item_key, + self._SingletonWrapper(self, item_key, value_callable, + *pargs, **kwargs)) def update(self, map_obj): if isinstance(map_obj, LazyItemsDict): for k in map_obj: @@ -1251,6 +1107,63 @@ class LazyItemsDict(dict): del self.lazy_items[item_key] dict.__delitem__(self, item_key) + def clear(self): + self.lazy_items.clear() + dict.clear(self) + + def copy(self): + return self.__copy__() + + def __copy__(self): + return self.__class__(self) + + def __deepcopy__(self, memo=None): + """ + WARNING: If any of the lazy items contains a bound method then it's + typical for deepcopy() to raise an exception like this: + + File "/usr/lib/python2.5/copy.py", line 189, in deepcopy + y = _reconstruct(x, rv, 1, memo) + File "/usr/lib/python2.5/copy.py", line 322, in _reconstruct + y = callable(*args) + File "/usr/lib/python2.5/copy_reg.py", line 92, in __newobj__ + return cls.__new__(cls, *args) + TypeError: instancemethod expected at least 2 arguments, got 0 + + If deepcopy() needs to work, this problem can be avoided by + implementing lazy items with normal (non-bound) functions. + """ + if memo is None: + memo = {} + from copy import deepcopy + result = self.__class__() + memo[id(self)] = result + for k in self: + k_copy = deepcopy(k, memo) + if k in self.lazy_items: + dict.__setitem__(result, k_copy, None) + result.lazy_items[k_copy] = \ + deepcopy(self.lazy_items[k], memo) + else: + dict.__setitem__(result, k_copy, deepcopy(self[k], memo)) + return result + + class _SingletonWrapper(object): + + __slots__ = ('_parent', '_key', '_callable', '_pargs', '_kwargs') + + def __init__(self, parent, key, value_callable, *pargs, **kwargs): + self._parent = parent + self._key = key + self._callable = value_callable + self._pargs = pargs + self._kwargs = kwargs + + def __call__(self): + value = self._callable(*self._pargs, **self._kwargs) + self._parent[self._key] = value + return value + class ConfigProtect(object): def __init__(self, myroot, protect_list, mask_list): self.myroot = myroot