From fcb5094705ee3dd12a50f4c969d81ab9f861bd3c Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 24 Sep 2009 17:24:01 +0000 Subject: [PATCH] Merged from trunk -r14368:14378 | 14369 | Don't wrap os.statvfs since it requires unicode string args | | zmedico | in py3k. | | 14370 | Fix 'RuntimeError: dictionary changed size during iteration' | | zmedico | in _owners_db._populate() with py3k. | | 14371 | Fix items to prevent infinite recursion. | | zmedico | | | 14372 | Remove redundant list() since starred expressions work with | | zmedico | iterators. | | 14373 | Fix 'RuntimeError: dictionary changed size during iteration' | | zmedico | in bindbapi.aux_update() with py3k. | | 14374 | Fix 'iteritems' attribute detection to look for 'items' | | zmedico | instead. | | 14375 | Fix another potential 'RuntimeError: dictionary changed size | | zmedico | during iteration' exception in _owners_db._populate() for | | | py3k. | | 14376 | Optimize unicode wrappers. Thanks to Marat Radchenko | | zmedico | for this patch from bug #276813. | | 14377 | For python 3.x with boolean target, len() results in | | zmedico | TypeError, so make __len__ return 0 or 1 in this case. | | 14378 | Do manual unicode handling (instead of using the wrapped os | | zmedico | module) in portdbapi.findname2(), since it's a hotspot. | | | Thanks to Marat Radchenko for the | | | initial patch. | svn path=/main/branches/prefix/; revision=14404 --- pym/portage/__init__.py | 59 +++++++++++++++++--------------- pym/portage/cache/template.py | 2 +- pym/portage/dbapi/bintree.py | 2 +- pym/portage/dbapi/porttree.py | 40 +++++++++++++++------- pym/portage/dbapi/vartree.py | 4 +-- pym/portage/dep.py | 2 +- pym/portage/proxy/objectproxy.py | 10 +++++- 7 files changed, 72 insertions(+), 47 deletions(-) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 836666428..71c496b29 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -142,22 +142,26 @@ _encodings = { if _encodings['merge'] is None: _encodings['merge'] = 'ascii' -def _unicode_encode(s, encoding=_encodings['content'], - errors='backslashreplace'): - if sys.hexversion >= 0x3000000: +if sys.hexversion >= 0x3000000: + def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, str): s = s.encode(encoding, errors) - elif isinstance(s, unicode): - s = s.encode(encoding, errors) - return s + return s -def _unicode_decode(s, encoding=_encodings['content'], errors='replace'): - if isinstance(s, bytes): - if sys.hexversion >= 0x3000000: + def _unicode_decode(s, encoding=_encodings['content'], errors='replace'): + if isinstance(s, bytes): s = str(s, encoding=encoding, errors=errors) - else: + return s +else: + def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): + if isinstance(s, unicode): + s = s.encode(encoding, errors) + return s + + def _unicode_decode(s, encoding=_encodings['content'], errors='replace'): + if isinstance(s, bytes): s = unicode(s, encoding=encoding, errors=errors) - return s + return s class _unicode_func_wrapper(object): """ @@ -190,24 +194,22 @@ class _unicode_func_wrapper(object): rval = self._func(*wrapped_args, **wrapped_kwargs) - if isinstance(rval, (bytes, basestring, list, tuple)): - if isinstance(rval, (bytes, basestring)): - rval = _unicode_decode(rval, - encoding=encoding, errors='replace') - else: - decoded_rval = [] - for x in rval: - try: - x = _unicode_decode(x, encoding=encoding, errors='strict') - except UnicodeDecodeError: - pass - else: - decoded_rval.append(x) - - if isinstance(rval, tuple): - rval = tuple(decoded_rval) + if isinstance(rval, (list, tuple)): + decoded_rval = [] + for x in rval: + try: + x = _unicode_decode(x, encoding=encoding, errors='strict') + except UnicodeDecodeError: + pass else: - rval = decoded_rval + decoded_rval.append(x) + + if isinstance(rval, tuple): + rval = tuple(decoded_rval) + else: + rval = decoded_rval + else: + rval = _unicode_decode(rval, encoding=encoding, errors='replace') return rval @@ -257,6 +259,7 @@ _os_overrides = { id(_os.fdopen) : _os.fdopen, id(_os.popen) : _os.popen, id(_os.read) : _os.read, + id(_os.statvfs) : _os.statvfs, id(_os.system) : _os.system, } diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py index 650f5135d..9e779520b 100644 --- a/pym/portage/cache/template.py +++ b/pym/portage/cache/template.py @@ -120,7 +120,7 @@ class database(object): yield (x, self[x]) def items(self): - return list(self.items()) + return list(self.iteritems()) def sync(self, rate=0): self.sync_rate = rate diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 976fe3e4e..cd3f3470e 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -124,7 +124,7 @@ class bindbapi(fakedbapi): encoding=_encodings['repo.content'], errors='backslashreplace') mydata[k] = v - for k, v in mydata.items(): + for k, v in list(mydata.items()): if not v: del mydata[k] mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index e9bde27fb..7f20e66fd 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -30,8 +30,10 @@ from portage import eclass_cache, auxdbkeys, doebuild, flatten, \ _eapi_is_deprecated from portage import os from portage import _encodings +from portage import _unicode_decode from portage import _unicode_encode +import os as _os import codecs import logging import stat @@ -448,42 +450,54 @@ class portdbapi(dbapi): the file we wanted. """ if not mycpv: - return "",0 + return ("", 0) mysplit = mycpv.split("/") psplit = pkgsplit(mysplit[1]) if psplit is None or len(mysplit) != 2: raise InvalidPackageName(mycpv) + # For optimal performace in this hot spot, we do manual unicode + # handling here instead of using the wrapped os module. + encoding = _encodings['fs'] + errors = 'strict' + if mytree: mytrees = [mytree] else: mytrees = self.porttrees[:] mytrees.reverse() - relative_path = os.path.join(mysplit[0], psplit[0], - mysplit[1] + ".ebuild") + relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \ + mysplit[1] + ".ebuild" if 'parse-eapi-glep-55' in self.doebuild_settings.features: glep55_startswith = '%s.ebuild-' % mysplit[1] for x in mytrees: - filename = x + os.sep + relative_path - if os.access(filename, os.R_OK): + filename = x + _os.sep + relative_path + if _os.access(_unicode_encode(filename, + encoding=encoding, errors=errors), _os.R_OK): return (filename, x) - pkgdir = os.path.join(x, mysplit[0], psplit[0]) + pkgdir = _os.path.join(x, mysplit[0], psplit[0]) try: - files = os.listdir(pkgdir) + files = _os.listdir(_unicode_encode(pkgdir, + encoding=encoding, errors=errors)) except OSError: continue for y in files: + try: + y = _unicode_decode(y, encoding=encoding, errors=errors) + except UnicodeDecodeError: + continue if y.startswith(glep55_startswith): - return (os.path.join(pkgdir, y), x) + return (_os.path.join(pkgdir, y), x) else: for x in mytrees: - file = x + os.sep + relative_path - if os.access(file, os.R_OK): - return[file, x] - return None, 0 + filename = x + _os.sep + relative_path + if _os.access(_unicode_encode(filename, + encoding=encoding, errors=errors), _os.R_OK): + return (filename, x) + return (None, 0) def _metadata_process(self, cpv, ebuild_path, repo_path): """ @@ -506,7 +520,7 @@ class portdbapi(dbapi): def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata, mtime): i = metadata - if hasattr(metadata, "iteritems"): + if hasattr(metadata, "items"): i = iter(metadata.items()) metadata = dict(i) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index c9d75d4c4..5d88f5b99 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -2231,7 +2231,7 @@ class vardbapi(dbapi): base_names = self._vardb._aux_cache["owners"]["base_names"] # Take inventory of all cached package hashes. - for name, hash_values in base_names.items(): + for name, hash_values in list(base_names.items()): if not isinstance(hash_values, dict): del base_names[name] continue @@ -2254,7 +2254,7 @@ class vardbapi(dbapi): # Delete any stale cache. stale_hashes = cached_hashes.difference(valid_pkg_hashes) if stale_hashes: - for base_name_hash, bucket in base_names.items(): + for base_name_hash, bucket in list(base_names.items()): for hash_value in stale_hashes.intersection(bucket): del bucket[hash_value] if not bucket: diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 97a613725..98448d959 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -393,7 +393,7 @@ class _use_dep(object): self.required = frozenset(chain( enabled_flags, disabled_flags, - *list(conditional.values()) + *conditional.values() )) self.enabled = frozenset(enabled_flags) diff --git a/pym/portage/proxy/objectproxy.py b/pym/portage/proxy/objectproxy.py index 00c5076d1..6dffff7d5 100644 --- a/pym/portage/proxy/objectproxy.py +++ b/pym/portage/proxy/objectproxy.py @@ -45,7 +45,15 @@ class ObjectProxy(object): return iter(object.__getattribute__(self, '_get_target')()) def __len__(self): - return len(object.__getattribute__(self, '_get_target')()) + try: + return len(object.__getattribute__(self, '_get_target')()) + except TypeError: + # For python 3.x with boolean target, len() results in + # TypeError, so return 0 or 1. + if bool(object.__getattribute__(self, '_get_target')()): + return 1 + else: + return 0 def __repr__(self): return repr(object.__getattribute__(self, '_get_target')()) -- 2.26.2