Merged from trunk -r14368:14378
authorFabian Groffen <grobian@gentoo.org>
Thu, 24 Sep 2009 17:24:01 +0000 (17:24 -0000)
committerFabian Groffen <grobian@gentoo.org>
Thu, 24 Sep 2009 17:24:01 +0000 (17:24 -0000)
   | 14369   | Don't wrap os.statvfs since it requires unicode string args  |
   | zmedico | in py3k.                                                     |

   | 14370   | Fix 'RuntimeError: dictionary changed size during iteration' |
   | zmedico | in _owners_db._populate() with py3k.                         |

   | 14371   | Fix items to prevent infinite recursion.                     |
   | zmedico |                                                              |

   | 14372   | Remove redundant list() since starred expressions work with  |
   | zmedico | iterators.                                                   |

   | 14373   | Fix 'RuntimeError: dictionary changed size during iteration' |
   | zmedico | in bindbapi.aux_update() with py3k.                          |

   | 14374   | Fix 'iteritems' attribute detection to look for 'items'      |
   | zmedico | instead.                                                     |

   | 14375   | Fix another potential 'RuntimeError: dictionary changed size |
   | zmedico | during iteration' exception in _owners_db._populate() for    |
   |         | py3k.                                                        |

   | 14376   | Optimize unicode wrappers. Thanks to Marat Radchenko         |
   | zmedico | <marat@slonopotamus.org> for this patch from bug #276813.    |

   | 14377   | For python 3.x with boolean target, len() results in         |
   | zmedico | TypeError, so make __len__ return 0 or 1 in this case.       |

   | 14378   | Do manual unicode handling (instead of using the wrapped os  |
   | zmedico | module) in portdbapi.findname2(), since it's a hotspot.      |
   |         | Thanks to Marat Radchenko <marat@slonopotamus.org> for the   |
   |         | initial patch.                                               |

svn path=/main/branches/prefix/; revision=14404

pym/portage/__init__.py
pym/portage/cache/template.py
pym/portage/dbapi/bintree.py
pym/portage/dbapi/porttree.py
pym/portage/dbapi/vartree.py
pym/portage/dep.py
pym/portage/proxy/objectproxy.py

index 8366664280795f47d28231902785b928fb19c890..71c496b2900f22e4b9e1d94fa7b844f195e63c74 100644 (file)
@@ -142,22 +142,26 @@ _encodings = {
 if _encodings['merge'] is None:
        _encodings['merge'] = 'ascii'
 
-def _unicode_encode(s, encoding=_encodings['content'],
-       errors='backslashreplace'):
-       if sys.hexversion >= 0x3000000:
+if sys.hexversion >= 0x3000000:
+       def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'):
                if isinstance(s, str):
                        s = s.encode(encoding, errors)
-       elif isinstance(s, unicode):
-               s = s.encode(encoding, errors)
-       return s
+               return s
 
-def _unicode_decode(s, encoding=_encodings['content'], errors='replace'):
-       if isinstance(s, bytes):
-               if sys.hexversion >= 0x3000000:
+       def _unicode_decode(s, encoding=_encodings['content'], errors='replace'):
+               if isinstance(s, bytes):
                        s = str(s, encoding=encoding, errors=errors)
-               else:
+               return s
+else:
+       def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'):
+               if isinstance(s, unicode):
+                       s = s.encode(encoding, errors)
+               return s
+
+       def _unicode_decode(s, encoding=_encodings['content'], errors='replace'):
+               if isinstance(s, bytes):
                        s = unicode(s, encoding=encoding, errors=errors)
-       return s
+               return s
 
 class _unicode_func_wrapper(object):
        """
@@ -190,24 +194,22 @@ class _unicode_func_wrapper(object):
 
                rval = self._func(*wrapped_args, **wrapped_kwargs)
 
-               if isinstance(rval, (bytes, basestring, list, tuple)):
-                       if isinstance(rval, (bytes, basestring)):
-                               rval = _unicode_decode(rval,
-                                       encoding=encoding, errors='replace')
-                       else:
-                               decoded_rval = []
-                               for x in rval:
-                                       try:
-                                               x = _unicode_decode(x, encoding=encoding, errors='strict')
-                                       except UnicodeDecodeError:
-                                               pass
-                                       else:
-                                               decoded_rval.append(x)
-
-                               if isinstance(rval, tuple):
-                                       rval = tuple(decoded_rval)
+               if isinstance(rval, (list, tuple)):
+                       decoded_rval = []
+                       for x in rval:
+                               try:
+                                       x = _unicode_decode(x, encoding=encoding, errors='strict')
+                               except UnicodeDecodeError:
+                                       pass
                                else:
-                                       rval = decoded_rval
+                                       decoded_rval.append(x)
+
+                       if isinstance(rval, tuple):
+                               rval = tuple(decoded_rval)
+                       else:
+                               rval = decoded_rval
+               else:
+                       rval = _unicode_decode(rval, encoding=encoding, errors='replace')
 
                return rval
 
@@ -257,6 +259,7 @@ _os_overrides = {
        id(_os.fdopen)        : _os.fdopen,
        id(_os.popen)         : _os.popen,
        id(_os.read)          : _os.read,
+       id(_os.statvfs)       : _os.statvfs,
        id(_os.system)        : _os.system,
 }
 
index 650f5135d5776e2526ba2a59dd0783382ab6ee9a..9e779520b3422c417503d7d03df03bfa5f33e994 100644 (file)
@@ -120,7 +120,7 @@ class database(object):
                        yield (x, self[x])
 
        def items(self):
-               return list(self.items())
+               return list(self.iteritems())
 
        def sync(self, rate=0):
                self.sync_rate = rate
index 976fe3e4e73301d31dca55263d885d556d8964ae..cd3f3470e9d95fc48b431a08986d594908891376 100644 (file)
@@ -124,7 +124,7 @@ class bindbapi(fakedbapi):
                                encoding=_encodings['repo.content'], errors='backslashreplace')
                        mydata[k] = v
 
-               for k, v in mydata.items():
+               for k, v in list(mydata.items()):
                        if not v:
                                del mydata[k]
                mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
index e9bde27fb110d3dd2d257ca6bd8c37f041e9eaec..7f20e66fd5d1eeaa4d4b0c49f5fe6ec523519936 100644 (file)
@@ -30,8 +30,10 @@ from portage import eclass_cache, auxdbkeys, doebuild, flatten, \
        _eapi_is_deprecated
 from portage import os
 from portage import _encodings
+from portage import _unicode_decode
 from portage import _unicode_encode
 
+import os as _os
 import codecs
 import logging
 import stat
@@ -448,42 +450,54 @@ class portdbapi(dbapi):
                the file we wanted.
                """
                if not mycpv:
-                       return "",0
+                       return ("", 0)
                mysplit = mycpv.split("/")
                psplit = pkgsplit(mysplit[1])
                if psplit is None or len(mysplit) != 2:
                        raise InvalidPackageName(mycpv)
 
+               # For optimal performace in this hot spot, we do manual unicode
+               # handling here instead of using the wrapped os module.
+               encoding = _encodings['fs']
+               errors = 'strict'
+
                if mytree:
                        mytrees = [mytree]
                else:
                        mytrees = self.porttrees[:]
                        mytrees.reverse()
 
-               relative_path = os.path.join(mysplit[0], psplit[0],
-                       mysplit[1] + ".ebuild")
+               relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \
+                       mysplit[1] + ".ebuild"
 
                if 'parse-eapi-glep-55' in self.doebuild_settings.features:
                        glep55_startswith = '%s.ebuild-' % mysplit[1]
                        for x in mytrees:
-                               filename = x + os.sep + relative_path
-                               if os.access(filename, os.R_OK):
+                               filename = x + _os.sep + relative_path
+                               if _os.access(_unicode_encode(filename,
+                                       encoding=encoding, errors=errors), _os.R_OK):
                                        return (filename, x)
 
-                               pkgdir = os.path.join(x, mysplit[0], psplit[0])
+                               pkgdir = _os.path.join(x, mysplit[0], psplit[0])
                                try:
-                                       files = os.listdir(pkgdir)
+                                       files = _os.listdir(_unicode_encode(pkgdir,
+                                               encoding=encoding, errors=errors))
                                except OSError:
                                        continue
                                for y in files:
+                                       try:
+                                               y = _unicode_decode(y, encoding=encoding, errors=errors)
+                                       except UnicodeDecodeError:
+                                               continue
                                        if y.startswith(glep55_startswith):
-                                               return (os.path.join(pkgdir, y), x)
+                                               return (_os.path.join(pkgdir, y), x)
                else:
                        for x in mytrees:
-                               file = x + os.sep + relative_path
-                               if os.access(file, os.R_OK):
-                                       return[file, x]
-               return None, 0
+                               filename = x + _os.sep + relative_path
+                               if _os.access(_unicode_encode(filename,
+                                       encoding=encoding, errors=errors), _os.R_OK):
+                                       return (filename, x)
+               return (None, 0)
 
        def _metadata_process(self, cpv, ebuild_path, repo_path):
                """
@@ -506,7 +520,7 @@ class portdbapi(dbapi):
        def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata, mtime):
 
                i = metadata
-               if hasattr(metadata, "iteritems"):
+               if hasattr(metadata, "items"):
                        i = iter(metadata.items())
                metadata = dict(i)
 
index c9d75d4c44cd645f7479f88ab28ed64f3869597f..5d88f5b9939c3052956eaf836ae37104e9bd6ee2 100644 (file)
@@ -2231,7 +2231,7 @@ class vardbapi(dbapi):
                        base_names = self._vardb._aux_cache["owners"]["base_names"]
 
                        # Take inventory of all cached package hashes.
-                       for name, hash_values in base_names.items():
+                       for name, hash_values in list(base_names.items()):
                                if not isinstance(hash_values, dict):
                                        del base_names[name]
                                        continue
@@ -2254,7 +2254,7 @@ class vardbapi(dbapi):
                        # Delete any stale cache.
                        stale_hashes = cached_hashes.difference(valid_pkg_hashes)
                        if stale_hashes:
-                               for base_name_hash, bucket in base_names.items():
+                               for base_name_hash, bucket in list(base_names.items()):
                                        for hash_value in stale_hashes.intersection(bucket):
                                                del bucket[hash_value]
                                        if not bucket:
index 97a613725a24bd58af2990b8519e9f57422c0e15..98448d9593f76bc59019d3dec5afe0ff0d8cdde0 100644 (file)
@@ -393,7 +393,7 @@ class _use_dep(object):
                self.required = frozenset(chain(
                        enabled_flags,
                        disabled_flags,
-                       *list(conditional.values())
+                       *conditional.values()
                ))
 
                self.enabled = frozenset(enabled_flags)
index 00c5076d18305a03330a4021da7e3b580152eff3..6dffff7d5ed57434b297eae91d1943eedcbf8dd1 100644 (file)
@@ -45,7 +45,15 @@ class ObjectProxy(object):
                return iter(object.__getattribute__(self, '_get_target')())
 
        def __len__(self):
-               return len(object.__getattribute__(self, '_get_target')())
+               try:
+                       return len(object.__getattribute__(self, '_get_target')())
+               except TypeError:
+                       # For python 3.x with boolean target, len() results in
+                       # TypeError, so return 0 or 1.
+                       if bool(object.__getattribute__(self, '_get_target')()):
+                               return 1
+                       else:
+                               return 0
 
        def __repr__(self):
                return repr(object.__getattribute__(self, '_get_target')())