Merged from trunk -r14396:14402
authorFabian Groffen <grobian@gentoo.org>
Thu, 24 Sep 2009 17:28:17 +0000 (17:28 -0000)
committerFabian Groffen <grobian@gentoo.org>
Thu, 24 Sep 2009 17:28:17 +0000 (17:28 -0000)
   | 14397   | Fix test case for recent changes.                            |
   | zmedico |                                                              |

   | 14398   | Misc performance enhancements. Thanks to Marat Radchenko     |
   | zmedico | <marat@slonopotamus.org> for this patch from bug #276813.    |

   | 14399   | Decode directory names where appropriate in digestcheck().   |
   | zmedico | Thanks to Arfrever for reporting.                            |

   | 14400   | Avoid unicode errors in python 3. Thanks to Arfrever for     |
   | zmedico | reporting.                                                   |

   | 14401   | Warn if cpv_getkey() is given invalid input that triggers    |
   | zmedico | legacy code.                                                 |

   | 14402   | Fix previous commit.                                         |
   | zmedico |                                                              |

svn path=/main/branches/prefix/; revision=14407

bin/filter-bash-environment.py
pym/portage/__init__.py
pym/portage/cache/flat_hash.py
pym/portage/cache/metadata.py
pym/portage/dbapi/porttree.py
pym/portage/tests/util/test_stackDicts.py

index f6605acdc2fe58e296ed692427d354ceec0011f1..f503ebe2950b42b3ab8fed6cd33358dcdb8cfc7e 100755 (executable)
@@ -3,6 +3,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import codecs
 import os, re, sys
 
 here_doc_re = re.compile(r'.*\s<<[-]?(\w+)$')
@@ -112,6 +113,13 @@ if __name__ == "__main__":
                parser.error("Missing required PATTERN argument.")
        file_in = sys.stdin
        file_out = sys.stdout
+       if sys.hexversion >= 0x3000000:
+               file_in = codecs.iterdecode(sys.stdin.buffer.raw,
+                       'utf_8', errors='replace')
+               import io
+               file_out = io.TextIOWrapper(sys.stdout.buffer,
+                       'utf_8', errors='backslashreplace')
+
        var_pattern = args[0].split()
 
        # Filter invalid variable names that are not supported by bash.
index 4578c42c0b32d907568d0bcaef6d12adeca568b9..03f2336c2dbd3aabe9c85e0a045d28f30edde5a0 100644 (file)
@@ -5379,6 +5379,20 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
                                return 0
                        continue
                for d in dirs:
+                       try:
+                               d = _unicode_decode(d,
+                                       encoding=_encodings['fs'], errors='strict')
+                       except UnicodeDecodeError:
+                               d = _unicode_decode(d,
+                                       encoding=_encodings['fs'], errors='replace')
+                               writemsg(_("!!! Path contains invalid "
+                                       "character(s) for encoding '%s': '%s'") \
+                                       % (_encodings['fs'], os.path.join(parent, d)),
+                                       noiselevel=-1)
+                               if strict:
+                                       return 0
+                               dirs.remove(d)
+                               continue
                        if d.startswith(".") or d == "CVS":
                                dirs.remove(d)
                for f in files:
@@ -8089,6 +8103,10 @@ def cpv_getkey(mycpv):
        m = _cpv_key_re.match(mycpv)
        if m is not None:
                return m.group(2)
+
+       warnings.warn("portage.cpv_getkey() called with invalid cpv: '%s'" \
+               % (mycpv,), DeprecationWarning)
+
        myslash = mycpv.split("/", 1)
        mysplit=pkgsplit(myslash[-1])
        if mysplit is None:
index 918c935f596d5a388fe68fa30dcebc55a094ba14..983055a4b8672775051d3061b49f56b3d9814ac9 100644 (file)
@@ -9,6 +9,7 @@ from portage.cache import cache_errors
 import errno
 import stat
 import sys
+import os as _os
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
@@ -32,33 +33,33 @@ class database(fs_template.FsBased):
                        self._ensure_dirs()
 
        def _getitem(self, cpv):
-               fp = os.path.join(self.location, cpv)
+               # Don't use os.path.join, for better performance.
+               fp = self.location + _os.sep + cpv
                try:
                        myf = codecs.open(_unicode_encode(fp,
                                encoding=_encodings['fs'], errors='strict'),
                                mode='r', encoding=_encodings['repo.content'],
                                errors='replace')
                        try:
-                               d = self._parse_data(myf.readlines(), cpv)
+                               d = self._parse_data(myf.read().split("\n"), cpv)
                                if '_mtime_' not in d:
                                        # Backward compatibility with old cache
                                        # that uses mtime mangling.
-                                       d['_mtime_'] = long(os.fstat(myf.fileno()).st_mtime)
+                                       d['_mtime_'] = long(_os.fstat(myf.fileno()).st_mtime)
                                return d
                        finally:
                                myf.close()
                except (IOError, OSError) as e:
                        if e.errno != errno.ENOENT:
                                raise cache_errors.CacheCorruption(cpv, e)
-                       raise KeyError(cpv)
+                       raise KeyError(cpv, e)
 
        def _parse_data(self, data, cpv):
                try:
-                       d = dict(map(lambda x:x.rstrip("\n").split("=", 1), data))
+                       return dict( x.split("=", 1) for x in data )
                except ValueError as e:
                        # If a line is missing an "=", the split length is 1 instead of 2.
                        raise cache_errors.CacheCorruption(cpv, e)
-               return d
 
        def _setitem(self, cpv, values):
 #              import pdb;pdb.set_trace()
@@ -101,7 +102,6 @@ class database(fs_template.FsBased):
                        os.remove(fp)
                        raise cache_errors.CacheCorruption(cpv, e)
 
-
        def _delitem(self, cpv):
 #              import pdb;pdb.set_trace()
                try:
@@ -112,11 +112,9 @@ class database(fs_template.FsBased):
                        else:
                                raise cache_errors.CacheCorruption(cpv, e)
 
-
        def __contains__(self, cpv):
                return os.path.exists(os.path.join(self.location, cpv))
 
-
        def __iter__(self):
                """generator for walking the dir struct"""
                dirs = [self.location]
@@ -140,4 +138,3 @@ class database(fs_template.FsBased):
                                        continue
                                yield p[len_base+1:]
                        dirs.pop(0)
-
index 81b1f17fd42de61e8fd687e4c8b2ac55b12613c0..2704bd5135a3fd14cfae5a4a2c0c853a51868400 100644 (file)
@@ -52,7 +52,7 @@ class database(flat_hash.database):
                                d.clear()
                                try:
                                        for i, key in enumerate(self.auxdbkey_order):
-                                               d[key] = data[i].rstrip("\n")
+                                               d[key] = data[i]
                                except IndexError:
                                        pass
                                break
index b073e4268c18a1a43cfae370bd2b76930fc29bb0..f68d92fd2e34e8d0cd680fa694735a0ea8349742 100644 (file)
@@ -379,9 +379,6 @@ class portdbapi(dbapi):
                        "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository",
                        "RESTRICT", "SLOT"])
 
-               # Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict
-               # initialization until the first aux_get call.
-               self._aux_cache_slot_dict = None
                self._aux_cache = {}
                self._broken_ebuilds = set()
 
@@ -546,9 +543,10 @@ class portdbapi(dbapi):
                return metadata
 
        def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
-
                try:
-                       st = os.stat(ebuild_path)
+                       # Don't use unicode-wrapped os module, for better performance.
+                       st = _os.stat(_unicode_encode(ebuild_path,
+                               encoding=_encodings['fs'], errors='strict'))
                        emtime = st[stat.ST_MTIME]
                except OSError:
                        writemsg(_("!!! aux_get(): ebuild for " \
@@ -668,8 +666,7 @@ class portdbapi(dbapi):
                                mydata["_eclasses_"] = {}
 
                # do we have a origin repository name for the current package
-               mydata["repository"] = self._repository_map.get(
-                       os.path.sep.join(myebuild.split(os.path.sep)[:-3]), "")
+               mydata["repository"] = self._repository_map.get(mylocation, "")
 
                mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", []))
                mydata["_mtime_"] = long(st.st_mtime)
@@ -687,10 +684,7 @@ class portdbapi(dbapi):
                returnme = [mydata.get(x, "") for x in mylist]
 
                if cache_me:
-                       if self._aux_cache_slot_dict is None:
-                               self._aux_cache_slot_dict = \
-                                       slot_dict_class(self._aux_cache_keys)
-                       aux_cache = self._aux_cache_slot_dict()
+                       aux_cache = {}
                        for x in self._aux_cache_keys:
                                aux_cache[x] = mydata.get(x, "")
                        self._aux_cache[mycpv] = aux_cache
index aa8a712efa2e3bc2ac1fae8d8d247fe433a2b697..0c109a9cf28782db9674c5652863cc5a7eda22c3 100644 (file)
@@ -19,7 +19,7 @@ class StackDictsTestCase(TestCase):
                          False, ["a"], False ),
                          ( [ { "a":"b" }, None ], { "a":"b" },
                          False, [], True ),
-                         ( [ None ], None, False, [], False ),
+                         ( [ None ], {}, False, [], False ),
                          ( [ None, {}], {}, False, [], True ) ]