Merged from trunk -r13919:13929
authorFabian Groffen <grobian@gentoo.org>
Fri, 14 Aug 2009 19:52:13 +0000 (19:52 -0000)
committerFabian Groffen <grobian@gentoo.org>
Fri, 14 Aug 2009 19:52:13 +0000 (19:52 -0000)
   | 13920   | correct pointer, as pointed out by darkside in IRC           |
   | grobian |                                                              |

   | 13921   | Bug #280460 - Fix UnicodeDecodeError in env_update() due to  |
   | zmedico | non-unicode strings from getconfig(). TODO: Make getconfig() |
   |         | return unicode.                                              |

   | 13924   | Fix UnicodeEncodeError in                                    |
   | zmedico | vardbapi._owners_cache._hash_str(), reported by jlec on irc. |

   | 13925   | Fix transposition of CodecInfo streamreader and streamwriter |
   | zmedico | attributes inside _gen_missing_encodings(). This solves      |
   |         | 'TypeError: an integer is required' exceptions from          |
   |         | codecs.open().readlines([sizehint]), triggered when python   |
   |         | is built with USE=build (stage 1). Thanks to Daniel Robbins  |
   |         | for reporting.                                               |

   | 13926   | Do not pass unicode strings into os.walk calls, since it can |
   | zmedico | cause internal os.path.join calls to raise                   |
   |         | UnicodeDecodeError.                                          |

   | 13927   | Remove odd os.getcwd() call used inside main() when          |
   | zmedico | searching for tests.                                         |

   | 13928   | Use realpath to resolve __file__ when searching for test     |
   | zmedico | inside main().                                               |

   | 13929   | Fix broken os.walk call in _post_src_install_uid_fix().      |
   | zmedico |                                                              |

svn path=/main/branches/prefix/; revision=14030

man/portage.5
pym/portage/__init__.py
pym/portage/cache/ebuild_xattr.py
pym/portage/dbapi/vartree.py
pym/portage/env/loaders.py
pym/portage/manifest.py
pym/portage/sets/files.py
pym/portage/tests/__init__.py
pym/portage/update.py
pym/portage/util.py

index a5c438612137db1313f52d93279b090ea9242898..37bf93148d7138775dde9d68621ca8d285411803 100644 (file)
@@ -525,7 +525,7 @@ versions earlier than 1.0.4496.  No problem!
 .BR package.unmask
 Just like package.mask above, except here you list packages you want to 
 unmask.  Useful for overriding the global package.mask file (see 
-below).  Note that this does not override packages that are masked via 
+above).  Note that this does not override packages that are masked via 
 KEYWORDS.
 .TP
 .BR package.use
index 873b3eb331d09111f0235bec5827228c3e9904e6..9f2a82e301545fd8eab2611a21cd9efe1f9eecc0 100644 (file)
@@ -686,6 +686,18 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None,
                        # broken symlink or file removed by a concurrent process
                        writemsg("!!! File Not Found: '%s'\n" % file_path, noiselevel=-1)
                        continue
+
+               # TODO: Make getconfig() return unicode.
+               unicode_config = {}
+               for k, v in myconfig.iteritems():
+                       if not isinstance(k, unicode):
+                               k = unicode(k, encoding='utf8', errors='replace')
+                       if not isinstance(v, unicode):
+                               v = unicode(v, encoding='utf8', errors='replace')
+                       unicode_config[k] = v
+               myconfig = unicode_config
+               del unicode_config
+
                config_list.append(myconfig)
                if "SPACE_SEPARATED" in myconfig:
                        space_separated.update(myconfig["SPACE_SEPARATED"].split())
@@ -4911,6 +4923,11 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
        """ epatch will just grab all the patches out of a directory, so we have to
        make sure there aren't any foreign files that it might grab."""
        filesdir = os.path.join(pkgdir, "files")
+       if isinstance(filesdir, unicode):
+               # Avoid UnicodeDecodeError raised from
+               # os.path.join when called by os.walk.
+               filesdir = filesdir.encode('utf_8', 'replace')
+
        for parent, dirs, files in os.walk(filesdir):
                for d in dirs:
                        if d.startswith(".") or d == "CVS":
@@ -5168,7 +5185,13 @@ def _post_src_install_uid_fix(mysettings):
                os.system("chflags -R nosunlnk,nouunlnk %s 2>/dev/null" % \
                        (_shell_quote(mysettings["D"]),))
 
-       for parent, dirs, files in os.walk(mysettings["D"]):
+       destdir = mysettings["D"]
+       if isinstance(destdir, unicode):
+               # Avoid UnicodeDecodeError raised from
+               # os.path.join when called by os.walk.
+               destdir = destdir.encode('utf_8', 'replace')
+
+       for parent, dirs, files in os.walk(destdir):
                for fname in chain(dirs, files):
                        fpath = os.path.join(parent, fname)
                        mystat = os.lstat(fpath)
@@ -8057,8 +8080,8 @@ def _gen_missing_encodings(missing_encodings):
                        decode=utf8decode,
                        incrementalencoder=Utf8IncrementalEncoder,
                        incrementaldecoder=Utf8IncrementalDecoder,
-                       streamreader=Utf8StreamWriter,
-                       streamwriter=Utf8StreamReader,
+                       streamreader=Utf8StreamReader,
+                       streamwriter=Utf8StreamWriter,
                )
 
                for alias in ('utf_8', 'u8', 'utf', 'utf8', 'utf8_ucs2', 'utf8_ucs4'):
index 83ed8a783c18a30dfc5d3ea76aa11dfba9d8d16a..98e848651a9ac819815069a28f4a174c9524a383 100644 (file)
@@ -153,7 +153,14 @@ class database(fs_template.FsBased):
                return os.path.exists(self.__get_path(cpv))
 
        def __iter__(self):
-               for root,dirs,files in os.walk(self.portdir):
+
+               portdir = self.portdir
+               if isinstance(portdir, unicode):
+                       # Avoid UnicodeDecodeError raised from
+                       # os.path.join when called by os.walk.
+                       portdir = portdir.encode('utf_8', 'replace')
+
+               for root, dirs, files in os.walk(portdir):
                        for file in files:
                                if file[-7:] == '.ebuild':
                                        cat = os.path.basename(os.path.dirname(root))
index 7f4d20397588a94820066961c2298e2a48aada16..7b201e9b094d3e4d218d5d51d0fd143bc83760bf 100644 (file)
@@ -2119,7 +2119,7 @@ class vardbapi(dbapi):
                        h = self._new_hash()
                        # Always use a constant utf_8 encoding here, since
                        # the "default" encoding can change.
-                       h.update(s)
+                       h.update(s.encode('utf_8', 'replace'))
                        h = h.hexdigest()
                        h = h[-self._hex_chars:]
                        h = int(h, 16)
@@ -3930,6 +3930,12 @@ class dblink(object):
                myfilelist = []
                mylinklist = []
                paths_with_newlines = []
+
+               if isinstance(srcroot, unicode):
+                       # Avoid UnicodeDecodeError raised from
+                       # os.path.join when called by os.walk.
+                       srcroot = srcroot.encode('utf_8', 'replace')
+
                srcroot_len = len(srcroot)
                def onerror(e):
                        raise
index 7b4d727213bb8face02bc92b62af196d4de5614e..854304125f331d8794d8483fd72d90287a95be75 100644 (file)
@@ -39,6 +39,12 @@ def RecursiveFileLoader(filename):
        @rtype: list
        @returns: List of files to process
        """
+
+       if isinstance(filename, unicode):
+               # Avoid UnicodeDecodeError raised from
+               # os.path.join when called by os.walk.
+               filename = filename.encode('utf_8', 'replace')
+
        try:
                st = os.stat(filename)
        except OSError:
index 80a0c16424c8a5167c8b0706646846284dc0adb4..a717d584194f5f395c9212eae64b3843b2fc233a 100644 (file)
@@ -306,7 +306,14 @@ class Manifest(object):
                cpvlist = []
                pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
                cat = self._pkgdir_category()
-               for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(self.pkgdir):
+
+               pkgdir = self.pkgdir
+               if isinstance(pkgdir, unicode):
+                       # Avoid UnicodeDecodeError raised from
+                       # os.path.join when called by os.walk.
+                       pkgdir = pkgdir.encode('utf_8', 'replace')
+
+               for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
                        break
                for f in pkgdir_files:
                        if f[:1] == ".":
@@ -334,8 +341,15 @@ class Manifest(object):
                                continue
                        self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes)
                recursive_files = []
-               cut_len = len(os.path.join(self.pkgdir, "files") + os.sep)
-               for parentdir, dirs, files in os.walk(os.path.join(self.pkgdir, "files")):
+
+               pkgdir = self.pkgdir
+               if isinstance(pkgdir, unicode):
+                       # Avoid UnicodeDecodeError raised from
+                       # os.path.join when called by os.walk.
+                       pkgdir = pkgdir.encode('utf_8', 'replace')
+
+               cut_len = len(os.path.join(pkgdir, "files") + os.sep)
+               for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
                        for f in files:
                                full_path = os.path.join(parentdir, f)
                                recursive_files.append(full_path[cut_len:])
index 3188025ef1f518e442085dfac968a66fc349f2e8..17d2f0bc5b4fc27fcbce3a58f196d4c0c979b576 100644 (file)
@@ -124,9 +124,22 @@ class StaticFileSet(EditablePackageSet):
                                directory = self._repopath_sub.sub(trees["porttree"].dbapi.treemap[match.groupdict()["reponame"]], directory)
                        except KeyError:
                                raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"])
+
+               if isinstance(directory, unicode):
+                       # Avoid UnicodeDecodeError raised from
+                       # os.path.join when called by os.walk.
+                       directory_unicode = directory
+                       directory = directory.encode('utf_8', 'replace')
+               else:
+                       directory_unicode = unicode(directory,
+                               encoding='utf_8', errors='replace')
+
                if os.path.isdir(directory):
                        directory = normalize_path(directory)
                        for parent, dirs, files in os.walk(directory):
+                               if not isinstance(parent, unicode):
+                                       parent = unicode(parent,
+                                               encoding='utf_8', errors='replace')
                                for d in dirs[:]:
                                        if d[:1] == '.':
                                                dirs.remove(d)
@@ -139,7 +152,7 @@ class StaticFileSet(EditablePackageSet):
                                        if filename.endswith(".metadata"):
                                                continue
                                        filename = os.path.join(parent,
-                                               filename)[1 + len(directory):]
+                                               filename)[1 + len(directory_unicode):]
                                        myname = name_pattern.replace("$name", filename)
                                        myname = myname.replace("${name}", filename)
                                        rValue[myname] = StaticFileSet(
index 927af49c74804e338746c539326c4945a883977a..8676c6ae25253b2cadeeb1602fc578e615b40c93 100644 (file)
@@ -12,13 +12,13 @@ def main():
 
        TEST_FILE = '__test__'
        suite = unittest.TestSuite()
-       basedir = os.path.dirname(__file__)
+       basedir = os.path.dirname(os.path.realpath(__file__))
        testDirs = []
 
   # the os.walk help mentions relative paths as being quirky
        # I was tired of adding dirs to the list, so now we add __test__
        # to each dir we want tested.
-       for root, dirs, files in os.walk(os.getcwd()):
+       for root, dirs, files in os.walk(basedir):
                if ".svn" in dirs:
                        dirs.remove('.svn')
                if TEST_FILE in files:
index c7001803ac7995d35f4fdfb67c476d94963975a3..4e309e4019cf00b08c6d67fed840822131c7912e 100644 (file)
@@ -146,6 +146,12 @@ def update_config_files(config_root, protect, protect_mask, update_iter):
        protect - list of paths from CONFIG_PROTECT
        protect_mask - list of paths from CONFIG_PROTECT_MASK
        update_iter - list of update commands as returned from parse_updates()"""
+
+       if isinstance(config_root, unicode):
+               # Avoid UnicodeDecodeError raised from
+               # os.path.join when called by os.walk.
+               config_root = config_root.encode('utf_8', 'replace')
+
        config_root = normalize_path(config_root)
        update_files = {}
        file_contents = {}
index 0cd40cc2bb7ab85bcc06431735b615c33ffc99b6..6e11c46f29c4115ae26b555c8ad5845d63c155fa 100644 (file)
@@ -781,6 +781,11 @@ def apply_recursive_permissions(top, uid=-1, gid=-1,
        Returns True if all permissions are applied and False if some are left
        unapplied."""
 
+       if isinstance(top, unicode):
+               # Avoid UnicodeDecodeError raised from
+               # os.path.join when called by os.walk.
+               top = top.encode('utf_8', 'replace')
+
        if onerror is None:
                # Default behavior is to dump errors to stderr so they won't
                # go unnoticed.  Callers can pass in a quiet instance.