From: Fabian Groffen Date: Fri, 14 Aug 2009 19:52:13 +0000 (-0000) Subject: Merged from trunk -r13919:13929 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=31897a93c5a9e41b3033587b552aaf2cedd020fe;p=portage.git Merged from trunk -r13919:13929 | 13920 | correct pointer, as pointed out by darkside in IRC | | grobian | | | 13921 | Bug #280460 - Fix UnicodeDecodeError in env_update() due to | | zmedico | non-unicode strings from getconfig(). TODO: Make getconfig() | | | return unicode. | | 13924 | Fix UnicodeEncodeError in | | zmedico | vardbapi._owners_cache._hash_str(), reported by jlec on irc. | | 13925 | Fix transposition of CodecInfo streamreader and streamwriter | | zmedico | attributes inside _gen_missing_encodings(). This solves | | | 'TypeError: an integer is required' exceptions from | | | codecs.open().readlines([sizehint]), triggered when python | | | is built with USE=build (stage 1). Thanks to Daniel Robbins | | | for reporting. | | 13926 | Do not pass unicode strings into os.walk calls, since it can | | zmedico | cause internal os.path.join calls to raise | | | UnicodeDecodeError. | | 13927 | Remove odd os.getcwd() call used inside main() when | | zmedico | searching for tests. | | 13928 | Use realpath to resolve __file__ when searching for test | | zmedico | inside main(). | | 13929 | Fix broken os.walk call in _post_src_install_uid_fix(). | | zmedico | | svn path=/main/branches/prefix/; revision=14030 --- diff --git a/man/portage.5 b/man/portage.5 index a5c438612..37bf93148 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -525,7 +525,7 @@ versions earlier than 1.0.4496. No problem! .BR package.unmask Just like package.mask above, except here you list packages you want to unmask. Useful for overriding the global package.mask file (see -below). Note that this does not override packages that are masked via +above). Note that this does not override packages that are masked via KEYWORDS. .TP .BR package.use diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 873b3eb33..9f2a82e30 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -686,6 +686,18 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, # broken symlink or file removed by a concurrent process writemsg("!!! File Not Found: '%s'\n" % file_path, noiselevel=-1) continue + + # TODO: Make getconfig() return unicode. + unicode_config = {} + for k, v in myconfig.iteritems(): + if not isinstance(k, unicode): + k = unicode(k, encoding='utf8', errors='replace') + if not isinstance(v, unicode): + v = unicode(v, encoding='utf8', errors='replace') + unicode_config[k] = v + myconfig = unicode_config + del unicode_config + config_list.append(myconfig) if "SPACE_SEPARATED" in myconfig: space_separated.update(myconfig["SPACE_SEPARATED"].split()) @@ -4911,6 +4923,11 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0): """ epatch will just grab all the patches out of a directory, so we have to make sure there aren't any foreign files that it might grab.""" filesdir = os.path.join(pkgdir, "files") + if isinstance(filesdir, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + filesdir = filesdir.encode('utf_8', 'replace') + for parent, dirs, files in os.walk(filesdir): for d in dirs: if d.startswith(".") or d == "CVS": @@ -5168,7 +5185,13 @@ def _post_src_install_uid_fix(mysettings): os.system("chflags -R nosunlnk,nouunlnk %s 2>/dev/null" % \ (_shell_quote(mysettings["D"]),)) - for parent, dirs, files in os.walk(mysettings["D"]): + destdir = mysettings["D"] + if isinstance(destdir, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + destdir = destdir.encode('utf_8', 'replace') + + for parent, dirs, files in os.walk(destdir): for fname in chain(dirs, files): fpath = os.path.join(parent, fname) mystat = os.lstat(fpath) @@ -8057,8 +8080,8 @@ def _gen_missing_encodings(missing_encodings): decode=utf8decode, incrementalencoder=Utf8IncrementalEncoder, incrementaldecoder=Utf8IncrementalDecoder, - streamreader=Utf8StreamWriter, - streamwriter=Utf8StreamReader, + streamreader=Utf8StreamReader, + streamwriter=Utf8StreamWriter, ) for alias in ('utf_8', 'u8', 'utf', 'utf8', 'utf8_ucs2', 'utf8_ucs4'): diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py index 83ed8a783..98e848651 100644 --- a/pym/portage/cache/ebuild_xattr.py +++ b/pym/portage/cache/ebuild_xattr.py @@ -153,7 +153,14 @@ class database(fs_template.FsBased): return os.path.exists(self.__get_path(cpv)) def __iter__(self): - for root,dirs,files in os.walk(self.portdir): + + portdir = self.portdir + if isinstance(portdir, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + portdir = portdir.encode('utf_8', 'replace') + + for root, dirs, files in os.walk(portdir): for file in files: if file[-7:] == '.ebuild': cat = os.path.basename(os.path.dirname(root)) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 7f4d20397..7b201e9b0 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -2119,7 +2119,7 @@ class vardbapi(dbapi): h = self._new_hash() # Always use a constant utf_8 encoding here, since # the "default" encoding can change. - h.update(s) + h.update(s.encode('utf_8', 'replace')) h = h.hexdigest() h = h[-self._hex_chars:] h = int(h, 16) @@ -3930,6 +3930,12 @@ class dblink(object): myfilelist = [] mylinklist = [] paths_with_newlines = [] + + if isinstance(srcroot, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + srcroot = srcroot.encode('utf_8', 'replace') + srcroot_len = len(srcroot) def onerror(e): raise diff --git a/pym/portage/env/loaders.py b/pym/portage/env/loaders.py index 7b4d72721..854304125 100644 --- a/pym/portage/env/loaders.py +++ b/pym/portage/env/loaders.py @@ -39,6 +39,12 @@ def RecursiveFileLoader(filename): @rtype: list @returns: List of files to process """ + + if isinstance(filename, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + filename = filename.encode('utf_8', 'replace') + try: st = os.stat(filename) except OSError: diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 80a0c1642..a717d5841 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -306,7 +306,14 @@ class Manifest(object): cpvlist = [] pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) cat = self._pkgdir_category() - for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(self.pkgdir): + + pkgdir = self.pkgdir + if isinstance(pkgdir, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + pkgdir = pkgdir.encode('utf_8', 'replace') + + for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir): break for f in pkgdir_files: if f[:1] == ".": @@ -334,8 +341,15 @@ class Manifest(object): continue self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes) recursive_files = [] - cut_len = len(os.path.join(self.pkgdir, "files") + os.sep) - for parentdir, dirs, files in os.walk(os.path.join(self.pkgdir, "files")): + + pkgdir = self.pkgdir + if isinstance(pkgdir, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + pkgdir = pkgdir.encode('utf_8', 'replace') + + cut_len = len(os.path.join(pkgdir, "files") + os.sep) + for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")): for f in files: full_path = os.path.join(parentdir, f) recursive_files.append(full_path[cut_len:]) diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index 3188025ef..17d2f0bc5 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -124,9 +124,22 @@ class StaticFileSet(EditablePackageSet): directory = self._repopath_sub.sub(trees["porttree"].dbapi.treemap[match.groupdict()["reponame"]], directory) except KeyError: raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"]) + + if isinstance(directory, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + directory_unicode = directory + directory = directory.encode('utf_8', 'replace') + else: + directory_unicode = unicode(directory, + encoding='utf_8', errors='replace') + if os.path.isdir(directory): directory = normalize_path(directory) for parent, dirs, files in os.walk(directory): + if not isinstance(parent, unicode): + parent = unicode(parent, + encoding='utf_8', errors='replace') for d in dirs[:]: if d[:1] == '.': dirs.remove(d) @@ -139,7 +152,7 @@ class StaticFileSet(EditablePackageSet): if filename.endswith(".metadata"): continue filename = os.path.join(parent, - filename)[1 + len(directory):] + filename)[1 + len(directory_unicode):] myname = name_pattern.replace("$name", filename) myname = myname.replace("${name}", filename) rValue[myname] = StaticFileSet( diff --git a/pym/portage/tests/__init__.py b/pym/portage/tests/__init__.py index 927af49c7..8676c6ae2 100644 --- a/pym/portage/tests/__init__.py +++ b/pym/portage/tests/__init__.py @@ -12,13 +12,13 @@ def main(): TEST_FILE = '__test__' suite = unittest.TestSuite() - basedir = os.path.dirname(__file__) + basedir = os.path.dirname(os.path.realpath(__file__)) testDirs = [] # the os.walk help mentions relative paths as being quirky # I was tired of adding dirs to the list, so now we add __test__ # to each dir we want tested. - for root, dirs, files in os.walk(os.getcwd()): + for root, dirs, files in os.walk(basedir): if ".svn" in dirs: dirs.remove('.svn') if TEST_FILE in files: diff --git a/pym/portage/update.py b/pym/portage/update.py index c7001803a..4e309e401 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -146,6 +146,12 @@ def update_config_files(config_root, protect, protect_mask, update_iter): protect - list of paths from CONFIG_PROTECT protect_mask - list of paths from CONFIG_PROTECT_MASK update_iter - list of update commands as returned from parse_updates()""" + + if isinstance(config_root, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + config_root = config_root.encode('utf_8', 'replace') + config_root = normalize_path(config_root) update_files = {} file_contents = {} diff --git a/pym/portage/util.py b/pym/portage/util.py index 0cd40cc2b..6e11c46f2 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -781,6 +781,11 @@ def apply_recursive_permissions(top, uid=-1, gid=-1, Returns True if all permissions are applied and False if some are left unapplied.""" + if isinstance(top, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + top = top.encode('utf_8', 'replace') + if onerror is None: # Default behavior is to dump errors to stderr so they won't # go unnoticed. Callers can pass in a quiet instance.