From: Fabian Groffen Date: Mon, 24 Aug 2009 09:38:13 +0000 (-0000) Subject: Merged from trunk -r14107:14117 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=3722205c9586684a2d9a57a8f11bf2542049ff32;p=portage.git Merged from trunk -r14107:14117 | 14108 | Use _encodings where appropriate. | | zmedico | | | 14109 | Use _encodings where appropriate. | | zmedico | | | 14110 | Use _encodings where appropriate. | | zmedico | | | 14111 | Use _encodings where appropriate and add | | zmedico | _encodings['stdio'] for stdout encoding. | | 14112 | Fix typo. | | zmedico | | | 14113 | Replace _content_encoding, _fs_encoding, and | | zmedico | _merge_encoding with direct usage of _encodings. | | 14114 | Support QA_DT_HASH_${ARCH} and QA_PRESTRIPPED_${ARCH} (bug | | arfrever | #271416). | | 14115 | Add support for QA_SONAME variable (bug #281964). Add | | arfrever | support for QA_NEEDED variable. | | 14116 | Improve an example. Patch by Jeremy Olexa. | | arfrever | | | 14117 | Rename QA_NEEDED to QA_DT_NEEDED. | | arfrever | | svn path=/main/branches/prefix/; revision=14143 --- diff --git a/bin/ebuild-helpers/prepstrip b/bin/ebuild-helpers/prepstrip index 49c886cb8..5c0a9dc5f 100755 --- a/bin/ebuild-helpers/prepstrip +++ b/bin/ebuild-helpers/prepstrip @@ -100,6 +100,8 @@ save_elf_debug() { if ! hasq binchecks ${RESTRICT} && \ ! hasq strip ${RESTRICT} ; then log=$T/scanelf-already-stripped.log + qa_var="QA_PRESTRIPPED_${ARCH/-/_}" + [[ -n ${!qa_var} ]] && QA_PRESTRIPPED="${!qa_var}" scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^$D##" > "$log" if [[ -n $QA_PRESTRIPPED && -s $log && \ ${QA_STRICT_PRESTRIPPED-unset} = unset ]] ; then diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 16c3c192c..26d992c46 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1862,6 +1862,7 @@ _source_ebuild() { # This needs to be exported since prepstrip is a separate shell script. [[ -n $QA_PRESTRIPPED ]] && export QA_PRESTRIPPED + eval "[[ -n \$QA_PRESTRIPPED_$ARCH ]] && export QA_PRESTRIPPED_$ARCH" } if ! hasq "$EBUILD_PHASE" clean cleanrm ; then diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 481ac149e..4f6478fa6 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -167,6 +167,8 @@ install_qa_check() { # Check for files built without respecting LDFLAGS if [[ "${LDFLAGS}" == *--hash-style=gnu* ]] && [[ "${PN}" != *-bin ]] ; then + qa_var="QA_DT_HASH_${ARCH/-/_}" + eval "[[ -n \${!qa_var} ]] && QA_DT_HASH=(\"\${${qa_var}[@]}\")" f=$(scanelf -qyRF '%k %p' -k .hash "${D}" | sed -e "s:\.hash ::") if [[ -n ${f} ]] ; then echo "${f}" > "${T}"/scanelf-ignored-LDFLAGS.log @@ -241,26 +243,71 @@ install_qa_check() { die "Aborting due to QA concerns: ${die_msg}" fi - # Run some sanity checks on shared libraries - for d in "${D}"lib* "${D}"usr/lib* ; do - f=$(scanelf -ByF '%S %p' "${d}"/lib*.so* | gawk '$2 == "" { print }') + # Check for shared libraries lacking SONAMEs + qa_var="QA_SONAME_${ARCH/-/_}" + eval "[[ -n \${!qa_var} ]] && QA_SONAME=(\"\${${qa_var}[@]}\")" + f=$(scanelf -ByF '%S %p' "${D}"{,usr/}lib*/lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${D}:/:") + if [[ -n ${f} ]] ; then + echo "${f}" > "${T}"/scanelf-missing-SONAME.log + if [[ "${QA_STRICT_SONAME-unset}" == unset ]] ; then + if [[ ${#QA_SONAME[@]} -gt 1 ]] ; then + for x in "${QA_SONAME[@]}" ; do + sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-SONAME.log + done + else + local shopts=$- + set -o noglob + for x in ${QA_SONAME} ; do + sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-SONAME.log + done + set +o noglob + set -${shopts} + fi + fi + f=$(<"${T}"/scanelf-missing-SONAME.log) if [[ -n ${f} ]] ; then vecho -ne '\a\n' eqawarn "QA Notice: The following shared libraries lack a SONAME" eqawarn "${f}" vecho -ne '\a\n' sleep 1 + else + rm -f "${T}"/scanelf-missing-SONAME.log fi + fi - f=$(scanelf -ByF '%n %p' "${d}"/lib*.so* | gawk '$2 == "" { print }') + # Check for shared libraries lacking NEEDED entries + qa_var="QA_DT_NEEDED_${ARCH/-/_}" + eval "[[ -n \${!qa_var} ]] && QA_DT_NEEDED=(\"\${${qa_var}[@]}\")" + f=$(scanelf -ByF '%n %p' "${D}"{,usr/}lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${D}:/:") + if [[ -n ${f} ]] ; then + echo "${f}" > "${T}"/scanelf-missing-NEEDED.log + if [[ "${QA_STRICT_DT_NEEDED-unset}" == unset ]] ; then + if [[ ${#QA_DT_NEEDED[@]} -gt 1 ]] ; then + for x in "${QA_DT_NEEDED[@]}" ; do + sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-NEEDED.log + done + else + local shopts=$- + set -o noglob + for x in ${QA_DT_NEEDED} ; do + sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-NEEDED.log + done + set +o noglob + set -${shopts} + fi + fi + f=$(<"${T}"/scanelf-missing-NEEDED.log) if [[ -n ${f} ]] ; then vecho -ne '\a\n' eqawarn "QA Notice: The following shared libraries lack NEEDED entries" eqawarn "${f}" vecho -ne '\a\n' sleep 1 + else + rm -f "${T}"/scanelf-missing-NEEDED.log fi - done + fi PORTAGE_QUIET=${tmp_quiet} fi diff --git a/man/ebuild.5 b/man/ebuild.5 index 90b94f0cc..7f4d4fa7f 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -510,6 +510,16 @@ LDFLAGS variable. This should contain a list of file paths, relative to the image directory, of files that contain pre-stripped binaries. The paths may contain regular expressions with escape\-quoted special characters. +.TP +\fBQA_SONAME\fR +This should contain a list of file paths, relative to the image directory, of +shared libraries that lack SONAMEs. The paths may contain regular expressions +with escape\-quoted special characters. +.TP +\fBQA_DT_NEEDED\fR +This should contain a list of file paths, relative to the image directory, of +shared libraries that lack NEEDED entries. The paths may contain regular +expressions with escape\-quoted special characters. .SH "PORTAGE DECLARATIONS" .TP .B inherit diff --git a/man/portage.5 b/man/portage.5 index 6cc46706a..fe457b79f 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -152,8 +152,8 @@ as if it were a single file. .I Example: .nf -/usr/portage/profiles/package.mask/removals -/usr/portage/profiles/package.mask/testing +${PORTDIR}/profiles/package.mask/removals +${PORTDIR}/profiles/package.mask/testing .fi .RS .TP diff --git a/pym/_emerge/JobStatusDisplay.py b/pym/_emerge/JobStatusDisplay.py index 6aa2d99b7..98724e8a7 100644 --- a/pym/_emerge/JobStatusDisplay.py +++ b/pym/_emerge/JobStatusDisplay.py @@ -13,6 +13,7 @@ except ImportError: import portage from portage import os +from portage import _encodings from portage.output import xtermTitle from _emerge.getloadavg import getloadavg @@ -70,7 +71,7 @@ class JobStatusDisplay(object): def _write(self, s): if sys.hexversion < 0x3000000 and isinstance(s, unicode): # avoid potential UnicodeEncodeError - s = portage._unicode_encode(s) + s = s.encode(_encodings['stdio'], 'backslashreplace') self.out.write(s) self.out.flush() diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 87cc5a194..005da2afc 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -126,17 +126,13 @@ _encodings = { 'fs' : 'utf_8', 'merge' : sys.getfilesystemencoding(), 'repo.content' : 'utf_8', + 'stdio' : 'utf_8', } # This can happen if python is built with USE=build (stage 1). if _encodings['merge'] is None: _encodings['merge'] = 'ascii' -# Deprecated attributes. Instead use _encodings directly. -_content_encoding = _encodings['content'] -_fs_encoding = _encodings['fs'] -_merge_encoding = _encodings['merge'] - def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, unicode): diff --git a/pym/portage/_selinux.py b/pym/portage/_selinux.py index 9c0f08299..ca6ec4dec 100644 --- a/pym/portage/_selinux.py +++ b/pym/portage/_selinux.py @@ -7,8 +7,7 @@ import os import shutil -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_encode from portage.localization import _ @@ -16,8 +15,8 @@ import selinux from selinux import is_selinux_enabled def copyfile(src, dest): - src = _unicode_encode(src, encoding=_fs_encoding, errors='strict') - dest = _unicode_encode(dest, encoding=_fs_encoding, errors='strict') + src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') + dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: raise OSError(_("copyfile: Failed getting context of \"%s\".") % src) @@ -36,8 +35,8 @@ def getcontext(): return ctx def mkdir(target, refdir): - target = _unicode_encode(target, encoding=_fs_encoding, errors='strict') - refdir = _unicode_encode(refdir, encoding=_fs_encoding, errors='strict') + target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') + refdir = _unicode_encode(refdir, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.getfilecon(refdir) if rc < 0: raise OSError( @@ -51,8 +50,8 @@ def mkdir(target, refdir): selinux.setfscreatecon() def rename(src, dest): - src = _unicode_encode(src, encoding=_fs_encoding, errors='strict') - dest = _unicode_encode(dest, encoding=_fs_encoding, errors='strict') + src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') + dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: raise OSError(_("rename: Failed getting context of \"%s\".") % src) @@ -69,13 +68,13 @@ def settype(newtype): return ":".join(ret) def setexec(ctx="\n"): - ctx = _unicode_encode(ctx, encoding=_content_encoding, errors='strict') + ctx = _unicode_encode(ctx, encoding=_encodings['content'], errors='strict') if selinux.setexeccon(ctx) < 0: raise OSError(_("setexec: Failed setting exec() context \"%s\".") % ctx) def setfscreate(ctx="\n"): ctx = _unicode_encode(ctx, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') if selinux.setfscreatecon(ctx) < 0: raise OSError( _("setfscreate: Failed setting fs create context \"%s\".") % ctx) @@ -83,7 +82,7 @@ def setfscreate(ctx="\n"): def spawn_wrapper(spawn_func, selinux_type): selinux_type = _unicode_encode(selinux_type, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') def wrapper_func(*args, **kwargs): con = settype(selinux_type) @@ -96,9 +95,9 @@ def spawn_wrapper(spawn_func, selinux_type): return wrapper_func def symlink(target, link, reflnk): - target = _unicode_encode(target, encoding=_fs_encoding, errors='strict') - link = _unicode_encode(link, encoding=_fs_encoding, errors='strict') - reflnk = _unicode_encode(reflnk, encoding=_fs_encoding, errors='strict') + target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') + link = _unicode_encode(link, encoding=_encodings['fs'], errors='strict') + reflnk = _unicode_encode(reflnk, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(reflnk) if rc < 0: raise OSError( diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py index bcaf30640..4406b4e77 100644 --- a/pym/portage/cache/ebuild_xattr.py +++ b/pym/portage/cache/ebuild_xattr.py @@ -10,7 +10,7 @@ from portage.cache import fs_template from portage.versions import catsplit from portage import cpv_getkey from portage import os -from portage import _fs_encoding +from portage import _encoding from portage import _unicode_decode import xattr from errno import ENODATA,ENOSPC,E2BIG @@ -159,7 +159,7 @@ class database(fs_template.FsBased): for file in files: try: file = _unicode_decode(file, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue if file[-7:] == '.ebuild': diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index f871c49ae..0a67d2dc0 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -7,8 +7,7 @@ import portage from portage.const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE from portage.localization import _ from portage import os -from portage import _fs_encoding -from portage import _merge_encoding +from portage import _encodings from portage import _unicode_encode import errno import stat @@ -29,7 +28,7 @@ def _generate_hash_function(hashtype, hashobject, origin="unknown"): @return: The hash and size of the data """ f = open(_unicode_encode(filename, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') blocksize = HASHING_BLOCKSIZE data = f.read(blocksize) size = 0L @@ -123,7 +122,7 @@ def perform_md5(x, calc_prelink=0): def _perform_md5_merge(x, **kwargs): return perform_md5(_unicode_encode(x, - encoding=_merge_encoding, errors='strict'), **kwargs) + encoding=_encodings['merge'], errors='strict'), **kwargs) def perform_all(x, calc_prelink=0): mydict = {} @@ -221,7 +220,7 @@ def perform_checksum(filename, hashname="MD5", calc_prelink=0): # Make sure filename is encoded with the correct encoding before # it is passed to spawn (for prelink) and/or the hash function. filename = _unicode_encode(filename, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') myfilename = filename prelink_tmpfile = None try: diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 0ba85867a..8de25b7f6 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -22,6 +22,7 @@ from portage.localization import _ from portage import dep_expand, listdir, _check_distfile, _movefile from portage import os +from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode @@ -76,7 +77,8 @@ class bindbapi(fakedbapi): def getitem(k): v = tbz2.getfile(k) if v is not None: - v = _unicode_decode(v) + v = _unicode_decode(v, + encoding=_encodings['repo.content'], errors='replace') return v else: getitem = self.bintree._remotepkgs[mycpv].get @@ -111,8 +113,10 @@ class bindbapi(fakedbapi): mydata = mytbz2.get_data() for k, v in values.iteritems(): - k = _unicode_encode(k) - v = _unicode_encode(v) + k = _unicode_encode(k, + encoding=_encodings['repo.content'], errors='backslashreplace') + v = _unicode_encode(v, + encoding=_encodings['repo.content'], errors='backslashreplace') mydata[k] = v for k, v in mydata.items(): @@ -655,8 +659,10 @@ class binarytree(object): urldata[1] + urldata[2], "Packages") pkgindex = self._new_pkgindex() try: - f = codecs.open(_unicode_encode(pkgindex_file), - encoding='utf_8', errors='replace') + f = codecs.open(_unicode_encode(pkgindex_file, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') try: pkgindex.read(f) finally: @@ -1097,8 +1103,10 @@ class binarytree(object): def _load_pkgindex(self): pkgindex = self._new_pkgindex() try: - f = codecs.open(_unicode_encode(self._pkgindex_file), - encoding='utf_8', errors='replace') + f = codecs.open(_unicode_encode(self._pkgindex_file, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') except EnvironmentError: pass else: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 56ed44945..34e9d178e 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -26,8 +26,9 @@ from portage.manifest import Manifest from portage import eclass_cache, auxdbkeys, doebuild, flatten, \ listdir, dep_expand, eapi_is_supported, key_expand, dep_check, \ _eapi_is_deprecated -from portage import _unicode_encode from portage import os +from portage import _encodings +from portage import _unicode_encode import codecs import logging @@ -172,8 +173,10 @@ class portdbapi(dbapi): repo_name_path = os.path.join(path, REPO_NAME_LOC) try: repo_name = codecs.open( - _unicode_encode(repo_name_path), mode='r', - encoding='utf_8', errors='replace').readline().strip() + _unicode_encode(repo_name_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace').readline().strip() except EnvironmentError: # warn about missing repo_name at some other time, since we # don't want to see a warning every time the portage module is @@ -621,8 +624,10 @@ class portdbapi(dbapi): if eapi is None and \ 'parse-eapi-ebuild-head' in self.doebuild_settings.features: eapi = portage._parse_eapi_ebuild_head(codecs.open( - _unicode_encode(myebuild), mode='r', - encoding='utf_8', errors='replace')) + _unicode_encode(myebuild, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace')) if eapi is not None: self.doebuild_settings.configdict['pkg']['EAPI'] = eapi diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index f1ed5a569..192b137d3 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -38,9 +38,7 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ # This is a special version of the os module, wrapped for unicode support. from portage import os -from portage import _content_encoding -from portage import _fs_encoding -from portage import _merge_encoding +from portage import _encodings from portage import _os_merge from portage import _selinux_merge from portage import _unicode_decode @@ -81,7 +79,8 @@ class PreservedLibsRegistry(object): self._data = None try: self._data = pickle.load( - open(_unicode_encode(self._filename), 'rb')) + open(_unicode_encode(self._filename, + encoding=_encodings['fs'], errors='strict'), 'rb')) except (ValueError, pickle.UnpicklingError), e: writemsg_level(_("!!! Error loading '%s': %s\n") % \ (self._filename, e), level=logging.ERROR, noiselevel=-1) @@ -328,7 +327,15 @@ class LinkageMap(object): raise CommandNotFound(args[0]) else: for l in proc.stdout: - l = portage._unicode_decode(l) + try: + l = _unicode_decode(l, + encoding=_encodings['content'], errors='strict') + except UnicodeDecodeError: + l = _unicode_decode(l, + encoding=_encodings['content'], errors='replace') + writemsg_level(_("\nError decoding characters " \ + "returned from scanelf: %s\n\n") % (l,), + level=logging.ERROR, noiselevel=-1) l = l[3:].rstrip("\n") if not l: continue @@ -1455,7 +1462,8 @@ class vardbapi(dbapi): now has a categories property that is generated from the available packages. """ - self.root = portage._unicode_decode(root) + self.root = _unicode_decode(root, + encoding=_encodings['content'], errors='strict') #cache for category directory mtimes self.mtdircache = {} @@ -1539,7 +1547,8 @@ class vardbapi(dbapi): except KeyError: continue h.update(_unicode_encode(counter, - encoding=_content_encoding, errors='replace')) + encoding=_encodings['repo.content'], + errors='backslashreplace')) return h.hexdigest() def cpv_inject(self, mycpv): @@ -1804,7 +1813,8 @@ class vardbapi(dbapi): # python-2.x, but buffering makes it much worse. open_kwargs["buffering"] = 0 try: - f = open(_unicode_encode(self._aux_cache_filename), + f = open(_unicode_encode(self._aux_cache_filename, + encoding=_encodings['fs'], errors='strict'), mode='rb', **open_kwargs) mypickle = pickle.Unpickler(f) try: @@ -1906,7 +1916,8 @@ class vardbapi(dbapi): if cache_valid: # Migrate old metadata to unicode. for k, v in metadata.iteritems(): - metadata[k] = portage._unicode_decode(v) + metadata[k] = _unicode_decode(v, + encoding=_encodings['repo.content'], errors='replace') mydata.update(metadata) pull_me.difference_update(mydata) @@ -1953,8 +1964,9 @@ class vardbapi(dbapi): try: myf = codecs.open( _unicode_encode(os.path.join(mydir, x), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') try: myd = myf.read() finally: @@ -2024,8 +2036,9 @@ class vardbapi(dbapi): try: cfile = codecs.open( _unicode_encode(self._counter_path, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') except EnvironmentError, e: new_vdb = not bool(self.cpv_all()) if not new_vdb: @@ -2092,7 +2105,8 @@ class vardbapi(dbapi): removed = 0 for filename in paths: - filename = portage._unicode_decode(filename) + filename = _unicode_decode(filename, + encoding=_encodings['content'], errors='strict') filename = normalize_path(filename) if relative_paths: relative_filename = filename @@ -2161,7 +2175,8 @@ class vardbapi(dbapi): # Always use a constant utf_8 encoding here, since # the "default" encoding can change. h.update(_unicode_encode(s, - encoding=_content_encoding, errors='replace')) + encoding=_encodings['repo.content'], + errors='backslashreplace')) h = h.hexdigest() h = h[-self._hex_chars:] h = int(h, 16) @@ -2610,8 +2625,9 @@ class dblink(object): pkgfiles = {} try: myc = codecs.open(_unicode_encode(contents_file, - encoding=_content_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') except EnvironmentError, e: if e.errno != errno.ENOENT: raise @@ -3087,13 +3103,15 @@ class dblink(object): obj = normalize_path(objkey) if os is _os_merge: try: - _unicode_encode(obj, encoding=_merge_encoding, errors='strict') + _unicode_encode(obj, + encoding=_encodings['merge'], errors='strict') except UnicodeEncodeError: # The package appears to have been merged with a # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: - _unicode_encode(obj, encoding=_fs_encoding, errors='strict') + _unicode_encode(obj, + encoding=_encodings['fs'], errors='strict') except UnicodeEncodeError: pass else: @@ -3290,9 +3308,11 @@ class dblink(object): os = _os_merge - filename = portage._unicode_decode(filename) + filename = _unicode_decode(filename, + encoding=_encodings['content'], errors='strict') - destroot = portage._unicode_decode(destroot) + destroot = _unicode_decode(destroot, + encoding=_encodings['content'], errors='strict') destfile = normalize_path( os.path.join(destroot, filename.lstrip(os.path.sep))) @@ -3484,7 +3504,8 @@ class dblink(object): new_contents = self.getcontents().copy() old_contents = self._installed_instance.getcontents() for f in sorted(preserve_paths): - f = portage._unicode_decode(f, encoding=_merge_encoding) + f = _unicode_decode(f, + encoding=_encodings['content'], errors='strict') f_abs = os.path.join(root, f.lstrip(os.sep)) contents_entry = old_contents.get(f_abs) if contents_entry is None: @@ -3926,10 +3947,14 @@ class dblink(object): os = _os_merge - srcroot = portage._unicode_decode(srcroot) - destroot = portage._unicode_decode(destroot) - inforoot = portage._unicode_decode(inforoot) - myebuild = portage._unicode_decode(myebuild) + srcroot = _unicode_decode(srcroot, + encoding=_encodings['content'], errors='strict') + destroot = _unicode_decode(destroot, + encoding=_encodings['content'], errors='strict') + inforoot = _unicode_decode(inforoot, + encoding=_encodings['content'], errors='strict') + myebuild = _unicode_decode(myebuild, + encoding=_encodings['content'], errors='strict') showMessage = self._display_merge scheduler = self._scheduler @@ -3947,9 +3972,9 @@ class dblink(object): try: val = codecs.open(_unicode_encode( os.path.join(inforoot, var_name), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace' - ).readline().strip() + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace').readline().strip() except EnvironmentError, e: if e.errno != errno.ENOENT: raise @@ -4037,14 +4062,14 @@ class dblink(object): for parent, dirs, files in os.walk(srcroot, onerror=onerror): try: parent = _unicode_decode(parent, - encoding=_merge_encoding, errors='strict') + encoding=_encodings['merge'], errors='strict') except UnicodeDecodeError: new_parent = _unicode_decode(parent, - encoding=_merge_encoding, errors='replace') + encoding=_encodings['merge'], errors='replace') new_parent = _unicode_encode(new_parent, - encoding=_merge_encoding, errors='backslashreplace') + encoding=_encodings['merge'], errors='backslashreplace') new_parent = _unicode_decode(new_parent, - encoding=_merge_encoding, errors='replace') + encoding=_encodings['merge'], errors='replace') os.rename(parent, new_parent) unicode_error = True unicode_errors.append(new_parent[srcroot_len:]) @@ -4053,16 +4078,16 @@ class dblink(object): for fname in files: try: fname = _unicode_decode(fname, - encoding=_merge_encoding, errors='strict') + encoding=_encodings['merge'], errors='strict') except UnicodeDecodeError: fpath = portage._os.path.join( - parent.encode(_merge_encoding), fname) + parent.encode(_encodings['merge']), fname) new_fname = _unicode_decode(fname, - encoding=_merge_encoding, errors='replace') + encoding=_encodings['merge'], errors='replace') new_fname = _unicode_encode(new_fname, - encoding=_merge_encoding, errors='backslashreplace') + encoding=_encodings['merge'], errors='backslashreplace') new_fname = _unicode_decode(new_fname, - encoding=_merge_encoding, errors='replace') + encoding=_encodings['merge'], errors='replace') new_fpath = os.path.join(parent, new_fname) os.rename(fpath, new_fpath) unicode_error = True @@ -4290,14 +4315,17 @@ class dblink(object): # write local package counter for recording counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) - open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER')), - 'w').write(str(counter)) + codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), + encoding=_encodings['fs'], errors='strict'), + 'w', encoding=_encodings['repo.content'], errors='backslashreplace' + ).write(str(counter)) # open CONTENTS file (possibly overwriting old one) for recording outfile = codecs.open(_unicode_encode( os.path.join(self.dbtmpdir, 'CONTENTS'), - encoding=_fs_encoding, errors='strict'), - mode='w', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='backslashreplace') self.updateprotect() @@ -4647,7 +4675,7 @@ class dblink(object): # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, - encoding=_merge_encoding) + encoding=_encodings['merge']) if mymtime != None: showMessage(">>> %s -> %s\n" % (mydest, myto)) outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") @@ -4687,7 +4715,7 @@ class dblink(object): # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. if movefile(mydest, mydest+".backup", mysettings=self.settings, - encoding=_merge_encoding) is None: + encoding=_encodings['merge']) is None: return 1 showMessage(_("bak %s %s.backup\n") % (mydest, mydest), level=logging.ERROR, noiselevel=-1) @@ -4792,7 +4820,7 @@ class dblink(object): mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, hardlink_candidates=hardlink_candidates, - encoding=_merge_encoding) + encoding=_encodings['merge']) if mymtime is None: return 1 if hardlink_candidates is not None: @@ -4809,7 +4837,7 @@ class dblink(object): # destination doesn't exist if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, - encoding=_merge_encoding) is not None: + encoding=_encodings['merge']) is not None: zing = ">>>" else: return 1 @@ -4897,8 +4925,8 @@ class dblink(object): return "" mydata = codecs.open( _unicode_encode(os.path.join(self.dbdir, name), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace' + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace' ).read().split() return " ".join(mydata) @@ -4909,22 +4937,27 @@ class dblink(object): if not os.path.exists(self.dbdir+"/"+fname): return "" return codecs.open(_unicode_encode(os.path.join(self.dbdir, fname), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace').read() + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace' + ).read() def setfile(self,fname,data): - mode = 'w' + kwargs = {} if fname == 'environment.bz2' or not isinstance(data, basestring): - mode = 'wb' - write_atomic(os.path.join(self.dbdir, fname), data, mode=mode) + kwargs['mode'] = 'wb' + else: + kwargs['mode'] = 'w' + kwargs['encoding'] = _encodings['repo.content'] + write_atomic(os.path.join(self.dbdir, fname), data, **kwargs) def getelements(self,ename): if not os.path.exists(self.dbdir+"/"+ename): return [] mylines = codecs.open(_unicode_encode( os.path.join(self.dbdir, ename), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace').readlines() + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace' + ).readlines() myreturn = [] for x in mylines: for y in x[:-1].split(): @@ -4934,8 +4967,9 @@ class dblink(object): def setelements(self,mylist,ename): myelement = codecs.open(_unicode_encode( os.path.join(self.dbdir, ename), - encoding=_fs_encoding, errors='strict'), mode='w', - encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='backslashreplace') for x in mylist: myelement.write(x+"\n") myelement.close() @@ -5013,7 +5047,8 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tarinfo.size = 0 tar.addfile(tarinfo) else: - f = open(_unicode_encode(path, encoding=_merge_encoding), 'rb') + f = open(_unicode_encode(path, + encoding=_encodings['merge'], errors='strict'), 'rb') try: tar.addfile(tarinfo, f) finally: diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py index a51f0864e..a563ad271 100644 --- a/pym/portage/elog/messages.py +++ b/pym/portage/elog/messages.py @@ -12,6 +12,9 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage.const import EBUILD_PHASES from portage.localization import _ from portage import os +from portage import _encodings +from portage import _unicode_encode +from portage import _unicode_decode import codecs import sys @@ -41,8 +44,9 @@ def collect_ebuild_messages(path): logentries[msgfunction] = [] lastmsgtype = None msgcontent = [] - for l in codecs.open(filename, mode='r', - encoding='utf_8', errors='replace'): + for l in codecs.open(_unicode_encode(filename, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace'): if not l: continue try: @@ -87,15 +91,16 @@ def _elog_base(level, msg, phase="other", key=None, color=None, out=None): if color is None: color = "GOOD" - if not isinstance(msg, unicode): - msg = unicode(msg, encoding='utf_8', errors='replace') + msg = _unicode_decode(msg, + encoding=_encodings['content'], errors='replace') formatted_msg = colorize(color, " * ") + msg + "\n" if sys.hexversion < 0x3000000 and \ out in (sys.stdout, sys.stderr) and isinstance(formatted_msg, unicode): # avoid potential UnicodeEncodeError - formatted_msg = formatted_msg.encode('utf_8', 'replace') + formatted_msg = formatted_msg.encode( + _encodings['stdio'], 'backslashreplace') out.write(formatted_msg) diff --git a/pym/portage/elog/mod_save.py b/pym/portage/elog/mod_save.py index 2d1c1614d..ca76e23be 100644 --- a/pym/portage/elog/mod_save.py +++ b/pym/portage/elog/mod_save.py @@ -6,6 +6,8 @@ import codecs import time from portage import os +from portage import _encodings +from portage import _unicode_encode from portage.data import portage_uid, portage_gid from portage.util import ensure_dirs from portage.const import EPREFIX @@ -20,8 +22,9 @@ def process(mysettings, key, logentries, fulltext): ensure_dirs(elogdir, uid=portage_uid, gid=portage_gid, mode=02770) elogfilename = elogdir+"/"+path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log" - elogfile = codecs.open(elogfilename, mode='w', - encoding='utf_8', errors='replace') + elogfile = codecs.open(_unicode_encode(elogfilename, + encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['content'], errors='backslashreplace') elogfile.write(fulltext) elogfile.close() diff --git a/pym/portage/elog/mod_save_summary.py b/pym/portage/elog/mod_save_summary.py index e1c41e03f..755644253 100644 --- a/pym/portage/elog/mod_save_summary.py +++ b/pym/portage/elog/mod_save_summary.py @@ -6,6 +6,8 @@ import codecs import time from portage import os +from portage import _encodings +from portage import _unicode_encode from portage.data import portage_uid, portage_gid from portage.localization import _ from portage.util import ensure_dirs, apply_permissions @@ -20,8 +22,9 @@ def process(mysettings, key, logentries, fulltext): # TODO: Locking elogfilename = elogdir+"/summary.log" - elogfile = codecs.open(elogfilename, mode='a', - encoding='utf_8', errors='replace') + elogfile = codecs.open(_unicode_encode(elogfilename, + encoding=_encodings['fs'], errors='strict'), + mode='a', encoding=_encodings['content'], errors='backslashreplace') apply_permissions(elogfilename, mode=060, mask=0) elogfile.write(_(">>> Messages generated by process %(pid)d on %(time)s for package %(pkg)s:\n\n") % {"pid": os.getpid(), "time": time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime(time.time())), "pkg": key}) diff --git a/pym/portage/elog/mod_syslog.py b/pym/portage/elog/mod_syslog.py index 0fe205644..d7e955f81 100644 --- a/pym/portage/elog/mod_syslog.py +++ b/pym/portage/elog/mod_syslog.py @@ -6,6 +6,7 @@ import sys import syslog from portage.const import EBUILD_PHASES +from portage import _encodings _pri = { "INFO" : syslog.LOG_INFO, @@ -25,6 +26,7 @@ def process(mysettings, key, logentries, fulltext): msgtext = "%s: %s: %s" % (key, phase, msgtext) if sys.hexversion < 0x3000000 and isinstance(msgtext, unicode): # Avoid TypeError from syslog.syslog() - msgtext = msgtext.encode('utf_8', 'replace') + msgtext = msgtext.encode(_encodings['content'], + 'backslashreplace') syslog.syslog(_pri[msgtype], msgtext) syslog.closelog() diff --git a/pym/portage/env/loaders.py b/pym/portage/env/loaders.py index e878ba449..5ec3a066f 100644 --- a/pym/portage/env/loaders.py +++ b/pym/portage/env/loaders.py @@ -7,8 +7,7 @@ import codecs import errno import stat from portage import os -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode from portage.localization import _ @@ -57,7 +56,7 @@ def RecursiveFileLoader(filename): for f in files: try: f = _unicode_decode(f, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue if f[:1] == '.' or f[-1:] == '~': @@ -152,8 +151,8 @@ class FileLoader(DataLoader): for fn in RecursiveFileLoader(self.fname): try: f = codecs.open(_unicode_encode(fn, - encoding=_fs_encoding, errors='strict'), mode='r', - encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), mode='r', + encoding=_encodings['content'], errors='replace') except EnvironmentError, e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise diff --git a/pym/portage/mail.py b/pym/portage/mail.py index ce2f6760d..260455739 100644 --- a/pym/portage/mail.py +++ b/pym/portage/mail.py @@ -13,7 +13,7 @@ import sys import time from portage import os -from portage import _content_encoding +from portage import _encodings from portage import _unicode_encode from portage.localization import _ import portage @@ -22,13 +22,13 @@ def create_message(sender, recipient, subject, body, attachments=None): if sys.hexversion < 0x3000000: sender = _unicode_encode(sender, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') recipient = _unicode_encode(recipient, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') subject = _unicode_encode(subject, - encoding=_content_encoding, errors='replace') + encoding=_encodings['content'], errors='backslashreplace') body = _unicode_encode(body, - encoding=_content_encoding, errors='replace') + encoding=_encodings['content'], errors='backslashreplace') if attachments == None: mymessage = TextMessage(body) @@ -41,7 +41,8 @@ def create_message(sender, recipient, subject, body, attachments=None): elif isinstance(x, basestring): if sys.hexversion < 0x3000000: x = _unicode_encode(x, - encoding=_content_encoding, errors='replace') + encoding=_encodings['content'], + errors='backslashreplace') mymessage.attach(TextMessage(x)) else: raise portage.exception.PortageException(_("Can't handle type of attachment: %s") % type(x)) @@ -92,17 +93,17 @@ def send_mail(mysettings, message): if sys.hexversion < 0x3000000: myrecipient = _unicode_encode(myrecipient, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') mymailhost = _unicode_encode(mymailhost, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') mymailport = _unicode_encode(mymailport, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') myfrom = _unicode_encode(myfrom, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') mymailuser = _unicode_encode(mymailuser, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') mymailpasswd = _unicode_encode(mymailpasswd, - encoding=_content_encoding, errors='strict') + encoding=_encodings['content'], errors='strict') # user wants to use a sendmail binary instead of smtp if mymailhost[0] == os.sep and os.path.exists(mymailhost): diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 4c4cb60b3..25893d759 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -12,8 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage import os -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode from portage.exception import DigestException, FileNotFound, \ @@ -145,8 +144,8 @@ class Manifest(object): Otherwise, a new dict will be created and returned.""" try: fd = codecs.open(_unicode_encode(file_path, - encoding=_fs_encoding, errors='strict'), mode='r', - encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), mode='r', + encoding=_encodings['repo.content'], errors='replace') if myhashdict is None: myhashdict = {} self._parseDigests(fd, myhashdict=myhashdict, **kwargs) @@ -233,8 +232,9 @@ class Manifest(object): if not force: try: f = codecs.open(_unicode_encode(self.getFullname(), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') oldentries = list(self._parseManifestLines(f)) f.close() if len(oldentries) == len(myentries): @@ -327,7 +327,7 @@ class Manifest(object): for f in pkgdir_files: try: f = _unicode_decode(f, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue if f[:1] == ".": @@ -362,7 +362,7 @@ class Manifest(object): for f in files: try: f = _unicode_decode(f, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue full_path = os.path.join(parentdir, f) @@ -523,8 +523,8 @@ class Manifest(object): if not os.path.exists(mfname): return rVal myfile = codecs.open(_unicode_encode(mfname, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace') lines = myfile.readlines() myfile.close() for l in lines: diff --git a/pym/portage/news.py b/pym/portage/news.py index f3482150d..ca608d722 100644 --- a/pym/portage/news.py +++ b/pym/portage/news.py @@ -12,8 +12,7 @@ import logging import os as _os import re from portage import os -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode from portage.util import apply_secpass_permissions, ensure_dirs, \ @@ -100,7 +99,7 @@ class NewsManager(object): news_dir = self._news_dir(repoid) try: news = _os.listdir(_unicode_encode(news_dir, - encoding=_fs_encoding, errors='strict')) + encoding=_encodings['fs'], errors='strict')) except OSError: return @@ -120,10 +119,10 @@ class NewsManager(object): for itemid in news: try: itemid = _unicode_decode(itemid, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: itemid = _unicode_decode(itemid, - encoding=_fs_encoding, errors='replace') + encoding=_encodings['fs'], errors='replace') writemsg_level( "!!! Invalid encoding in news item name: '%s'\n" % \ itemid, level=logging.ERROR, noiselevel=-1) @@ -253,8 +252,9 @@ class NewsItem(object): def parse(self): lines = codecs.open(_unicode_encode(self.path, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace').readlines() + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace' + ).readlines() self.restrictions = {} invalids = [] for i, line in enumerate(lines): diff --git a/pym/portage/output.py b/pym/portage/output.py index c47b65cc0..c0839fa53 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -17,8 +17,7 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage import os -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_encode from portage.const import COLOR_MAP_FILE, EPREFIX from portage.exception import CommandNotFound, FileNotFound, \ @@ -169,8 +168,8 @@ def _parse_color_map(config_root='/', onerror=None): try: lineno=0 for line in codecs.open(_unicode_encode(myfile, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace'): + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace'): lineno += 1 commenter_pos = line.find("#") @@ -470,7 +469,7 @@ class EOutput(object): def _write(self, f, s): if sys.hexversion < 0x3000000 and isinstance(s, unicode): # avoid potential UnicodeEncodeError - s = s.encode(_content_encoding, 'replace') + s = s.encode(_encodings['stdio'], 'backslashreplace') f.write(s) f.flush() diff --git a/pym/portage/process.py b/pym/portage/process.py index 1b5846883..1822a3082 100644 --- a/pym/portage/process.py +++ b/pym/portage/process.py @@ -10,7 +10,7 @@ import sys import traceback from portage import os -from portage import _content_encoding +from portage import _encodings from portage import _unicode_encode import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -184,8 +184,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.iteritems(): - env_bytes[_unicode_encode(k, encoding=_content_encoding)] = \ - _unicode_encode(v, encoding=_content_encoding) + env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ + _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index d39087e1e..62d7f0757 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -6,7 +6,7 @@ import re from itertools import chain from portage import os -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path @@ -130,16 +130,16 @@ class StaticFileSet(EditablePackageSet): try: directory = _unicode_decode(directory, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') # Now verify that we can also encode it. _unicode_encode(directory, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeError: directory = _unicode_decode(directory, - encoding=_fs_encoding, errors='replace') + encoding=_encodings['fs'], errors='replace') raise SetConfigError( _("Directory path contains invalid character(s) for encoding '%s': '%s'") \ - % (_fs_encoding, directory)) + % (_encodings['fs'], directory)) if os.path.isdir(directory): directory = normalize_path(directory) @@ -147,7 +147,7 @@ class StaticFileSet(EditablePackageSet): for parent, dirs, files in os.walk(directory): try: parent = _unicode_decode(parent, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue for d in dirs[:]: @@ -156,7 +156,7 @@ class StaticFileSet(EditablePackageSet): for filename in files: try: filename = _unicode_decode(filename, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue if filename[:1] == '.': diff --git a/pym/portage/tests/__init__.py b/pym/portage/tests/__init__.py index 4a5ced8a3..6e7380409 100644 --- a/pym/portage/tests/__init__.py +++ b/pym/portage/tests/__init__.py @@ -8,16 +8,16 @@ import time import unittest from portage import os -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_encode from portage import _unicode_decode def main(): TEST_FILE = _unicode_encode('__test__', - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') svn_dirname = _unicode_encode('.svn', - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') suite = unittest.TestSuite() basedir = os.path.dirname(os.path.realpath(__file__)) testDirs = [] @@ -30,7 +30,7 @@ def main(): dirs.remove(svn_dirname) try: root = _unicode_decode(root, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue diff --git a/pym/portage/tests/ebuild/test_spawn.py b/pym/portage/tests/ebuild/test_spawn.py index 908fce606..97a5f42d0 100644 --- a/pym/portage/tests/ebuild/test_spawn.py +++ b/pym/portage/tests/ebuild/test_spawn.py @@ -6,8 +6,7 @@ import codecs import errno import sys from portage import os -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_encode from portage.tests import TestCase @@ -34,8 +33,8 @@ class SpawnTestCase(TestCase): free=1, fd_pipes={0:sys.stdin.fileno(), 1:null_fd, 2:null_fd}) os.close(null_fd) f = codecs.open(_unicode_encode(logfile, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='strict') log_content = f.read() f.close() # When logging passes through a pty, this comparison will fail diff --git a/pym/portage/update.py b/pym/portage/update.py index d38ddf942..ca67370cd 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -8,8 +8,7 @@ import re import sys from portage import os -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode import portage @@ -72,8 +71,8 @@ def fixdbentries(update_iter, dbdir): for myfile in [f for f in os.listdir(dbdir) if f not in ignored_dbentries]: file_path = os.path.join(dbdir, myfile) mydata[myfile] = codecs.open(_unicode_encode(file_path, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace').read() updated_items = update_dbentries(update_iter, mydata) for myfile, mycontent in updated_items.iteritems(): @@ -110,8 +109,9 @@ def grab_updates(updpath, prev_mtimes=None): if file_path not in prev_mtimes or \ long(prev_mtimes[file_path]) != long(mystat.st_mtime): content = codecs.open(_unicode_encode(file_path, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace').read() + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace' + ).read() update_data.append((file_path, mystat, content)) return update_data @@ -172,7 +172,7 @@ def update_config_files(config_root, protect, protect_mask, update_iter): for y in dirs: try: y = _unicode_decode(y, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: dirs.remove(y) continue @@ -181,7 +181,7 @@ def update_config_files(config_root, protect, protect_mask, update_iter): for y in files: try: y = _unicode_decode(y, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue if y.startswith("."): @@ -195,8 +195,8 @@ def update_config_files(config_root, protect, protect_mask, update_iter): try: file_contents[x] = codecs.open( _unicode_encode(os.path.join(abs_user_config, x), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace').readlines() except IOError: if file_contents.has_key(x): diff --git a/pym/portage/util.py b/pym/portage/util.py index 983c08b0a..685843cac 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -23,8 +23,7 @@ import sys import portage from portage import os -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _os_merge from portage import _unicode_encode from portage import _unicode_decode @@ -65,7 +64,8 @@ def writemsg(mystr,noiselevel=0,fd=None): if noiselevel <= noiselimit: if sys.hexversion < 0x3000000: # avoid potential UnicodeEncodeError - mystr = _unicode_encode(mystr) + mystr = _unicode_encode(mystr, + encoding=_encodings['stdio'], errors='backslashreplace') fd.write(mystr) fd.flush() @@ -329,8 +329,8 @@ def grablines(myfilename,recursive=0): else: try: myfile = codecs.open(_unicode_encode(myfilename, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace') mylines = myfile.readlines() myfile.close() except IOError, e: @@ -397,11 +397,11 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): # (produces spurious \0 characters with python-2.6.2) if sys.hexversion < 0x3000000: content = open(_unicode_encode(mycfg, - encoding=_fs_encoding, errors='strict'), 'rb').read() + encoding=_encodings['fs'], errors='strict'), 'rb').read() else: content = open(_unicode_encode(mycfg, - encoding=_fs_encoding, errors='strict'), mode='r', - encoding=_content_encoding, errors='replace').read() + encoding=_encodings['fs'], errors='strict'), mode='r', + encoding=_encodings['content'], errors='replace').read() if content and content[-1] != '\n': content += '\n' except IOError, e: @@ -594,7 +594,7 @@ def pickle_read(filename,default=None,debug=0): data = None try: myf = open(_unicode_encode(filename, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') mypickle = pickle.Unpickler(myf) data = mypickle.load() myf.close() @@ -909,8 +909,8 @@ class atomic_ofstream(ObjectProxy): open_func = open else: open_func = codecs.open - kargs.setdefault('encoding', _content_encoding) - kargs.setdefault('errors', 'replace') + kargs.setdefault('encoding', _encodings['content']) + kargs.setdefault('errors', 'backslashreplace') if follow_links: canonical_path = os.path.realpath(filename) @@ -919,7 +919,7 @@ class atomic_ofstream(ObjectProxy): try: object.__setattr__(self, '_file', open_func(_unicode_encode(tmp_name, - encoding=_fs_encoding, errors='strict'), + encoding=_encodings['fs'], errors='strict'), mode=mode, **kargs)) return except IOError, e: @@ -933,7 +933,7 @@ class atomic_ofstream(ObjectProxy): tmp_name = "%s.%i" % (filename, os.getpid()) object.__setattr__(self, '_file', open_func(_unicode_encode(tmp_name, - encoding=_fs_encoding, errors='strict'), + encoding=_encodings['fs'], errors='strict'), mode=mode, **kargs)) def _get_target(self): diff --git a/pym/portage/xpak.py b/pym/portage/xpak.py index da68af59c..5b08c0a3f 100644 --- a/pym/portage/xpak.py +++ b/pym/portage/xpak.py @@ -22,7 +22,7 @@ import shutil from portage import os from portage import normalize_path -from portage import _fs_encoding +from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode @@ -30,23 +30,24 @@ def addtolist(mylist, curdir): """(list, dir) --- Takes an array(list) and appends all files from dir down the directory tree. Returns nothing. list is modified.""" curdir = normalize_path(_unicode_decode(curdir, - encoding=_fs_encoding, errors='strict')) + encoding=_encodings['fs'], errors='strict')) for parent, dirs, files in os.walk(curdir): parent = _unicode_decode(parent, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') if parent != curdir: mylist.append(parent[len(curdir) + 1:] + os.sep) for x in dirs: try: - _unicode_decode(x, encoding=_fs_encoding, errors='strict') + _unicode_decode(x, encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: dirs.remove(x) for x in files: try: - x = _unicode_decode(x, encoding=_fs_encoding, errors='strict') + x = _unicode_decode(x, + encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue mylist.append(os.path.join(parent, x)[len(curdir) + 1:]) @@ -82,13 +83,13 @@ def xpak(rootdir,outfile=None): mylist.sort() mydata = {} for x in mylist: - x = _unicode_encode(x, encoding=_fs_encoding, errors='strict') + x = _unicode_encode(x, encoding=_encodings['fs'], errors='strict') mydata[x] = open(os.path.join(rootdir, x), 'rb').read() xpak_segment = xpak_mem(mydata) if outfile: outf = open(_unicode_encode(outfile, - encoding=_fs_encoding, errors='strict'), 'wb') + encoding=_encodings['fs'], errors='strict'), 'wb') outf.write(xpak_segment) outf.close() else: @@ -118,9 +119,9 @@ def xsplit(infile): 'infile.index' contains the index segment. 'infile.dat' contails the data segment.""" infile = _unicode_decode(infile, - encoding=_fs_encoding, errors='strict') + encoding=_encodings['fs'], errors='strict') myfile = open(_unicode_encode(infile, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') mydat=myfile.read() myfile.close() @@ -129,11 +130,11 @@ def xsplit(infile): return False myfile = open(_unicode_encode(infile + '.index', - encoding=_fs_encoding, errors='strict'), 'wb') + encoding=_encodings['fs'], errors='strict'), 'wb') myfile.write(splits[0]) myfile.close() myfile = open(_unicode_encode(infile + '.dat', - encoding=_fs_encoding, errors='strict'), 'wb') + encoding=_encodings['fs'], errors='strict'), 'wb') myfile.write(splits[1]) myfile.close() return True @@ -149,7 +150,7 @@ def xsplit_mem(mydat): def getindex(infile): """(infile) -- grabs the index segment from the infile and returns it.""" myfile = open(_unicode_encode(infile, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') myheader=myfile.read(16) if myheader[0:8] != _unicode_encode('XPAKPACK'): myfile.close() @@ -163,7 +164,7 @@ def getboth(infile): """(infile) -- grabs the index and data segments from the infile. Returns an array [indexSegment,dataSegment]""" myfile = open(_unicode_encode(infile, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') myheader=myfile.read(16) if myheader[0:8] != _unicode_encode('XPAKPACK'): myfile.close() @@ -238,7 +239,7 @@ def xpand(myid,mydest): if not os.path.exists(dirname): os.makedirs(dirname) mydat = open(_unicode_encode(myname, - encoding=_fs_encoding, errors='strict'), 'wb') + encoding=_encodings['fs'], errors='strict'), 'wb') mydat.write(mydata[datapos:datapos+datalen]) mydat.close() startpos=startpos+namelen+12 @@ -284,7 +285,7 @@ class tbz2(object): def recompose_mem(self, xpdata): self.scan() # Don't care about condition... We'll rewrite the data anyway. myfile = open(_unicode_encode(self.file, - encoding=_fs_encoding, errors='strict'), 'ab+') + encoding=_encodings['fs'], errors='strict'), 'ab+') if not myfile: raise IOError myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF. @@ -322,7 +323,7 @@ class tbz2(object): return 1 self.filestat=mystat a = open(_unicode_encode(self.file, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') a.seek(-16,2) trailer=a.read() self.infosize=0 @@ -366,7 +367,7 @@ class tbz2(object): if not myresult: return mydefault a = open(_unicode_encode(self.file, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') a.seek(self.datapos+myresult[0],0) myreturn=a.read(myresult[1]) a.close() @@ -391,7 +392,7 @@ class tbz2(object): os.chdir("/") origdir="/" a = open(_unicode_encode(self.file, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') if not os.path.exists(mydest): os.makedirs(mydest) os.chdir(mydest) @@ -406,7 +407,7 @@ class tbz2(object): if not os.path.exists(dirname): os.makedirs(dirname) mydat = open(_unicode_encode(myname, - encoding=_fs_encoding, errors='strict'), 'wb') + encoding=_encodings['fs'], errors='strict'), 'wb') a.seek(self.datapos+datapos) mydat.write(a.read(datalen)) mydat.close() @@ -420,7 +421,7 @@ class tbz2(object): if not self.scan(): return 0 a = open(_unicode_encode(self.file, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') mydata = {} startpos=0 while ((startpos+8)