Merged from trunk -r14107:14117
authorFabian Groffen <grobian@gentoo.org>
Mon, 24 Aug 2009 09:38:13 +0000 (09:38 -0000)
committerFabian Groffen <grobian@gentoo.org>
Mon, 24 Aug 2009 09:38:13 +0000 (09:38 -0000)
   | 14108    | Use _encodings where appropriate.                           |
   | zmedico  |                                                             |

   | 14109    | Use _encodings where appropriate.                           |
   | zmedico  |                                                             |

   | 14110    | Use _encodings where appropriate.                           |
   | zmedico  |                                                             |

   | 14111    | Use _encodings where appropriate and add                    |
   | zmedico  | _encodings['stdio'] for stdout encoding.                    |

   | 14112    | Fix typo.                                                   |
   | zmedico  |                                                             |

   | 14113    | Replace _content_encoding, _fs_encoding, and                |
   | zmedico  | _merge_encoding with direct usage of _encodings.            |

   | 14114    | Support QA_DT_HASH_${ARCH} and QA_PRESTRIPPED_${ARCH} (bug  |
   | arfrever | #271416).                                                   |

   | 14115    | Add support for QA_SONAME variable (bug #281964). Add       |
   | arfrever | support for QA_NEEDED variable.                             |

   | 14116    | Improve an example. Patch by Jeremy Olexa.                  |
   | arfrever |                                                             |

   | 14117    | Rename QA_NEEDED to QA_DT_NEEDED.                           |
   | arfrever |                                                             |

svn path=/main/branches/prefix/; revision=14143

29 files changed:
bin/ebuild-helpers/prepstrip
bin/ebuild.sh
bin/misc-functions.sh
man/ebuild.5
man/portage.5
pym/_emerge/JobStatusDisplay.py
pym/portage/__init__.py
pym/portage/_selinux.py
pym/portage/cache/ebuild_xattr.py
pym/portage/checksum.py
pym/portage/dbapi/bintree.py
pym/portage/dbapi/porttree.py
pym/portage/dbapi/vartree.py
pym/portage/elog/messages.py
pym/portage/elog/mod_save.py
pym/portage/elog/mod_save_summary.py
pym/portage/elog/mod_syslog.py
pym/portage/env/loaders.py
pym/portage/mail.py
pym/portage/manifest.py
pym/portage/news.py
pym/portage/output.py
pym/portage/process.py
pym/portage/sets/files.py
pym/portage/tests/__init__.py
pym/portage/tests/ebuild/test_spawn.py
pym/portage/update.py
pym/portage/util.py
pym/portage/xpak.py

index 49c886cb8b7d2ac970adf6c0b0966c2860e993b5..5c0a9dc5fa473db0fd805181d3b344e27bb59ef4 100755 (executable)
@@ -100,6 +100,8 @@ save_elf_debug() {
 if ! hasq binchecks ${RESTRICT} && \
        ! hasq strip ${RESTRICT} ; then
        log=$T/scanelf-already-stripped.log
+       qa_var="QA_PRESTRIPPED_${ARCH/-/_}"
+       [[ -n ${!qa_var} ]] && QA_PRESTRIPPED="${!qa_var}"
        scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^$D##" > "$log"
        if [[ -n $QA_PRESTRIPPED && -s $log && \
                ${QA_STRICT_PRESTRIPPED-unset} = unset ]] ; then
index 16c3c192c753d69396a67fda2a88b3cb66fb2d95..26d992c46ee193eab7a2dc4cfabeced719e2f51a 100755 (executable)
@@ -1862,6 +1862,7 @@ _source_ebuild() {
 
        # This needs to be exported since prepstrip is a separate shell script.
        [[ -n $QA_PRESTRIPPED ]] && export QA_PRESTRIPPED
+       eval "[[ -n \$QA_PRESTRIPPED_$ARCH ]] && export QA_PRESTRIPPED_$ARCH"
 }
 
 if ! hasq "$EBUILD_PHASE" clean cleanrm ; then
index 481ac149e9d5c0cc358a6d92459f1496c5366c74..4f6478fa6a7bfb8ff6b3908961b80d61c3bf9d91 100644 (file)
@@ -167,6 +167,8 @@ install_qa_check() {
 
                # Check for files built without respecting LDFLAGS
                if [[ "${LDFLAGS}" == *--hash-style=gnu* ]] && [[ "${PN}" != *-bin ]] ; then
+                       qa_var="QA_DT_HASH_${ARCH/-/_}"
+                       eval "[[ -n \${!qa_var} ]] && QA_DT_HASH=(\"\${${qa_var}[@]}\")"
                        f=$(scanelf -qyRF '%k %p' -k .hash "${D}" | sed -e "s:\.hash ::")
                        if [[ -n ${f} ]] ; then
                                echo "${f}" > "${T}"/scanelf-ignored-LDFLAGS.log
@@ -241,26 +243,71 @@ install_qa_check() {
                        die "Aborting due to QA concerns: ${die_msg}"
                fi
 
-               # Run some sanity checks on shared libraries
-               for d in "${D}"lib* "${D}"usr/lib* ; do
-                       f=$(scanelf -ByF '%S %p' "${d}"/lib*.so* | gawk '$2 == "" { print }')
+               # Check for shared libraries lacking SONAMEs
+               qa_var="QA_SONAME_${ARCH/-/_}"
+               eval "[[ -n \${!qa_var} ]] && QA_SONAME=(\"\${${qa_var}[@]}\")"
+               f=$(scanelf -ByF '%S %p' "${D}"{,usr/}lib*/lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${D}:/:")
+               if [[ -n ${f} ]] ; then
+                       echo "${f}" > "${T}"/scanelf-missing-SONAME.log
+                       if [[ "${QA_STRICT_SONAME-unset}" == unset ]] ; then
+                               if [[ ${#QA_SONAME[@]} -gt 1 ]] ; then
+                                       for x in "${QA_SONAME[@]}" ; do
+                                               sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-SONAME.log
+                                       done
+                               else
+                                       local shopts=$-
+                                       set -o noglob
+                                       for x in ${QA_SONAME} ; do
+                                               sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-SONAME.log
+                                       done
+                                       set +o noglob
+                                       set -${shopts}
+                               fi
+                       fi
+                       f=$(<"${T}"/scanelf-missing-SONAME.log)
                        if [[ -n ${f} ]] ; then
                                vecho -ne '\a\n'
                                eqawarn "QA Notice: The following shared libraries lack a SONAME"
                                eqawarn "${f}"
                                vecho -ne '\a\n'
                                sleep 1
+                       else
+                               rm -f "${T}"/scanelf-missing-SONAME.log
                        fi
+               fi
 
-                       f=$(scanelf -ByF '%n %p' "${d}"/lib*.so* | gawk '$2 == "" { print }')
+               # Check for shared libraries lacking NEEDED entries
+               qa_var="QA_DT_NEEDED_${ARCH/-/_}"
+               eval "[[ -n \${!qa_var} ]] && QA_DT_NEEDED=(\"\${${qa_var}[@]}\")"
+               f=$(scanelf -ByF '%n %p' "${D}"{,usr/}lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${D}:/:")
+               if [[ -n ${f} ]] ; then
+                       echo "${f}" > "${T}"/scanelf-missing-NEEDED.log
+                       if [[ "${QA_STRICT_DT_NEEDED-unset}" == unset ]] ; then
+                               if [[ ${#QA_DT_NEEDED[@]} -gt 1 ]] ; then
+                                       for x in "${QA_DT_NEEDED[@]}" ; do
+                                               sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-NEEDED.log
+                                       done
+                               else
+                                       local shopts=$-
+                                       set -o noglob
+                                       for x in ${QA_DT_NEEDED} ; do
+                                               sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-NEEDED.log
+                                       done
+                                       set +o noglob
+                                       set -${shopts}
+                               fi
+                       fi
+                       f=$(<"${T}"/scanelf-missing-NEEDED.log)
                        if [[ -n ${f} ]] ; then
                                vecho -ne '\a\n'
                                eqawarn "QA Notice: The following shared libraries lack NEEDED entries"
                                eqawarn "${f}"
                                vecho -ne '\a\n'
                                sleep 1
+                       else
+                               rm -f "${T}"/scanelf-missing-NEEDED.log
                        fi
-               done
+               fi
 
                PORTAGE_QUIET=${tmp_quiet}
        fi
index 90b94f0cca9251d5cc43c9bcda4764efe0d6cf64..7f4d4fa7f01e6d60440557eca57831cae7ba82c2 100644 (file)
@@ -510,6 +510,16 @@ LDFLAGS variable.
 This should contain a list of file paths, relative to the image directory, of
 files that contain pre-stripped binaries. The paths may contain regular
 expressions with escape\-quoted special characters.
+.TP
+\fBQA_SONAME\fR
+This should contain a list of file paths, relative to the image directory, of
+shared libraries that lack SONAMEs. The paths may contain regular expressions
+with escape\-quoted special characters.
+.TP
+\fBQA_DT_NEEDED\fR
+This should contain a list of file paths, relative to the image directory, of
+shared libraries that lack NEEDED entries. The paths may contain regular
+expressions with escape\-quoted special characters.
 .SH "PORTAGE DECLARATIONS"
 .TP
 .B inherit
index 6cc46706a1f451d78c366e03e52cae6ba7eebe08..fe457b79f5c2176a930ee90b39e0cde24703fcd7 100644 (file)
@@ -152,8 +152,8 @@ as if it were a single file.
 
 .I Example:
 .nf
-/usr/portage/profiles/package.mask/removals
-/usr/portage/profiles/package.mask/testing
+${PORTDIR}/profiles/package.mask/removals
+${PORTDIR}/profiles/package.mask/testing
 .fi
 .RS
 .TP
index 6aa2d99b7083d2162f8dfaf2155904aa2ed73f0f..98724e8a7bb5dcd1a87ea5302fbb6df9bcf87ee5 100644 (file)
@@ -13,6 +13,7 @@ except ImportError:
 
 import portage
 from portage import os
+from portage import _encodings
 from portage.output import xtermTitle
 
 from _emerge.getloadavg import getloadavg
@@ -70,7 +71,7 @@ class JobStatusDisplay(object):
        def _write(self, s):
                if sys.hexversion < 0x3000000 and isinstance(s, unicode):
                        # avoid potential UnicodeEncodeError
-                       s = portage._unicode_encode(s)
+                       s = s.encode(_encodings['stdio'], 'backslashreplace')
                self.out.write(s)
                self.out.flush()
 
index 87cc5a194995f6ab2f9ef5f08de56c93165c0663..005da2afccdbbe1fc149ec1f6284494cbcc2fe93 100644 (file)
@@ -126,17 +126,13 @@ _encodings = {
        'fs'                     : 'utf_8',
        'merge'                  : sys.getfilesystemencoding(),
        'repo.content'           : 'utf_8',
+       'stdio'                  : 'utf_8',
 }
 
 # This can happen if python is built with USE=build (stage 1).
 if _encodings['merge'] is None:
        _encodings['merge'] = 'ascii'
 
-# Deprecated attributes. Instead use _encodings directly.
-_content_encoding = _encodings['content']
-_fs_encoding = _encodings['fs']
-_merge_encoding = _encodings['merge']
-
 def _unicode_encode(s, encoding=_encodings['content'],
        errors='backslashreplace'):
        if isinstance(s, unicode):
index 9c0f082993247410796568490a226f507a6bbff7..ca6ec4decd1abcd326ca7337abf09ce89ed0b1dc 100644 (file)
@@ -7,8 +7,7 @@
 import os
 import shutil
 
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_encode
 from portage.localization import _
 
@@ -16,8 +15,8 @@ import selinux
 from selinux import is_selinux_enabled
 
 def copyfile(src, dest):
-       src = _unicode_encode(src, encoding=_fs_encoding, errors='strict')
-       dest = _unicode_encode(dest, encoding=_fs_encoding, errors='strict')
+       src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict')
+       dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict')
        (rc, ctx) = selinux.lgetfilecon(src)
        if rc < 0:
                raise OSError(_("copyfile: Failed getting context of \"%s\".") % src)
@@ -36,8 +35,8 @@ def getcontext():
        return ctx
 
 def mkdir(target, refdir):
-       target = _unicode_encode(target, encoding=_fs_encoding, errors='strict')
-       refdir = _unicode_encode(refdir, encoding=_fs_encoding, errors='strict')
+       target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict')
+       refdir = _unicode_encode(refdir, encoding=_encodings['fs'], errors='strict')
        (rc, ctx) = selinux.getfilecon(refdir)
        if rc < 0:
                raise OSError(
@@ -51,8 +50,8 @@ def mkdir(target, refdir):
                selinux.setfscreatecon()
 
 def rename(src, dest):
-       src = _unicode_encode(src, encoding=_fs_encoding, errors='strict')
-       dest = _unicode_encode(dest, encoding=_fs_encoding, errors='strict')
+       src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict')
+       dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict')
        (rc, ctx) = selinux.lgetfilecon(src)
        if rc < 0:
                raise OSError(_("rename: Failed getting context of \"%s\".") % src)
@@ -69,13 +68,13 @@ def settype(newtype):
        return ":".join(ret)
 
 def setexec(ctx="\n"):
-       ctx = _unicode_encode(ctx, encoding=_content_encoding, errors='strict')
+       ctx = _unicode_encode(ctx, encoding=_encodings['content'], errors='strict')
        if selinux.setexeccon(ctx) < 0:
                raise OSError(_("setexec: Failed setting exec() context \"%s\".") % ctx)
 
 def setfscreate(ctx="\n"):
        ctx = _unicode_encode(ctx,
-               encoding=_content_encoding, errors='strict')
+               encoding=_encodings['content'], errors='strict')
        if selinux.setfscreatecon(ctx) < 0:
                raise OSError(
                        _("setfscreate: Failed setting fs create context \"%s\".") % ctx)
@@ -83,7 +82,7 @@ def setfscreate(ctx="\n"):
 def spawn_wrapper(spawn_func, selinux_type):
 
        selinux_type = _unicode_encode(selinux_type,
-               encoding=_content_encoding, errors='strict')
+               encoding=_encodings['content'], errors='strict')
 
        def wrapper_func(*args, **kwargs):
                con = settype(selinux_type)
@@ -96,9 +95,9 @@ def spawn_wrapper(spawn_func, selinux_type):
        return wrapper_func
 
 def symlink(target, link, reflnk):
-       target = _unicode_encode(target, encoding=_fs_encoding, errors='strict')
-       link = _unicode_encode(link, encoding=_fs_encoding, errors='strict')
-       reflnk = _unicode_encode(reflnk, encoding=_fs_encoding, errors='strict')
+       target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict')
+       link = _unicode_encode(link, encoding=_encodings['fs'], errors='strict')
+       reflnk = _unicode_encode(reflnk, encoding=_encodings['fs'], errors='strict')
        (rc, ctx) = selinux.lgetfilecon(reflnk)
        if rc < 0:
                raise OSError(
index bcaf30640e86d7d681651d89ecfe489853ba1dfa..4406b4e77e87888b6fa2165fa05a1a3c439d2a8b 100644 (file)
@@ -10,7 +10,7 @@ from portage.cache import fs_template
 from portage.versions import catsplit
 from portage import cpv_getkey
 from portage import os
-from portage import _fs_encoding
+from portage import _encoding
 from portage import _unicode_decode
 import xattr
 from errno import ENODATA,ENOSPC,E2BIG
@@ -159,7 +159,7 @@ class database(fs_template.FsBased):
                        for file in files:
                                try:
                                        file = _unicode_decode(file,
-                                               encoding=_fs_encoding, errors='strict')
+                                               encoding=_encodings['fs'], errors='strict')
                                except UnicodeDecodeError:
                                        continue
                                if file[-7:] == '.ebuild':
index f871c49ae572dd3ef720ff2805e2a234bcfa2d65..0a67d2dc0caa0943ad91cf74bfbc0336a01f1113 100644 (file)
@@ -7,8 +7,7 @@ import portage
 from portage.const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE
 from portage.localization import _
 from portage import os
-from portage import _fs_encoding
-from portage import _merge_encoding
+from portage import _encodings
 from portage import _unicode_encode
 import errno
 import stat
@@ -29,7 +28,7 @@ def _generate_hash_function(hashtype, hashobject, origin="unknown"):
                @return: The hash and size of the data
                """
                f = open(_unicode_encode(filename,
-                       encoding=_fs_encoding, errors='strict'), 'rb')
+                       encoding=_encodings['fs'], errors='strict'), 'rb')
                blocksize = HASHING_BLOCKSIZE
                data = f.read(blocksize)
                size = 0L
@@ -123,7 +122,7 @@ def perform_md5(x, calc_prelink=0):
 
 def _perform_md5_merge(x, **kwargs):
        return perform_md5(_unicode_encode(x,
-               encoding=_merge_encoding, errors='strict'), **kwargs)
+               encoding=_encodings['merge'], errors='strict'), **kwargs)
 
 def perform_all(x, calc_prelink=0):
        mydict = {}
@@ -221,7 +220,7 @@ def perform_checksum(filename, hashname="MD5", calc_prelink=0):
        # Make sure filename is encoded with the correct encoding before
        # it is passed to spawn (for prelink) and/or the hash function.
        filename = _unicode_encode(filename,
-               encoding=_fs_encoding, errors='strict')
+               encoding=_encodings['fs'], errors='strict')
        myfilename = filename
        prelink_tmpfile = None
        try:
index 0ba85867a12bfb3c9f20bbd97e2788429fe608aa..8de25b7f62d4d5fff6a1082060be7f783ed32c8e 100644 (file)
@@ -22,6 +22,7 @@ from portage.localization import _
 
 from portage import dep_expand, listdir, _check_distfile, _movefile
 from portage import os
+from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
 
@@ -76,7 +77,8 @@ class bindbapi(fakedbapi):
                        def getitem(k):
                                v = tbz2.getfile(k)
                                if v is not None:
-                                       v = _unicode_decode(v)
+                                       v = _unicode_decode(v,
+                                               encoding=_encodings['repo.content'], errors='replace')
                                return v
                else:
                        getitem = self.bintree._remotepkgs[mycpv].get
@@ -111,8 +113,10 @@ class bindbapi(fakedbapi):
                mydata = mytbz2.get_data()
 
                for k, v in values.iteritems():
-                       k = _unicode_encode(k)
-                       v = _unicode_encode(v)
+                       k = _unicode_encode(k,
+                               encoding=_encodings['repo.content'], errors='backslashreplace')
+                       v = _unicode_encode(v,
+                               encoding=_encodings['repo.content'], errors='backslashreplace')
                        mydata[k] = v
 
                for k, v in mydata.items():
@@ -655,8 +659,10 @@ class binarytree(object):
                                urldata[1] + urldata[2], "Packages")
                        pkgindex = self._new_pkgindex()
                        try:
-                               f = codecs.open(_unicode_encode(pkgindex_file),
-                                       encoding='utf_8', errors='replace')
+                               f = codecs.open(_unicode_encode(pkgindex_file,
+                                       encoding=_encodings['fs'], errors='strict'),
+                                       mode='r', encoding=_encodings['repo.content'],
+                                       errors='replace')
                                try:
                                        pkgindex.read(f)
                                finally:
@@ -1097,8 +1103,10 @@ class binarytree(object):
        def _load_pkgindex(self):
                pkgindex = self._new_pkgindex()
                try:
-                       f = codecs.open(_unicode_encode(self._pkgindex_file), 
-                               encoding='utf_8', errors='replace')
+                       f = codecs.open(_unicode_encode(self._pkgindex_file,
+                               encoding=_encodings['fs'], errors='strict'),
+                               mode='r', encoding=_encodings['repo.content'],
+                               errors='replace')
                except EnvironmentError:
                        pass
                else:
index 56ed4494512161634c964527efb283a6eac41dcc..34e9d178ef12d70f673694ec5eaac0e463ff5621 100644 (file)
@@ -26,8 +26,9 @@ from portage.manifest import Manifest
 from portage import eclass_cache, auxdbkeys, doebuild, flatten, \
        listdir, dep_expand, eapi_is_supported, key_expand, dep_check, \
        _eapi_is_deprecated
-from portage import _unicode_encode
 from portage import os
+from portage import _encodings
+from portage import _unicode_encode
 
 import codecs
 import logging
@@ -172,8 +173,10 @@ class portdbapi(dbapi):
                        repo_name_path = os.path.join(path, REPO_NAME_LOC)
                        try:
                                repo_name = codecs.open(
-                                       _unicode_encode(repo_name_path), mode='r',
-                                       encoding='utf_8', errors='replace').readline().strip()
+                                       _unicode_encode(repo_name_path,
+                                       encoding=_encodings['fs'], errors='strict'),
+                                       mode='r', encoding=_encodings['repo.content'],
+                                       errors='replace').readline().strip()
                        except EnvironmentError:
                                # warn about missing repo_name at some other time, since we
                                # don't want to see a warning every time the portage module is
@@ -621,8 +624,10 @@ class portdbapi(dbapi):
                        if eapi is None and \
                                'parse-eapi-ebuild-head' in self.doebuild_settings.features:
                                eapi = portage._parse_eapi_ebuild_head(codecs.open(
-                                       _unicode_encode(myebuild), mode='r',
-                                       encoding='utf_8', errors='replace'))
+                                       _unicode_encode(myebuild,
+                                       encoding=_encodings['fs'], errors='strict'),
+                                       mode='r', encoding=_encodings['repo.content'],
+                                       errors='replace'))
 
                        if eapi is not None:
                                self.doebuild_settings.configdict['pkg']['EAPI'] = eapi
index f1ed5a56934eced7fe91f5b221c2d45a83566c07..192b137d3247546ddbe75d08b01d3e09bebf8647 100644 (file)
@@ -38,9 +38,7 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \
 
 # This is a special version of the os module, wrapped for unicode support.
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
-from portage import _merge_encoding
+from portage import _encodings
 from portage import _os_merge
 from portage import _selinux_merge
 from portage import _unicode_decode
@@ -81,7 +79,8 @@ class PreservedLibsRegistry(object):
                self._data = None
                try:
                        self._data = pickle.load(
-                               open(_unicode_encode(self._filename), 'rb'))
+                               open(_unicode_encode(self._filename,
+                                       encoding=_encodings['fs'], errors='strict'), 'rb'))
                except (ValueError, pickle.UnpicklingError), e:
                        writemsg_level(_("!!! Error loading '%s': %s\n") % \
                                (self._filename, e), level=logging.ERROR, noiselevel=-1)
@@ -328,7 +327,15 @@ class LinkageMap(object):
                                raise CommandNotFound(args[0])
                        else:
                                for l in proc.stdout:
-                                       l = portage._unicode_decode(l)
+                                       try:
+                                               l = _unicode_decode(l,
+                                                       encoding=_encodings['content'], errors='strict')
+                                       except UnicodeDecodeError:
+                                               l = _unicode_decode(l,
+                                                       encoding=_encodings['content'], errors='replace')
+                                               writemsg_level(_("\nError decoding characters " \
+                                                       "returned from scanelf: %s\n\n") % (l,),
+                                                       level=logging.ERROR, noiselevel=-1)
                                        l = l[3:].rstrip("\n")
                                        if not l:
                                                continue
@@ -1455,7 +1462,8 @@ class vardbapi(dbapi):
                now has a categories property that is generated from the
                available packages.
                """
-               self.root = portage._unicode_decode(root)
+               self.root = _unicode_decode(root,
+                       encoding=_encodings['content'], errors='strict')
 
                #cache for category directory mtimes
                self.mtdircache = {}
@@ -1539,7 +1547,8 @@ class vardbapi(dbapi):
                        except KeyError:
                                continue
                        h.update(_unicode_encode(counter,
-                               encoding=_content_encoding, errors='replace'))
+                               encoding=_encodings['repo.content'],
+                               errors='backslashreplace'))
                return h.hexdigest()
 
        def cpv_inject(self, mycpv):
@@ -1804,7 +1813,8 @@ class vardbapi(dbapi):
                        # python-2.x, but buffering makes it much worse.
                        open_kwargs["buffering"] = 0
                try:
-                       f = open(_unicode_encode(self._aux_cache_filename),
+                       f = open(_unicode_encode(self._aux_cache_filename,
+                               encoding=_encodings['fs'], errors='strict'),
                                mode='rb', **open_kwargs)
                        mypickle = pickle.Unpickler(f)
                        try:
@@ -1906,7 +1916,8 @@ class vardbapi(dbapi):
                if cache_valid:
                        # Migrate old metadata to unicode.
                        for k, v in metadata.iteritems():
-                               metadata[k] = portage._unicode_decode(v)
+                               metadata[k] = _unicode_decode(v,
+                                       encoding=_encodings['repo.content'], errors='replace')
 
                        mydata.update(metadata)
                        pull_me.difference_update(mydata)
@@ -1953,8 +1964,9 @@ class vardbapi(dbapi):
                        try:
                                myf = codecs.open(
                                        _unicode_encode(os.path.join(mydir, x),
-                                       encoding=_fs_encoding, errors='strict'),
-                                       mode='r', encoding=_content_encoding, errors='replace')
+                                       encoding=_encodings['fs'], errors='strict'),
+                                       mode='r', encoding=_encodings['repo.content'],
+                                       errors='replace')
                                try:
                                        myd = myf.read()
                                finally:
@@ -2024,8 +2036,9 @@ class vardbapi(dbapi):
                try:
                        cfile = codecs.open(
                                _unicode_encode(self._counter_path,
-                               encoding=_fs_encoding, errors='strict'),
-                               mode='r', encoding=_content_encoding, errors='replace')
+                               encoding=_encodings['fs'], errors='strict'),
+                               mode='r', encoding=_encodings['repo.content'],
+                               errors='replace')
                except EnvironmentError, e:
                        new_vdb = not bool(self.cpv_all())
                        if not new_vdb:
@@ -2092,7 +2105,8 @@ class vardbapi(dbapi):
                removed = 0
 
                for filename in paths:
-                       filename = portage._unicode_decode(filename)
+                       filename = _unicode_decode(filename,
+                               encoding=_encodings['content'], errors='strict')
                        filename = normalize_path(filename)
                        if relative_paths:
                                relative_filename = filename
@@ -2161,7 +2175,8 @@ class vardbapi(dbapi):
                        # Always use a constant utf_8 encoding here, since
                        # the "default" encoding can change.
                        h.update(_unicode_encode(s,
-                               encoding=_content_encoding, errors='replace'))
+                               encoding=_encodings['repo.content'],
+                               errors='backslashreplace'))
                        h = h.hexdigest()
                        h = h[-self._hex_chars:]
                        h = int(h, 16)
@@ -2610,8 +2625,9 @@ class dblink(object):
                pkgfiles = {}
                try:
                        myc = codecs.open(_unicode_encode(contents_file,
-                               encoding=_content_encoding, errors='strict'),
-                               mode='r', encoding=_content_encoding, errors='replace')
+                               encoding=_encodings['fs'], errors='strict'),
+                               mode='r', encoding=_encodings['repo.content'],
+                               errors='replace')
                except EnvironmentError, e:
                        if e.errno != errno.ENOENT:
                                raise
@@ -3087,13 +3103,15 @@ class dblink(object):
                                obj = normalize_path(objkey)
                                if os is _os_merge:
                                        try:
-                                               _unicode_encode(obj, encoding=_merge_encoding, errors='strict')
+                                               _unicode_encode(obj,
+                                                       encoding=_encodings['merge'], errors='strict')
                                        except UnicodeEncodeError:
                                                # The package appears to have been merged with a 
                                                # different value of sys.getfilesystemencoding(),
                                                # so fall back to utf_8 if appropriate.
                                                try:
-                                                       _unicode_encode(obj, encoding=_fs_encoding, errors='strict')
+                                                       _unicode_encode(obj,
+                                                               encoding=_encodings['fs'], errors='strict')
                                                except UnicodeEncodeError:
                                                        pass
                                                else:
@@ -3290,9 +3308,11 @@ class dblink(object):
 
                os = _os_merge
 
-               filename = portage._unicode_decode(filename)
+               filename = _unicode_decode(filename,
+                       encoding=_encodings['content'], errors='strict')
 
-               destroot = portage._unicode_decode(destroot)
+               destroot = _unicode_decode(destroot,
+                       encoding=_encodings['content'], errors='strict')
 
                destfile = normalize_path(
                        os.path.join(destroot, filename.lstrip(os.path.sep)))
@@ -3484,7 +3504,8 @@ class dblink(object):
                new_contents = self.getcontents().copy()
                old_contents = self._installed_instance.getcontents()
                for f in sorted(preserve_paths):
-                       f = portage._unicode_decode(f, encoding=_merge_encoding)
+                       f = _unicode_decode(f,
+                               encoding=_encodings['content'], errors='strict')
                        f_abs = os.path.join(root, f.lstrip(os.sep))
                        contents_entry = old_contents.get(f_abs)
                        if contents_entry is None:
@@ -3926,10 +3947,14 @@ class dblink(object):
 
                os = _os_merge
 
-               srcroot = portage._unicode_decode(srcroot)
-               destroot = portage._unicode_decode(destroot)
-               inforoot = portage._unicode_decode(inforoot)
-               myebuild = portage._unicode_decode(myebuild)
+               srcroot = _unicode_decode(srcroot,
+                       encoding=_encodings['content'], errors='strict')
+               destroot = _unicode_decode(destroot,
+                       encoding=_encodings['content'], errors='strict')
+               inforoot = _unicode_decode(inforoot,
+                       encoding=_encodings['content'], errors='strict')
+               myebuild = _unicode_decode(myebuild,
+                       encoding=_encodings['content'], errors='strict')
 
                showMessage = self._display_merge
                scheduler = self._scheduler
@@ -3947,9 +3972,9 @@ class dblink(object):
                        try:
                                val = codecs.open(_unicode_encode(
                                        os.path.join(inforoot, var_name),
-                                       encoding=_fs_encoding, errors='strict'),
-                                       mode='r', encoding=_content_encoding, errors='replace'
-                                       ).readline().strip()
+                                       encoding=_encodings['fs'], errors='strict'),
+                                       mode='r', encoding=_encodings['repo.content'],
+                                       errors='replace').readline().strip()
                        except EnvironmentError, e:
                                if e.errno != errno.ENOENT:
                                        raise
@@ -4037,14 +4062,14 @@ class dblink(object):
                        for parent, dirs, files in os.walk(srcroot, onerror=onerror):
                                try:
                                        parent = _unicode_decode(parent,
-                                               encoding=_merge_encoding, errors='strict')
+                                               encoding=_encodings['merge'], errors='strict')
                                except UnicodeDecodeError:
                                        new_parent = _unicode_decode(parent,
-                                               encoding=_merge_encoding, errors='replace')
+                                               encoding=_encodings['merge'], errors='replace')
                                        new_parent = _unicode_encode(new_parent,
-                                               encoding=_merge_encoding, errors='backslashreplace')
+                                               encoding=_encodings['merge'], errors='backslashreplace')
                                        new_parent = _unicode_decode(new_parent,
-                                               encoding=_merge_encoding, errors='replace')
+                                               encoding=_encodings['merge'], errors='replace')
                                        os.rename(parent, new_parent)
                                        unicode_error = True
                                        unicode_errors.append(new_parent[srcroot_len:])
@@ -4053,16 +4078,16 @@ class dblink(object):
                                for fname in files:
                                        try:
                                                fname = _unicode_decode(fname,
-                                                       encoding=_merge_encoding, errors='strict')
+                                                       encoding=_encodings['merge'], errors='strict')
                                        except UnicodeDecodeError:
                                                fpath = portage._os.path.join(
-                                                       parent.encode(_merge_encoding), fname)
+                                                       parent.encode(_encodings['merge']), fname)
                                                new_fname = _unicode_decode(fname,
-                                                       encoding=_merge_encoding, errors='replace')
+                                                       encoding=_encodings['merge'], errors='replace')
                                                new_fname = _unicode_encode(new_fname,
-                                                       encoding=_merge_encoding, errors='backslashreplace')
+                                                       encoding=_encodings['merge'], errors='backslashreplace')
                                                new_fname = _unicode_decode(new_fname,
-                                                       encoding=_merge_encoding, errors='replace')
+                                                       encoding=_encodings['merge'], errors='replace')
                                                new_fpath = os.path.join(parent, new_fname)
                                                os.rename(fpath, new_fpath)
                                                unicode_error = True
@@ -4290,14 +4315,17 @@ class dblink(object):
 
                # write local package counter for recording
                counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv)
-               open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER')),
-                       'w').write(str(counter))
+               codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
+                       encoding=_encodings['fs'], errors='strict'),
+                       'w', encoding=_encodings['repo.content'], errors='backslashreplace'
+                       ).write(str(counter))
 
                # open CONTENTS file (possibly overwriting old one) for recording
                outfile = codecs.open(_unicode_encode(
                        os.path.join(self.dbtmpdir, 'CONTENTS'),
-                       encoding=_fs_encoding, errors='strict'),
-                       mode='w', encoding=_content_encoding, errors='replace')
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='w', encoding=_encodings['repo.content'],
+                       errors='backslashreplace')
 
                self.updateprotect()
 
@@ -4647,7 +4675,7 @@ class dblink(object):
                                # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly
                                mymtime = movefile(mysrc, mydest, newmtime=thismtime,
                                        sstat=mystat, mysettings=self.settings,
-                                       encoding=_merge_encoding)
+                                       encoding=_encodings['merge'])
                                if mymtime != None:
                                        showMessage(">>> %s -> %s\n" % (mydest, myto))
                                        outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n")
@@ -4687,7 +4715,7 @@ class dblink(object):
                                                # a non-directory and non-symlink-to-directory.  Won't work for us.  Move out of the way.
                                                if movefile(mydest, mydest+".backup",
                                                        mysettings=self.settings,
-                                                       encoding=_merge_encoding) is None:
+                                                       encoding=_encodings['merge']) is None:
                                                        return 1
                                                showMessage(_("bak %s %s.backup\n") % (mydest, mydest),
                                                        level=logging.ERROR, noiselevel=-1)
@@ -4792,7 +4820,7 @@ class dblink(object):
                                        mymtime = movefile(mysrc, mydest, newmtime=thismtime,
                                                sstat=mystat, mysettings=self.settings,
                                                hardlink_candidates=hardlink_candidates,
-                                               encoding=_merge_encoding)
+                                               encoding=_encodings['merge'])
                                        if mymtime is None:
                                                return 1
                                        if hardlink_candidates is not None:
@@ -4809,7 +4837,7 @@ class dblink(object):
                                        # destination doesn't exist
                                        if movefile(mysrc, mydest, newmtime=thismtime,
                                                sstat=mystat, mysettings=self.settings,
-                                               encoding=_merge_encoding) is not None:
+                                               encoding=_encodings['merge']) is not None:
                                                zing = ">>>"
                                        else:
                                                return 1
@@ -4897,8 +4925,8 @@ class dblink(object):
                        return ""
                mydata = codecs.open(
                        _unicode_encode(os.path.join(self.dbdir, name),
-                       encoding=_fs_encoding, errors='strict'),
-                       mode='r', encoding=_content_encoding, errors='replace'
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='r', encoding=_encodings['repo.content'], errors='replace'
                        ).read().split()
                return " ".join(mydata)
 
@@ -4909,22 +4937,27 @@ class dblink(object):
                if not os.path.exists(self.dbdir+"/"+fname):
                        return ""
                return codecs.open(_unicode_encode(os.path.join(self.dbdir, fname),
-                       encoding=_fs_encoding, errors='strict'), 
-                       mode='r', encoding=_content_encoding, errors='replace').read()
+                       encoding=_encodings['fs'], errors='strict'), 
+                       mode='r', encoding=_encodings['repo.content'], errors='replace'
+                       ).read()
 
        def setfile(self,fname,data):
-               mode = 'w'
+               kwargs = {}
                if fname == 'environment.bz2' or not isinstance(data, basestring):
-                       mode = 'wb'
-               write_atomic(os.path.join(self.dbdir, fname), data, mode=mode)
+                       kwargs['mode'] = 'wb'
+               else:
+                       kwargs['mode'] = 'w'
+                       kwargs['encoding'] = _encodings['repo.content']
+               write_atomic(os.path.join(self.dbdir, fname), data, **kwargs)
 
        def getelements(self,ename):
                if not os.path.exists(self.dbdir+"/"+ename):
                        return []
                mylines = codecs.open(_unicode_encode(
                        os.path.join(self.dbdir, ename),
-                       encoding=_fs_encoding, errors='strict'),
-                       mode='r', encoding=_content_encoding, errors='replace').readlines()
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='r', encoding=_encodings['repo.content'], errors='replace'
+                       ).readlines()
                myreturn = []
                for x in mylines:
                        for y in x[:-1].split():
@@ -4934,8 +4967,9 @@ class dblink(object):
        def setelements(self,mylist,ename):
                myelement = codecs.open(_unicode_encode(
                        os.path.join(self.dbdir, ename),
-                       encoding=_fs_encoding, errors='strict'), mode='w',
-                       encoding=_content_encoding, errors='replace')
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='w', encoding=_encodings['repo.content'],
+                       errors='backslashreplace')
                for x in mylist:
                        myelement.write(x+"\n")
                myelement.close()
@@ -5013,7 +5047,8 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None):
                                tarinfo.size = 0
                                tar.addfile(tarinfo)
                        else:
-                               f = open(_unicode_encode(path, encoding=_merge_encoding), 'rb')
+                               f = open(_unicode_encode(path,
+                                       encoding=_encodings['merge'], errors='strict'), 'rb')
                                try:
                                        tar.addfile(tarinfo, f)
                                finally:
index a51f0864e05417b0d9e3aa90a10bb2f06ac9eb39..a563ad271d3c3606961cc976053864a5dbe2c97a 100644 (file)
@@ -12,6 +12,9 @@ portage.proxy.lazyimport.lazyimport(globals(),
 from portage.const import EBUILD_PHASES
 from portage.localization import _
 from portage import os
+from portage import _encodings
+from portage import _unicode_encode
+from portage import _unicode_decode
 
 import codecs
 import sys
@@ -41,8 +44,9 @@ def collect_ebuild_messages(path):
                        logentries[msgfunction] = []
                lastmsgtype = None
                msgcontent = []
-               for l in codecs.open(filename, mode='r',
-                       encoding='utf_8', errors='replace'):
+               for l in codecs.open(_unicode_encode(filename,
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='r', encoding=_encodings['repo.content'], errors='replace'):
                        if not l:
                                continue
                        try:
@@ -87,15 +91,16 @@ def _elog_base(level, msg, phase="other", key=None, color=None, out=None):
        if color is None:
                color = "GOOD"
 
-       if not isinstance(msg, unicode):
-               msg = unicode(msg, encoding='utf_8', errors='replace')
+       msg = _unicode_decode(msg,
+               encoding=_encodings['content'], errors='replace')
 
        formatted_msg = colorize(color, " * ") + msg + "\n"
 
        if sys.hexversion < 0x3000000 and \
                out in (sys.stdout, sys.stderr) and isinstance(formatted_msg, unicode):
                # avoid potential UnicodeEncodeError
-               formatted_msg = formatted_msg.encode('utf_8', 'replace')
+               formatted_msg = formatted_msg.encode(
+                       _encodings['stdio'], 'backslashreplace')
 
        out.write(formatted_msg)
 
index 2d1c1614d33051fc103884045e591569e1817abc..ca76e23be6792692c42fc7bea5100ba44e3cf1e2 100644 (file)
@@ -6,6 +6,8 @@
 import codecs
 import time
 from portage import os
+from portage import _encodings
+from portage import _unicode_encode
 from portage.data import portage_uid, portage_gid
 from portage.util import ensure_dirs
 from portage.const import EPREFIX
@@ -20,8 +22,9 @@ def process(mysettings, key, logentries, fulltext):
        ensure_dirs(elogdir, uid=portage_uid, gid=portage_gid, mode=02770)
 
        elogfilename = elogdir+"/"+path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log"
-       elogfile = codecs.open(elogfilename, mode='w',
-               encoding='utf_8', errors='replace')
+       elogfile = codecs.open(_unicode_encode(elogfilename,
+               encoding=_encodings['fs'], errors='strict'),
+               mode='w', encoding=_encodings['content'], errors='backslashreplace')
        elogfile.write(fulltext)
        elogfile.close()
 
index e1c41e03f901fcdcc7ad41903ca32c0cee09f3a4..755644253cb68f9d720ae399dcac992063e5f603 100644 (file)
@@ -6,6 +6,8 @@
 import codecs
 import time
 from portage import os
+from portage import _encodings
+from portage import _unicode_encode
 from portage.data import portage_uid, portage_gid
 from portage.localization import _
 from portage.util import ensure_dirs, apply_permissions
@@ -20,8 +22,9 @@ def process(mysettings, key, logentries, fulltext):
 
        # TODO: Locking
        elogfilename = elogdir+"/summary.log"
-       elogfile = codecs.open(elogfilename, mode='a',
-               encoding='utf_8', errors='replace')
+       elogfile = codecs.open(_unicode_encode(elogfilename,
+               encoding=_encodings['fs'], errors='strict'),
+               mode='a', encoding=_encodings['content'], errors='backslashreplace')
        apply_permissions(elogfilename, mode=060, mask=0)
        elogfile.write(_(">>> Messages generated by process %(pid)d on %(time)s for package %(pkg)s:\n\n") %
                        {"pid": os.getpid(), "time": time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime(time.time())), "pkg": key})
index 0fe2056445531a5e0191a19fa6f88fdd0f605f8c..d7e955f815fbba46bb74466a3cdbd3b262eab7de 100644 (file)
@@ -6,6 +6,7 @@
 import sys
 import syslog
 from portage.const import EBUILD_PHASES
+from portage import _encodings
 
 _pri = {
        "INFO"   : syslog.LOG_INFO, 
@@ -25,6 +26,7 @@ def process(mysettings, key, logentries, fulltext):
                        msgtext = "%s: %s: %s" % (key, phase, msgtext)
                        if sys.hexversion < 0x3000000 and isinstance(msgtext, unicode):
                                # Avoid TypeError from syslog.syslog()
-                               msgtext = msgtext.encode('utf_8', 'replace')
+                               msgtext = msgtext.encode(_encodings['content'], 
+                                       'backslashreplace')
                        syslog.syslog(_pri[msgtype], msgtext)
        syslog.closelog()
index e878ba449c64ce6a3645afb7a2d4415abf1b1570..5ec3a066fb52536fe0ff9ef2dba1c60de4509b3b 100644 (file)
@@ -7,8 +7,7 @@ import codecs
 import errno
 import stat
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
 from portage.localization import _
@@ -57,7 +56,7 @@ def RecursiveFileLoader(filename):
                        for f in files:
                                try:
                                        f = _unicode_decode(f,
-                                               encoding=_fs_encoding, errors='strict')
+                                               encoding=_encodings['fs'], errors='strict')
                                except UnicodeDecodeError:
                                        continue
                                if f[:1] == '.' or f[-1:] == '~':
@@ -152,8 +151,8 @@ class FileLoader(DataLoader):
                for fn in RecursiveFileLoader(self.fname):
                        try:
                                f = codecs.open(_unicode_encode(fn,
-                                       encoding=_fs_encoding, errors='strict'), mode='r',
-                                       encoding=_content_encoding, errors='replace')
+                                       encoding=_encodings['fs'], errors='strict'), mode='r',
+                                       encoding=_encodings['content'], errors='replace')
                        except EnvironmentError, e:
                                if e.errno not in (errno.ENOENT, errno.ESTALE):
                                        raise
index ce2f6760de443e07ce0e0ab5fbd887ad8270d650..26045573926a0b4aa11bd647da0b3569aad4d924 100644 (file)
@@ -13,7 +13,7 @@ import sys
 import time
 
 from portage import os
-from portage import _content_encoding
+from portage import _encodings
 from portage import _unicode_encode
 from portage.localization import _
 import portage
@@ -22,13 +22,13 @@ def create_message(sender, recipient, subject, body, attachments=None):
 
        if sys.hexversion < 0x3000000:
                sender = _unicode_encode(sender,
-                       encoding=_content_encoding, errors='strict')
+                       encoding=_encodings['content'], errors='strict')
                recipient = _unicode_encode(recipient,
-                       encoding=_content_encoding, errors='strict')
+                       encoding=_encodings['content'], errors='strict')
                subject = _unicode_encode(subject,
-                       encoding=_content_encoding, errors='replace')
+                       encoding=_encodings['content'], errors='backslashreplace')
                body = _unicode_encode(body,
-                       encoding=_content_encoding, errors='replace')
+                       encoding=_encodings['content'], errors='backslashreplace')
 
        if attachments == None:
                mymessage = TextMessage(body)
@@ -41,7 +41,8 @@ def create_message(sender, recipient, subject, body, attachments=None):
                        elif isinstance(x, basestring):
                                if sys.hexversion < 0x3000000:
                                        x = _unicode_encode(x,
-                                               encoding=_content_encoding, errors='replace')
+                                               encoding=_encodings['content'],
+                                               errors='backslashreplace')
                                mymessage.attach(TextMessage(x))
                        else:
                                raise portage.exception.PortageException(_("Can't handle type of attachment: %s") % type(x))
@@ -92,17 +93,17 @@ def send_mail(mysettings, message):
 
        if sys.hexversion < 0x3000000:
                myrecipient = _unicode_encode(myrecipient,
-                       encoding=_content_encoding, errors='strict')
+                       encoding=_encodings['content'], errors='strict')
                mymailhost = _unicode_encode(mymailhost,
-                       encoding=_content_encoding, errors='strict')
+                       encoding=_encodings['content'], errors='strict')
                mymailport = _unicode_encode(mymailport,
-                       encoding=_content_encoding, errors='strict')
+                       encoding=_encodings['content'], errors='strict')
                myfrom = _unicode_encode(myfrom,
-                       encoding=_content_encoding, errors='strict')
+                       encoding=_encodings['content'], errors='strict')
                mymailuser = _unicode_encode(mymailuser,
-                       encoding=_content_encoding, errors='strict')
+                       encoding=_encodings['content'], errors='strict')
                mymailpasswd = _unicode_encode(mymailpasswd,
-                       encoding=_content_encoding, errors='strict')
+                       encoding=_encodings['content'], errors='strict')
 
        # user wants to use a sendmail binary instead of smtp
        if mymailhost[0] == os.sep and os.path.exists(mymailhost):
index 4c4cb60b3e70171e858f7d9591086a09606ce8db..25893d759cdfe2cad3a692a7b19aa45391a7f499 100644 (file)
@@ -12,8 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 )
 
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
 from portage.exception import DigestException, FileNotFound, \
@@ -145,8 +144,8 @@ class Manifest(object):
                   Otherwise, a new dict will be created and returned."""
                try:
                        fd = codecs.open(_unicode_encode(file_path,
-                               encoding=_fs_encoding, errors='strict'), mode='r',
-                               encoding=_content_encoding, errors='replace')
+                               encoding=_encodings['fs'], errors='strict'), mode='r',
+                               encoding=_encodings['repo.content'], errors='replace')
                        if myhashdict is None:
                                myhashdict = {}
                        self._parseDigests(fd, myhashdict=myhashdict, **kwargs)
@@ -233,8 +232,9 @@ class Manifest(object):
                        if not force:
                                try:
                                        f = codecs.open(_unicode_encode(self.getFullname(),
-                                               encoding=_fs_encoding, errors='strict'),
-                                               mode='r', encoding=_content_encoding, errors='replace')
+                                               encoding=_encodings['fs'], errors='strict'),
+                                               mode='r', encoding=_encodings['repo.content'],
+                                               errors='replace')
                                        oldentries = list(self._parseManifestLines(f))
                                        f.close()
                                        if len(oldentries) == len(myentries):
@@ -327,7 +327,7 @@ class Manifest(object):
                for f in pkgdir_files:
                        try:
                                f = _unicode_decode(f,
-                                       encoding=_fs_encoding, errors='strict')
+                                       encoding=_encodings['fs'], errors='strict')
                        except UnicodeDecodeError:
                                continue
                        if f[:1] == ".":
@@ -362,7 +362,7 @@ class Manifest(object):
                        for f in files:
                                try:
                                        f = _unicode_decode(f,
-                                               encoding=_fs_encoding, errors='strict')
+                                               encoding=_encodings['fs'], errors='strict')
                                except UnicodeDecodeError:
                                        continue
                                full_path = os.path.join(parentdir, f)
@@ -523,8 +523,8 @@ class Manifest(object):
                if not os.path.exists(mfname):
                        return rVal
                myfile = codecs.open(_unicode_encode(mfname,
-                       encoding=_fs_encoding, errors='strict'),
-                       mode='r', encoding=_content_encoding, errors='replace')
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='r', encoding=_encodings['repo.content'], errors='replace')
                lines = myfile.readlines()
                myfile.close()
                for l in lines:
index f3482150def91e8b2124177c666b413389abb081..ca608d72250089e4af01a39ac85e1f3ec143d6ec 100644 (file)
@@ -12,8 +12,7 @@ import logging
 import os as _os
 import re
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
 from portage.util import apply_secpass_permissions, ensure_dirs, \
@@ -100,7 +99,7 @@ class NewsManager(object):
                news_dir = self._news_dir(repoid)
                try:
                        news = _os.listdir(_unicode_encode(news_dir,
-                               encoding=_fs_encoding, errors='strict'))
+                               encoding=_encodings['fs'], errors='strict'))
                except OSError:
                        return
 
@@ -120,10 +119,10 @@ class NewsManager(object):
                        for itemid in news:
                                try:
                                        itemid = _unicode_decode(itemid,
-                                               encoding=_fs_encoding, errors='strict')
+                                               encoding=_encodings['fs'], errors='strict')
                                except UnicodeDecodeError:
                                        itemid = _unicode_decode(itemid,
-                                               encoding=_fs_encoding, errors='replace')
+                                               encoding=_encodings['fs'], errors='replace')
                                        writemsg_level(
                                                "!!! Invalid encoding in news item name: '%s'\n" % \
                                                itemid, level=logging.ERROR, noiselevel=-1)
@@ -253,8 +252,9 @@ class NewsItem(object):
 
        def parse(self):
                lines = codecs.open(_unicode_encode(self.path,
-                       encoding=_fs_encoding, errors='strict'),
-                       mode='r', encoding=_content_encoding, errors='replace').readlines()
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='r', encoding=_encodings['content'], errors='replace'
+                       ).readlines()
                self.restrictions = {}
                invalids = []
                for i, line in enumerate(lines):
index c47b65cc0394402691e3230af8da3bc4d1ff9f4f..c0839fa53c1770303df46a08056c064482cc08ae 100644 (file)
@@ -17,8 +17,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 )
 
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_encode
 from portage.const import COLOR_MAP_FILE, EPREFIX
 from portage.exception import CommandNotFound, FileNotFound, \
@@ -169,8 +168,8 @@ def _parse_color_map(config_root='/', onerror=None):
        try:
                lineno=0
                for line in codecs.open(_unicode_encode(myfile,
-                       encoding=_fs_encoding, errors='strict'),
-                       mode='r', encoding=_content_encoding, errors='replace'):
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='r', encoding=_encodings['content'], errors='replace'):
                        lineno += 1
 
                        commenter_pos = line.find("#")
@@ -470,7 +469,7 @@ class EOutput(object):
        def _write(self, f, s):
                if sys.hexversion < 0x3000000 and isinstance(s, unicode):
                        # avoid potential UnicodeEncodeError
-                       s = s.encode(_content_encoding, 'replace')
+                       s = s.encode(_encodings['stdio'], 'backslashreplace')
                f.write(s)
                f.flush()
 
index 1b584688387a79be14b74d52ba0c19e5047037da..1822a3082ef25931c840f22281b29be85e038ed2 100644 (file)
@@ -10,7 +10,7 @@ import sys
 import traceback
 
 from portage import os
-from portage import _content_encoding
+from portage import _encodings
 from portage import _unicode_encode
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
@@ -184,8 +184,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
        # Avoid a potential UnicodeEncodeError from os.execve().
        env_bytes = {}
        for k, v in env.iteritems():
-               env_bytes[_unicode_encode(k, encoding=_content_encoding)] = \
-                       _unicode_encode(v, encoding=_content_encoding)
+               env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
+                       _unicode_encode(v, encoding=_encodings['content'])
        env = env_bytes
        del env_bytes
 
index d39087e1e49487e2c8842e7b0f3370b1c7f3824c..62d7f07573927c2b591faa43b26ba9538abeadfb 100644 (file)
@@ -6,7 +6,7 @@ import re
 from itertools import chain
 
 from portage import os
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
 from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path
@@ -130,16 +130,16 @@ class StaticFileSet(EditablePackageSet):
 
                try:
                        directory = _unicode_decode(directory,
-                               encoding=_fs_encoding, errors='strict')
+                               encoding=_encodings['fs'], errors='strict')
                        # Now verify that we can also encode it.
                        _unicode_encode(directory,
-                               encoding=_fs_encoding, errors='strict')
+                               encoding=_encodings['fs'], errors='strict')
                except UnicodeError:
                        directory = _unicode_decode(directory,
-                               encoding=_fs_encoding, errors='replace')
+                               encoding=_encodings['fs'], errors='replace')
                        raise SetConfigError(
                                _("Directory path contains invalid character(s) for encoding '%s': '%s'") \
-                               % (_fs_encoding, directory))
+                               % (_encodings['fs'], directory))
 
                if os.path.isdir(directory):
                        directory = normalize_path(directory)
@@ -147,7 +147,7 @@ class StaticFileSet(EditablePackageSet):
                        for parent, dirs, files in os.walk(directory):
                                try:
                                        parent = _unicode_decode(parent,
-                                               encoding=_fs_encoding, errors='strict')
+                                               encoding=_encodings['fs'], errors='strict')
                                except UnicodeDecodeError:
                                        continue
                                for d in dirs[:]:
@@ -156,7 +156,7 @@ class StaticFileSet(EditablePackageSet):
                                for filename in files:
                                        try:
                                                filename = _unicode_decode(filename,
-                                                       encoding=_fs_encoding, errors='strict')
+                                                       encoding=_encodings['fs'], errors='strict')
                                        except UnicodeDecodeError:
                                                continue
                                        if filename[:1] == '.':
index 4a5ced8a35b8dbbd60f1ed31f03907f0185f9787..6e7380409949f55673635859470655c5d5ba6654 100644 (file)
@@ -8,16 +8,16 @@ import time
 import unittest
 
 from portage import os
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_encode
 from portage import _unicode_decode
 
 def main():
 
        TEST_FILE = _unicode_encode('__test__',
-               encoding=_fs_encoding, errors='strict')
+               encoding=_encodings['fs'], errors='strict')
        svn_dirname = _unicode_encode('.svn',
-               encoding=_fs_encoding, errors='strict')
+               encoding=_encodings['fs'], errors='strict')
        suite = unittest.TestSuite()
        basedir = os.path.dirname(os.path.realpath(__file__))
        testDirs = []
@@ -30,7 +30,7 @@ def main():
                        dirs.remove(svn_dirname)
                try:
                        root = _unicode_decode(root,
-                               encoding=_fs_encoding, errors='strict')
+                               encoding=_encodings['fs'], errors='strict')
                except UnicodeDecodeError:
                        continue
 
index 908fce6061e0e2f91dc1ddcc04161fb8987d44d4..97a5f42d0aa9385400b2e648cb329ad327dffcdb 100644 (file)
@@ -6,8 +6,7 @@ import codecs
 import errno
 import sys
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_encode
 from portage.tests import TestCase
 
@@ -34,8 +33,8 @@ class SpawnTestCase(TestCase):
                                free=1, fd_pipes={0:sys.stdin.fileno(), 1:null_fd, 2:null_fd})
                        os.close(null_fd)
                        f = codecs.open(_unicode_encode(logfile,
-                               encoding=_fs_encoding, errors='strict'),
-                               mode='r', encoding=_content_encoding, errors='strict')
+                               encoding=_encodings['fs'], errors='strict'),
+                               mode='r', encoding=_encodings['content'], errors='strict')
                        log_content = f.read()
                        f.close()
                        # When logging passes through a pty, this comparison will fail
index d38ddf942ce60211538f320f604869b7d24505bf..ca67370cd2680d08dba971d11a2a856f26cf2be9 100644 (file)
@@ -8,8 +8,7 @@ import re
 import sys
 
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
 import portage
@@ -72,8 +71,8 @@ def fixdbentries(update_iter, dbdir):
        for myfile in [f for f in os.listdir(dbdir) if f not in ignored_dbentries]:
                file_path = os.path.join(dbdir, myfile)
                mydata[myfile] = codecs.open(_unicode_encode(file_path,
-                       encoding=_fs_encoding, errors='strict'),
-                       mode='r', encoding=_content_encoding,
+                       encoding=_encodings['fs'], errors='strict'),
+                       mode='r', encoding=_encodings['repo.content'],
                        errors='replace').read()
        updated_items = update_dbentries(update_iter, mydata)
        for myfile, mycontent in updated_items.iteritems():
@@ -110,8 +109,9 @@ def grab_updates(updpath, prev_mtimes=None):
                if file_path not in prev_mtimes or \
                long(prev_mtimes[file_path]) != long(mystat.st_mtime):
                        content = codecs.open(_unicode_encode(file_path,
-                               encoding=_fs_encoding, errors='strict'),
-                               mode='r', encoding=_content_encoding, errors='replace').read()
+                               encoding=_encodings['fs'], errors='strict'),
+                               mode='r', encoding=_encodings['repo.content'], errors='replace'
+                               ).read()
                        update_data.append((file_path, mystat, content))
        return update_data
 
@@ -172,7 +172,7 @@ def update_config_files(config_root, protect, protect_mask, update_iter):
                                for y in dirs:
                                        try:
                                                y = _unicode_decode(y,
-                                                       encoding=_fs_encoding, errors='strict')
+                                                       encoding=_encodings['fs'], errors='strict')
                                        except UnicodeDecodeError:
                                                dirs.remove(y)
                                                continue
@@ -181,7 +181,7 @@ def update_config_files(config_root, protect, protect_mask, update_iter):
                                for y in files:
                                        try:
                                                y = _unicode_decode(y,
-                                                       encoding=_fs_encoding, errors='strict')
+                                                       encoding=_encodings['fs'], errors='strict')
                                        except UnicodeDecodeError:
                                                continue
                                        if y.startswith("."):
@@ -195,8 +195,8 @@ def update_config_files(config_root, protect, protect_mask, update_iter):
                try:
                        file_contents[x] = codecs.open(
                                _unicode_encode(os.path.join(abs_user_config, x),
-                               encoding=_fs_encoding, errors='strict'),
-                               mode='r', encoding=_content_encoding,
+                               encoding=_encodings['fs'], errors='strict'),
+                               mode='r', encoding=_encodings['content'],
                                errors='replace').readlines()
                except IOError:
                        if file_contents.has_key(x):
index 983c08b0a084eaa0cad1b43cb3d9314c891674e4..685843cac19ff41dd8661735e2552583177e227b 100644 (file)
@@ -23,8 +23,7 @@ import sys
 
 import portage
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _os_merge
 from portage import _unicode_encode
 from portage import _unicode_decode
@@ -65,7 +64,8 @@ def writemsg(mystr,noiselevel=0,fd=None):
        if noiselevel <= noiselimit:
                if sys.hexversion < 0x3000000:
                        # avoid potential UnicodeEncodeError
-                       mystr = _unicode_encode(mystr)
+                       mystr = _unicode_encode(mystr,
+                               encoding=_encodings['stdio'], errors='backslashreplace')
                fd.write(mystr)
                fd.flush()
 
@@ -329,8 +329,8 @@ def grablines(myfilename,recursive=0):
        else:
                try:
                        myfile = codecs.open(_unicode_encode(myfilename,
-                               encoding=_fs_encoding, errors='strict'),
-                               mode='r', encoding=_content_encoding, errors='replace')
+                               encoding=_encodings['fs'], errors='strict'),
+                               mode='r', encoding=_encodings['content'], errors='replace')
                        mylines = myfile.readlines()
                        myfile.close()
                except IOError, e:
@@ -397,11 +397,11 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True):
                # (produces spurious \0 characters with python-2.6.2)
                if sys.hexversion < 0x3000000:
                        content = open(_unicode_encode(mycfg,
-                               encoding=_fs_encoding, errors='strict'), 'rb').read()
+                               encoding=_encodings['fs'], errors='strict'), 'rb').read()
                else:
                        content = open(_unicode_encode(mycfg,
-                               encoding=_fs_encoding, errors='strict'), mode='r',
-                               encoding=_content_encoding, errors='replace').read()
+                               encoding=_encodings['fs'], errors='strict'), mode='r',
+                               encoding=_encodings['content'], errors='replace').read()
                if content and content[-1] != '\n':
                        content += '\n'
        except IOError, e:
@@ -594,7 +594,7 @@ def pickle_read(filename,default=None,debug=0):
        data = None
        try:
                myf = open(_unicode_encode(filename,
-                       encoding=_fs_encoding, errors='strict'), 'rb')
+                       encoding=_encodings['fs'], errors='strict'), 'rb')
                mypickle = pickle.Unpickler(myf)
                data = mypickle.load()
                myf.close()
@@ -909,8 +909,8 @@ class atomic_ofstream(ObjectProxy):
                        open_func = open
                else:
                        open_func = codecs.open
-                       kargs.setdefault('encoding', _content_encoding)
-                       kargs.setdefault('errors', 'replace')
+                       kargs.setdefault('encoding', _encodings['content'])
+                       kargs.setdefault('errors', 'backslashreplace')
 
                if follow_links:
                        canonical_path = os.path.realpath(filename)
@@ -919,7 +919,7 @@ class atomic_ofstream(ObjectProxy):
                        try:
                                object.__setattr__(self, '_file',
                                        open_func(_unicode_encode(tmp_name,
-                                               encoding=_fs_encoding, errors='strict'),
+                                               encoding=_encodings['fs'], errors='strict'),
                                                mode=mode, **kargs))
                                return
                        except IOError, e:
@@ -933,7 +933,7 @@ class atomic_ofstream(ObjectProxy):
                tmp_name = "%s.%i" % (filename, os.getpid())
                object.__setattr__(self, '_file',
                        open_func(_unicode_encode(tmp_name,
-                               encoding=_fs_encoding, errors='strict'),
+                               encoding=_encodings['fs'], errors='strict'),
                                mode=mode, **kargs))
 
        def _get_target(self):
index da68af59c476578c548d785f0b249e3aa2c4f648..5b08c0a3f8284043e0eddb240314ec0120bbddbf 100644 (file)
@@ -22,7 +22,7 @@ import shutil
 
 from portage import os
 from portage import normalize_path
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _unicode_decode
 from portage import _unicode_encode
 
@@ -30,23 +30,24 @@ def addtolist(mylist, curdir):
        """(list, dir) --- Takes an array(list) and appends all files from dir down
        the directory tree. Returns nothing. list is modified."""
        curdir = normalize_path(_unicode_decode(curdir,
-               encoding=_fs_encoding, errors='strict'))
+               encoding=_encodings['fs'], errors='strict'))
        for parent, dirs, files in os.walk(curdir):
 
                parent = _unicode_decode(parent,
-                       encoding=_fs_encoding, errors='strict')
+                       encoding=_encodings['fs'], errors='strict')
                if parent != curdir:
                        mylist.append(parent[len(curdir) + 1:] + os.sep)
 
                for x in dirs:
                        try:
-                               _unicode_decode(x, encoding=_fs_encoding, errors='strict')
+                               _unicode_decode(x, encoding=_encodings['fs'], errors='strict')
                        except UnicodeDecodeError:
                                dirs.remove(x)
 
                for x in files:
                        try:
-                               x = _unicode_decode(x, encoding=_fs_encoding, errors='strict')
+                               x = _unicode_decode(x,
+                                       encoding=_encodings['fs'], errors='strict')
                        except UnicodeDecodeError:
                                continue
                        mylist.append(os.path.join(parent, x)[len(curdir) + 1:])
@@ -82,13 +83,13 @@ def xpak(rootdir,outfile=None):
        mylist.sort()
        mydata = {}
        for x in mylist:
-               x = _unicode_encode(x, encoding=_fs_encoding, errors='strict')
+               x = _unicode_encode(x, encoding=_encodings['fs'], errors='strict')
                mydata[x] = open(os.path.join(rootdir, x), 'rb').read()
 
        xpak_segment = xpak_mem(mydata)
        if outfile:
                outf = open(_unicode_encode(outfile,
-                       encoding=_fs_encoding, errors='strict'), 'wb')
+                       encoding=_encodings['fs'], errors='strict'), 'wb')
                outf.write(xpak_segment)
                outf.close()
        else:
@@ -118,9 +119,9 @@ def xsplit(infile):
        'infile.index' contains the index segment.
        'infile.dat' contails the data segment."""
        infile = _unicode_decode(infile,
-               encoding=_fs_encoding, errors='strict')
+               encoding=_encodings['fs'], errors='strict')
        myfile = open(_unicode_encode(infile,
-               encoding=_fs_encoding, errors='strict'), 'rb')
+               encoding=_encodings['fs'], errors='strict'), 'rb')
        mydat=myfile.read()
        myfile.close()
        
@@ -129,11 +130,11 @@ def xsplit(infile):
                return False
        
        myfile = open(_unicode_encode(infile + '.index',
-               encoding=_fs_encoding, errors='strict'), 'wb')
+               encoding=_encodings['fs'], errors='strict'), 'wb')
        myfile.write(splits[0])
        myfile.close()
        myfile = open(_unicode_encode(infile + '.dat',
-               encoding=_fs_encoding, errors='strict'), 'wb')
+               encoding=_encodings['fs'], errors='strict'), 'wb')
        myfile.write(splits[1])
        myfile.close()
        return True
@@ -149,7 +150,7 @@ def xsplit_mem(mydat):
 def getindex(infile):
        """(infile) -- grabs the index segment from the infile and returns it."""
        myfile = open(_unicode_encode(infile,
-               encoding=_fs_encoding, errors='strict'), 'rb')
+               encoding=_encodings['fs'], errors='strict'), 'rb')
        myheader=myfile.read(16)
        if myheader[0:8] != _unicode_encode('XPAKPACK'):
                myfile.close()
@@ -163,7 +164,7 @@ def getboth(infile):
        """(infile) -- grabs the index and data segments from the infile.
        Returns an array [indexSegment,dataSegment]"""
        myfile = open(_unicode_encode(infile,
-               encoding=_fs_encoding, errors='strict'), 'rb')
+               encoding=_encodings['fs'], errors='strict'), 'rb')
        myheader=myfile.read(16)
        if myheader[0:8] != _unicode_encode('XPAKPACK'):
                myfile.close()
@@ -238,7 +239,7 @@ def xpand(myid,mydest):
                        if not os.path.exists(dirname):
                                os.makedirs(dirname)
                mydat = open(_unicode_encode(myname,
-                       encoding=_fs_encoding, errors='strict'), 'wb')
+                       encoding=_encodings['fs'], errors='strict'), 'wb')
                mydat.write(mydata[datapos:datapos+datalen])
                mydat.close()
                startpos=startpos+namelen+12
@@ -284,7 +285,7 @@ class tbz2(object):
        def recompose_mem(self, xpdata):
                self.scan() # Don't care about condition... We'll rewrite the data anyway.
                myfile = open(_unicode_encode(self.file,
-                       encoding=_fs_encoding, errors='strict'), 'ab+')
+                       encoding=_encodings['fs'], errors='strict'), 'ab+')
                if not myfile:
                        raise IOError
                myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF.
@@ -322,7 +323,7 @@ class tbz2(object):
                                        return 1
                        self.filestat=mystat
                        a = open(_unicode_encode(self.file,
-                               encoding=_fs_encoding, errors='strict'), 'rb')
+                               encoding=_encodings['fs'], errors='strict'), 'rb')
                        a.seek(-16,2)
                        trailer=a.read()
                        self.infosize=0
@@ -366,7 +367,7 @@ class tbz2(object):
                if not myresult:
                        return mydefault
                a = open(_unicode_encode(self.file,
-                       encoding=_fs_encoding, errors='strict'), 'rb')
+                       encoding=_encodings['fs'], errors='strict'), 'rb')
                a.seek(self.datapos+myresult[0],0)
                myreturn=a.read(myresult[1])
                a.close()
@@ -391,7 +392,7 @@ class tbz2(object):
                        os.chdir("/")
                        origdir="/"
                a = open(_unicode_encode(self.file,
-                       encoding=_fs_encoding, errors='strict'), 'rb')
+                       encoding=_encodings['fs'], errors='strict'), 'rb')
                if not os.path.exists(mydest):
                        os.makedirs(mydest)
                os.chdir(mydest)
@@ -406,7 +407,7 @@ class tbz2(object):
                                if not os.path.exists(dirname):
                                        os.makedirs(dirname)
                        mydat = open(_unicode_encode(myname,
-                               encoding=_fs_encoding, errors='strict'), 'wb')
+                               encoding=_encodings['fs'], errors='strict'), 'wb')
                        a.seek(self.datapos+datapos)
                        mydat.write(a.read(datalen))
                        mydat.close()
@@ -420,7 +421,7 @@ class tbz2(object):
                if not self.scan():
                        return 0
                a = open(_unicode_encode(self.file,
-                       encoding=_fs_encoding, errors='strict'), 'rb')
+                       encoding=_encodings['fs'], errors='strict'), 'rb')
                mydata = {}
                startpos=0
                while ((startpos+8)<self.indexsize):
@@ -440,7 +441,7 @@ class tbz2(object):
                        return None
 
                a = open(_unicode_encode(self.file,
-                       encoding=_fs_encoding, errors='strict'), 'rb')
+                       encoding=_encodings['fs'], errors='strict'), 'rb')
                a.seek(self.datapos)
                mydata =a.read(self.datasize)
                a.close()