From 8cfab0162e1ee6c7eae866d22d58c696ebc60bac Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 14 Aug 2009 20:03:05 +0000 Subject: [PATCH] Merged from trunk -r13929:13939 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit | 13930 | Convert paths to unicode when working with CONTENTS. | | zmedico | | | 13931 | Add unicode conversions in various logging code. | | zmedico | | | 13932 | Convert unicode if necessary before writing to stdout. | | zmedico | | | 13933 | Inside _post_src_install_uid_fix() (since we're already | | zmedico | walking the files here), compute total size of installed | | | files and save it as SIZE in the vardb. This will be useful | | | for the packagekit backend. | | 13934 | Bug #277544 - Add missing -c switch in bzip2 arguments. | | zmedico | Thanks to Michał Górny for the | | | patch. | | 13935 | Bug #277537 - Synchronize order of PATH elements with the | | zmedico | order defined by /etc/profile (promote paths under /usr to | | | the front). Thanks to Michał Górny | | | . | | 13936 | Implement nonfatal(). | | arfrever | | | 13937 | Bug #280521 - Update selinux support to use the libselinux | | zmedico | swig wrapper api instead of python-selinux. Thanks to Chris | | | PeBenito for the initial patch which | | | I've tweaked with whitespace changes and unicode support. | | 13938 | Filter PORTAGE_NONFATAL from the environment where | | zmedico | appropriate. | | 13939 | Add a new portage.util.shlex_split() function to wrap all | | zmedico | shlex.split() calls, since shlex.split() doesn't behave | | | well with unicode strings. | svn path=/main/branches/prefix/; revision=14031 --- bin/ebuild.sh | 4 +- bin/isolated-functions.sh | 18 +++++- bin/misc-functions.sh | 2 +- bin/portageq | 12 ++-- pym/_emerge/BinpkgFetcher.py | 4 +- pym/_emerge/JobStatusDisplay.py | 23 ++++---- pym/_emerge/Scheduler.py | 4 ++ pym/_emerge/actions.py | 7 +-- pym/_emerge/emergelog.py | 7 +++ pym/_emerge/main.py | 3 +- pym/portage/__init__.py | 42 +++++++------- pym/portage/_selinux.py | 97 ++++++++++++++++++++++++++++++++- pym/portage/dbapi/vartree.py | 27 +++++++-- pym/portage/elog/messages.py | 18 ++++-- pym/portage/getbinpkg.py | 6 +- pym/portage/sets/dbapi.py | 4 +- pym/portage/sets/libs.py | 4 +- pym/portage/util.py | 13 +++++ pym/repoman/utilities.py | 5 +- 19 files changed, 230 insertions(+), 70 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 8f42772f0..d8ed4ce5f 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -22,7 +22,7 @@ ROOTPATH=${ROOTPATH##:} ROOTPATH=${ROOTPATH%%:} PREROOTPATH=${PREROOTPATH##:} PREROOTPATH=${PREROOTPATH%%:} -#PATH=$PORTAGE_BIN_PATH/ebuild-helpers:$PREROOTPATH${PREROOTPATH:+:}/usr/local/sbin:/sbin:/usr/sbin:/usr/local/bin:/bin:/usr/bin${ROOTPATH:+:}$ROOTPATH +#PATH=$PORTAGE_BIN_PATH/ebuild-helpers:$PREROOTPATH${PREROOTPATH:+:}/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin${ROOTPATH:+:}$ROOTPATH # PREFIX: we deviate in path order, should we split up DEFAULT_PATH? PATH="$PORTAGE_BIN_PATH/ebuild-helpers:$PREROOTPATH${PREROOTPATH:+:}${DEFAULT_PATH}:${ROOTPATH:+:}$ROOTPATH" export PATH @@ -1911,7 +1911,7 @@ ebuild_main() { ;; esac - #PATH=$ebuild_helpers_path:$PREROOTPATH${PREROOTPATH:+:}/usr/local/sbin:/sbin:/usr/sbin:/usr/local/bin:/bin:/usr/bin${ROOTPATH:+:}$ROOTPATH + #PATH=$ebuild_helpers_path:$PREROOTPATH${PREROOTPATH:+:}/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin${ROOTPATH:+:}$ROOTPATH # PREFIX: same deviation as at the top of this file PATH="$ebuild_helpers_path:$PREROOTPATH${PREROOTPATH:+:}${DEFAULT_PATH}${ROOTPATH:+:}$ROOTPATH" unset ebuild_helpers_path diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 20ab0f50f..c3821d606 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -53,7 +53,23 @@ dump_trace() { done } +nonfatal() { + if has "${EAPI:-0}" 0 1 2; then + die "$FUNCNAME() not supported in this EAPI" + fi + if [[ $# -lt 1 ]]; then + die "$FUNCNAME(): Missing argument" + fi + + PORTAGE_NONFATAL=1 "$@" +} + die() { + if [[ $PORTAGE_NONFATAL -eq 1 ]]; then + echo -e " $WARN*$NORMAL ${FUNCNAME[1]}: WARNING: $@" >&2 + return 1 + fi + set +e if [ -n "${QA_INTERCEPTORS}" ] ; then # die was called from inside inherit. We need to clean up @@ -537,7 +553,7 @@ save_ebuild_env() { PORTAGE_COLORMAP PORTAGE_CONFIGROOT PORTAGE_DEBUG \ PORTAGE_DEPCACHEDIR PORTAGE_GID PORTAGE_INST_GID \ PORTAGE_INST_UID PORTAGE_LOG_FILE PORTAGE_MASTER_PID \ - PORTAGE_QUIET \ + PORTAGE_NONFATAL PORTAGE_QUIET \ PORTAGE_REPO_NAME PORTAGE_RESTRICT PORTAGE_UPDATE_ENV \ PORTAGE_VERBOSE PORTAGE_WORKDIR_MODE PORTDIR \ PORTDIR_OVERLAY ${!PORTAGE_SANDBOX_*} PREROOTPATH \ diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 4713a774b..481ac149e 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -895,7 +895,7 @@ dyn_package() { PORTAGE_BINPKG_TMPFILE="${PKGDIR}/${CATEGORY}/${PF}.tbz2" mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed" tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${D}" . | \ - bzip2 -f > "$PORTAGE_BINPKG_TMPFILE" + bzip2 -cf > "$PORTAGE_BINPKG_TMPFILE" assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'" PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ python -c "from portage import xpak; t=xpak.tbz2('${PORTAGE_BINPKG_TMPFILE}'); t.recompose('${PORTAGE_BUILDDIR}/build-info')" diff --git a/bin/portageq b/bin/portageq index b8ff3e8aa..7d967098c 100755 --- a/bin/portageq +++ b/bin/portageq @@ -251,12 +251,12 @@ def is_protected(argv): err.flush() return 2 - import shlex from portage.util import ConfigProtect settings = portage.settings - protect = shlex.split(settings.get("CONFIG_PROTECT", "")) - protect_mask = shlex.split(settings.get("CONFIG_PROTECT_MASK", "")) + protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) + protect_mask = portage.util.shlex_split( + settings.get("CONFIG_PROTECT_MASK", "")) protect_obj = ConfigProtect(root, protect, protect_mask) if protect_obj.isprotected(f): @@ -284,12 +284,12 @@ def filter_protected(argv): except OSError: pass - import shlex from portage.util import ConfigProtect settings = portage.settings - protect = shlex.split(settings.get("CONFIG_PROTECT", "")) - protect_mask = shlex.split(settings.get("CONFIG_PROTECT_MASK", "")) + protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) + protect_mask = portage.util.shlex_split( + settings.get("CONFIG_PROTECT_MASK", "")) protect_obj = ConfigProtect(root, protect, protect_mask) protected = 0 diff --git a/pym/_emerge/BinpkgFetcher.py b/pym/_emerge/BinpkgFetcher.py index 47c114dad..90320c017 100644 --- a/pym/_emerge/BinpkgFetcher.py +++ b/pym/_emerge/BinpkgFetcher.py @@ -5,7 +5,7 @@ from _emerge.SpawnProcess import SpawnProcess import urlparse import sys -import shlex + # for an explanation on this logic, see pym/_emerge/__init__.py import os import sys @@ -83,7 +83,7 @@ class BinpkgFetcher(SpawnProcess): fetch_env = dict(settings.iteritems()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ - for x in shlex.split(fcmd)] + for x in portage.util.shlex_split(fcmd)] if self.fd_pipes is None: self.fd_pipes = {} diff --git a/pym/_emerge/JobStatusDisplay.py b/pym/_emerge/JobStatusDisplay.py index 63c91e314..507a5057c 100644 --- a/pym/_emerge/JobStatusDisplay.py +++ b/pym/_emerge/JobStatusDisplay.py @@ -75,6 +75,13 @@ class JobStatusDisplay(object): temporarily overridden stdout.""" return sys.stdout + def _write(self, s): + if sys.hexversion < 0x3000000 and isinstance(s, unicode): + # avoid potential UnicodeEncodeError + s = s.encode('utf_8', 'replace') + self.out.write(s) + self.out.flush() + def _init_term(self): """ Initialize term control codes. @@ -112,23 +119,19 @@ class JobStatusDisplay(object): return ">>> %s" % msg def _erase(self): - self.out.write( + self._write( self._term_codes['carriage_return'] + \ self._term_codes['clr_eol']) - self.out.flush() self._displayed = False def _display(self, line): - self.out.write(line) - self.out.flush() + self._write(line) self._displayed = True def _update(self, msg): - out = self.out if not self._isatty: - out.write(self._format_msg(msg) + self._term_codes['newline']) - self.out.flush() + self._write(self._format_msg(msg) + self._term_codes['newline']) self._displayed = True return @@ -144,8 +147,7 @@ class JobStatusDisplay(object): if self._isatty and self._displayed: self._erase() - self.out.write(self._format_msg(msg) + self._term_codes['newline']) - self.out.flush() + self._write(self._format_msg(msg) + self._term_codes['newline']) self._displayed = False if was_displayed: @@ -159,8 +161,7 @@ class JobStatusDisplay(object): object.__setattr__(self, name, 0) if self._displayed: - self.out.write(self._term_codes['newline']) - self.out.flush() + self._write(self._term_codes['newline']) self._displayed = False def __setattr__(self, name, value): diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index 6016e9478..e82ade829 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -479,6 +479,10 @@ class Scheduler(PollScheduler): return self._pkg(cpv, type_name, root_config, installed=installed) def _append_to_log_path(self, log_path, msg): + + if not isinstance(msg, unicode): + msg = unicode(msg, encoding='utf_8', errors='replace') + f = codecs.open(log_path, mode='a', encoding='utf_8', errors='replace') try: diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 4fed7955b..397c2d95a 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -9,7 +9,6 @@ import os import platform import pwd import re -import shlex import signal import socket import stat @@ -1835,8 +1834,8 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # defaults. portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1) - rsync_opts.extend( - shlex.split(settings.get("PORTAGE_RSYNC_OPTS",""))) + rsync_opts.extend(portage.util.shlex_split( + settings.get("PORTAGE_RSYNC_OPTS", ""))) for opt in ("--recursive", "--times"): if opt not in rsync_opts: portage.writemsg(yellow("WARNING:") + " adding required option " + \ @@ -1914,7 +1913,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): user_name="" updatecache_flg=True all_rsync_opts = set(rsync_opts) - extra_rsync_opts = shlex.split( + extra_rsync_opts = portage.util.shlex_split( settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")) all_rsync_opts.update(extra_rsync_opts) family = socket.AF_INET diff --git a/pym/_emerge/emergelog.py b/pym/_emerge/emergelog.py index 67aa3f00e..5d2256061 100644 --- a/pym/_emerge/emergelog.py +++ b/pym/_emerge/emergelog.py @@ -22,6 +22,13 @@ from portage.const import EPREFIX _emerge_log_dir = EPREFIX + '/var/log' def emergelog(xterm_titles, mystr, short_msg=None): + + if not isinstance(mystr, unicode): + mystr = unicode(mystr, encoding='utf_8', errors='replace') + + if short_msg is not None and not isinstance(short_msg, unicode): + short_msg = unicode(short_msg, encoding='utf_8', errors='replace') + if xterm_titles and short_msg: if "HOSTNAME" in os.environ: short_msg = os.environ["HOSTNAME"]+": "+short_msg diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 5edb4c5ab..05eb91080 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -3,7 +3,6 @@ # $Id$ import logging -import shlex import signal import sys import textwrap @@ -792,7 +791,7 @@ def ionice(settings): ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND") if ionice_cmd: - ionice_cmd = shlex.split(ionice_cmd) + ionice_cmd = portage.util.shlex_split(ionice_cmd) if not ionice_cmd: return diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 9f2a82e30..50a1a129e 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1061,7 +1061,8 @@ class config(object): "A", "AA", "CATEGORY", "DEPEND", "DESCRIPTION", "EAPI", "EBUILD_PHASE", "EMERGE_FROM", "HOMEPAGE", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PF", "PKGUSE", - "PORTAGE_CONFIGROOT", "PORTAGE_IUSE", "PORTAGE_REPO_NAME", + "PORTAGE_CONFIGROOT", "PORTAGE_IUSE", + "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME", "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "RESTRICT", "ROOT", "SLOT", "SRC_URI", "EPREFIX", "EROOT" ] @@ -3869,9 +3870,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", # no digests because fetch was not called for a specific package mydigests = {} - import shlex ro_distdirs = [x for x in \ - shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \ + util.shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \ if os.path.isdir(x)] fsmirrors = [] @@ -4429,8 +4429,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", "URI": loc, "FILE": myfile } - import shlex - myfetch = shlex.split(locfetch) + + myfetch = util.shlex_split(locfetch) myfetch = [varexpand(x, mydict=variables) for x in myfetch] myret = -1 try: @@ -5191,10 +5191,14 @@ def _post_src_install_uid_fix(mysettings): # os.path.join when called by os.walk. destdir = destdir.encode('utf_8', 'replace') + size = 0 + for parent, dirs, files in os.walk(destdir): for fname in chain(dirs, files): fpath = os.path.join(parent, fname) mystat = os.lstat(fpath) + if stat.S_ISREG(mystat.st_mode): + size += mystat.st_size if mystat.st_uid != portage_uid and \ mystat.st_gid != portage_gid: continue @@ -5208,6 +5212,9 @@ def _post_src_install_uid_fix(mysettings): mode=mystat.st_mode, stat_cached=mystat, follow_links=False) + open(os.path.join(mysettings['PORTAGE_BUILDDIR'], + 'build-info', 'SIZE'), 'w').write(str(size) + '\n') + if bsd_chflags: # Restore all of the flags saved above. os.system("mtree -e -p %s -U -k flags < %s > /dev/null" % \ @@ -6724,8 +6731,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): os.unlink(dest) if selinux_enabled: - sid = selinux.get_lsid(src) - selinux.secure_symlink(target,dest,sid) + selinux.symlink(target, dest, src) else: os.symlink(target,dest) lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) @@ -6780,7 +6786,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev): try: if selinux_enabled: - ret=selinux.secure_rename(src,dest) + ret = selinux.rename(src, dest) else: ret=os.rename(src,dest) renamefailed=0 @@ -6798,8 +6804,8 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if stat.S_ISREG(sstat[stat.ST_MODE]): try: # For safety copy then move it over. if selinux_enabled: - selinux.secure_copy(src,dest+"#new") - selinux.secure_rename(dest+"#new",dest) + selinux.copyfile(src, dest + "#new") + selinux.rename(dest + "#new", dest) else: shutil.copyfile(src,dest+"#new") os.rename(dest+"#new",dest) @@ -6812,15 +6818,13 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, return None else: #we don't yet handle special, so we need to fall back to /bin/mv - if selinux_enabled: - a=commands.getstatusoutput(MOVE_BINARY+" -c -f "+"'"+src+"' '"+dest+"'") - else: - a=commands.getstatusoutput(MOVE_BINARY+" -f "+"'"+src+"' '"+dest+"'") - if a[0]!=0: - print "!!! Failed to move special file:" - print "!!! '"+src+"' to '"+dest+"'" - print "!!!",a - return None # failure + a = commands.getstatusoutput("%s -f %s %s" % \ + (MOVE_BINARY, _shell_quote(src), _shell_quote(dest))) + if a[0] != os.EX_OK: + writemsg("!!! Failed to move special file:\n", noiselevel=-1) + writemsg("!!! '%s' to '%s'\n" % (src, dest), noiselevel=-1) + writemsg("!!! %s\n" % a, noiselevel=-1) + return None # failure try: if didcopy: if stat.S_ISLNK(sstat[stat.ST_MODE]): diff --git a/pym/portage/_selinux.py b/pym/portage/_selinux.py index b5afd9233..2a50f7434 100644 --- a/pym/portage/_selinux.py +++ b/pym/portage/_selinux.py @@ -2,7 +2,98 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import os import selinux -from selinux import is_selinux_enabled -from selinux_aux import setexec, secure_symlink, secure_rename, \ - secure_copy, secure_mkdir, getcontext, get_sid, get_lsid +import shutil +from selinux import is_selinux_enabled, getfilecon, lgetfilecon + +def copyfile(src, dest): + if isinstance(src, unicode): + src = src.encode('utf_8', 'replace') + if isinstance(dest, unicode): + dest = dest.encode('utf_8', 'replace') + (rc, ctx) = selinux.lgetfilecon(src) + if rc < 0: + raise OSError("copyfile: Failed getting context of \"%s\"." % src) + + setfscreate(ctx) + try: + shutil.copyfile(src, dest) + finally: + setfscreate() + +def getcontext(): + (rc, ctx) = selinux.getcon() + if rc < 0: + raise OSError("getcontext: Failed getting current process context.") + + return ctx + +def mkdir(target, refdir): + if isinstance(target, unicode): + target = target.encode('utf_8', 'replace') + if isinstance(refdir, unicode): + refdir = refdir.encode('utf_8', 'replace') + (rc, ctx) = selinux.getfilecon(refdir) + if rc < 0: + raise OSError( + "mkdir: Failed getting context of reference directory \"%s\"." \ + % refdir) + + setfscreatecon(ctx) + try: + os.mkdir(target) + finally: + setfscreatecon() + +def rename(src, dest): + if isinstance(src, unicode): + src = src.encode('utf_8', 'replace') + if isinstance(dest, unicode): + dest = dest.encode('utf_8', 'replace') + (rc, ctx) = selinux.lgetfilecon(src) + if rc < 0: + raise OSError("rename: Failed getting context of \"%s\"." % src) + + setfscreate(ctx) + try: + os.rename(src,dest) + finally: + setfscreate() + +def setexec(ctx="\n"): + if selinux.setexeccon(ctx) < 0: + raise OSError("setexec: Failed setting exec() context \"%s\"." % ctx) + +def setfscreate(ctx="\n"): + if selinux.setfscreatecon(ctx) < 0: + raise OSError( + "setfscreate: Failed setting fs create context \"%s\"." % ctx) + +def spawn(selinux_type, spawn_func, mycommand, opt_name=None, **keywords): + con = getcontext().split(":") + con[2] = selinux_type + setexec(":".join(con)) + try: + return spawn_func(mycommand, opt_name=opt_name, **keywords) + finally: + setexec() + +def symlink(target, link, reflnk): + if isinstance(target, unicode): + target = target.encode('utf_8', 'replace') + if isinstance(link, unicode): + link = link.encode('utf_8', 'replace') + if isinstance(reflnk, unicode): + reflnk = reflnk.encode('utf_8', 'replace') + (rc, ctx) = selinux.lgetfilecon(reflnk) + if rc < 0: + raise OSError( + "symlink: Failed getting context of reference symlink \"%s\"." \ + % reflnk) + + setfscreate(ctx) + try: + os.symlink(target, link) + finally: + setfscreate() diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 7b201e9b0..a64b15fa8 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -41,7 +41,6 @@ from portage.cache.mappings import slot_dict_class import codecs import os, re, shutil, stat, errno, copy, subprocess import logging -import shlex import sys from itertools import izip @@ -2047,11 +2046,16 @@ class vardbapi(dbapi): if not hasattr(pkg, "getcontents"): pkg = self._dblink(pkg) root = self.root + if not isinstance(root, unicode): + root = unicode(root, encoding='utf_8', errors='replace') root_len = len(root) - 1 new_contents = pkg.getcontents().copy() removed = 0 for filename in paths: + if not isinstance(filename, unicode): + filename = unicode(filename, + encoding='utf_8', errors='replace') filename = normalize_path(filename) if relative_paths: relative_filename = filename @@ -2498,8 +2502,9 @@ class dblink(object): self.myroot=myroot protect_obj = ConfigProtect(myroot, - shlex.split(mysettings.get("CONFIG_PROTECT", "")), - shlex.split(mysettings.get("CONFIG_PROTECT_MASK", ""))) + portage.util.shlex_split(mysettings.get("CONFIG_PROTECT", "")), + portage.util.shlex_split( + mysettings.get("CONFIG_PROTECT_MASK", ""))) self.updateprotect = protect_obj.updateprotect self.isprotected = protect_obj.isprotected self._installed_instance = None @@ -3227,6 +3232,14 @@ class dblink(object): if the file is not owned by this package. """ + if not isinstance(filename, unicode): + filename = unicode(filename, + encoding='utf_8', errors='replace') + + if not isinstance(destroot, unicode): + destroot = unicode(destroot, + encoding='utf_8', errors='replace') + destfile = normalize_path( os.path.join(destroot, filename.lstrip(os.path.sep))) @@ -3415,6 +3428,8 @@ class dblink(object): new_contents = self.getcontents().copy() old_contents = self._installed_instance.getcontents() for f in sorted(preserve_paths): + if not isinstance(f, unicode): + f = unicode(f, encoding='utf_8', errors='replace') f_abs = os.path.join(root, f.lstrip(os.sep)) contents_entry = old_contents.get(f_abs) if contents_entry is None: @@ -3601,7 +3616,8 @@ class dblink(object): def _collision_protect(self, srcroot, destroot, mypkglist, mycontents): collision_ignore = set([normalize_path(myignore) for myignore in \ - shlex.split(self.settings.get("COLLISION_IGNORE", ""))]) + portage.util.shlex_split( + self.settings.get("COLLISION_IGNORE", ""))]) # For collisions with preserved libraries, the current package # will assume ownership and the libraries will be unregistered. @@ -4443,6 +4459,9 @@ class dblink(object): mydest = join(destroot, offset, x) # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") myrealdest = join(sep, offset, x) + if not isinstance(myrealdest, unicode): + myrealdest = unicode(myrealdest, + encoding='utf_8', errors='replace') # stat file once, test using S_* macros many times (faster that way) mystat = os.lstat(mysrc) mymode = mystat[stat.ST_MODE] diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py index 9832f2660..3b44e0fe8 100644 --- a/pym/portage/elog/messages.py +++ b/pym/portage/elog/messages.py @@ -79,15 +79,23 @@ def _elog_base(level, msg, phase="other", key=None, color=None, out=None): global _msgbuffer - if color == None: + if out is None: + out = sys.stdout + + if color is None: color = "GOOD" + if not isinstance(msg, unicode): + msg = unicode(msg, encoding='utf_8', errors='replace') + formatted_msg = colorize(color, " * ") + msg + "\n" - if out is None: - sys.stdout.write(formatted_msg) - else: - out.write(formatted_msg) + if sys.hexversion < 0x3000000 and \ + out in (sys.stdout, sys.stderr) and isinstance(formatted_msg, unicode): + # avoid potential UnicodeEncodeError + formatted_msg = formatted_msg.encode('utf_8', 'replace') + + out.write(formatted_msg) if key not in _msgbuffer: _msgbuffer[key] = {} diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index bf616637b..70cc2e513 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -5,7 +5,7 @@ from portage.output import colorize from portage.cache.mappings import slot_dict_class -import portage.xpak +import portage import HTMLParser import sys import os @@ -401,10 +401,10 @@ def file_get(baseurl,dest,conn=None,fcmd=None): "URI": baseurl, "FILE": os.path.basename(baseurl) } - import shlex + from portage.util import varexpand from portage.process import spawn - myfetch = shlex.split(fcmd) + myfetch = portage.util.shlex_split(fcmd) myfetch = [varexpand(x, mydict=variables) for x in myfetch] fd_pipes= { 0:sys.stdin.fileno(), diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py index d6e78bfb6..0f82bd1c9 100644 --- a/pym/portage/sets/dbapi.py +++ b/pym/portage/sets/dbapi.py @@ -7,6 +7,7 @@ from portage.dep import Atom from portage.localization import _ from portage.sets.base import PackageSet from portage.sets import SetConfigError, get_boolean +import portage __all__ = ["CategorySet", "DowngradeSet", "EverythingSet", "OwnerSet", "VariableSet"] @@ -84,9 +85,8 @@ class OwnerSet(PackageSet): if not "files" in options: raise SetConfigError(_("no files given")) - import shlex return cls(vardb=trees["vartree"].dbapi, - files=frozenset(shlex.split(options["files"]))) + files=frozenset(portage.util.shlex_split(options["files"]))) singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/sets/libs.py b/pym/portage/sets/libs.py index 806cb63f6..cbaf57c27 100644 --- a/pym/portage/sets/libs.py +++ b/pym/portage/sets/libs.py @@ -6,6 +6,7 @@ from portage.localization import _ from portage.sets.base import PackageSet from portage.sets import get_boolean from portage.versions import catpkgsplit +import portage class LibraryConsumerSet(PackageSet): _operations = ["merge", "unmerge"] @@ -47,8 +48,7 @@ class LibraryFileConsumerSet(LibraryConsumerSet): self._setAtoms(self.mapPathsToAtoms(consumers)) def singleBuilder(cls, options, settings, trees): - import shlex - files = tuple(shlex.split(options.get("files", ""))) + files = tuple(portage.util.shlex_split(options.get("files", ""))) if not files: raise SetConfigError(_("no files given")) debug = get_boolean(options, "debug", False) diff --git a/pym/portage/util.py b/pym/portage/util.py index 6e11c46f2..32ac48884 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -351,6 +351,19 @@ def writedict(mydict,myfilename,writekey=True): return 0 return 1 +def shlex_split(s): + """ + This is equivalent to shlex.split but it temporarily encodes unicode + strings to bytes since shlex.split() doesn't handle unicode strings. + """ + is_unicode = isinstance(s, unicode) + if is_unicode: + s = s.encode('utf_8', 'replace') + rval = shlex.split(s) + if is_unicode: + rval = [unicode(x, encoding='utf_8', errors='replace') for x in rval] + return rval + class _tolerant_shlex(shlex.shlex): def sourcehook(self, newfile): try: diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py index 6003deeac..9ea5ff467 100644 --- a/pym/repoman/utilities.py +++ b/pym/repoman/utilities.py @@ -238,7 +238,7 @@ def format_qa_output(formatter, stats, fails, dofull, dofail, options, qawarning def editor_is_executable(editor): """ Given an EDITOR string, validate that it refers to - an executable. This uses shlex.split() to split the + an executable. This uses shlex_split() to split the first component and do a PATH lookup if necessary. @param editor: An EDITOR value from the environment. @@ -246,8 +246,7 @@ def editor_is_executable(editor): @rtype: bool @returns: True if an executable is found, False otherwise. """ - import shlex - editor_split = shlex.split(editor) + editor_split = util.shlex_split(editor) if not editor_split: return False filename = editor_split[0] -- 2.26.2