Add a new portage.util.shlex_split() function to wrap all shlex.split() calls,

since shlex.split() doesn't behave well with unicode strings.

svn path=/main/trunk/; revision=13939

diff --git a/bin/portageq b/bin/portageq
index 825152911b3366564bfd52bd64620d302a9aa9e2..084c20cbb3a201e382cb527892d5063346f2d1a0 100755 (executable)
--- a/bin/portageq
+++ b/bin/portageq
@@ -251,12 +251,12 @@ def is_protected(argv):
                err.flush()
                return 2
 
-       import shlex
        from portage.util import ConfigProtect
 
        settings = portage.settings
-       protect = shlex.split(settings.get("CONFIG_PROTECT", ""))
-       protect_mask = shlex.split(settings.get("CONFIG_PROTECT_MASK", ""))
+       protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
+       protect_mask = portage.util.shlex_split(
+               settings.get("CONFIG_PROTECT_MASK", ""))
        protect_obj = ConfigProtect(root, protect, protect_mask)
 
        if protect_obj.isprotected(f):
@@ -284,12 +284,12 @@ def filter_protected(argv):
        except OSError:
                pass
 
-       import shlex
        from portage.util import ConfigProtect
 
        settings = portage.settings
-       protect = shlex.split(settings.get("CONFIG_PROTECT", ""))
-       protect_mask = shlex.split(settings.get("CONFIG_PROTECT_MASK", ""))
+       protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
+       protect_mask = portage.util.shlex_split(
+               settings.get("CONFIG_PROTECT_MASK", ""))
        protect_obj = ConfigProtect(root, protect, protect_mask)
 
        protected = 0

diff --git a/pym/_emerge/BinpkgFetcher.py b/pym/_emerge/BinpkgFetcher.py
index 977ccb4d8e1e8b66cdeea8d8d60c869201d4239a..f3b11bb6ce5a61cebd74fd0b23ba77e7c04a3e96 100644 (file)
--- a/pym/_emerge/BinpkgFetcher.py
+++ b/pym/_emerge/BinpkgFetcher.py
@@ -5,7 +5,7 @@
 from _emerge.SpawnProcess import SpawnProcess
 import urlparse
 import sys
-import shlex
+
 try:
        import portage
 except ImportError:
@@ -83,7 +83,7 @@ class BinpkgFetcher(SpawnProcess):
 
                fetch_env = dict(settings.iteritems())
                fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \
-                       for x in shlex.split(fcmd)]
+                       for x in portage.util.shlex_split(fcmd)]
 
                if self.fd_pipes is None:
                        self.fd_pipes = {}

diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 94ec37e8afd972bb600eda20214be9d140de48db..4632c10f24fb6369104928913e03e3bf9963fa3c 100644 (file)
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -9,7 +9,6 @@ import os
 import platform
 import pwd
 import re
-import shlex
 import signal
 import socket
 import stat
@@ -1833,8 +1832,8 @@ def action_sync(settings, trees, mtimedb, myopts, myaction):
                        # defaults.
 
                        portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-                       rsync_opts.extend(
-                               shlex.split(settings.get("PORTAGE_RSYNC_OPTS","")))
+                       rsync_opts.extend(portage.util.shlex_split(
+                               settings.get("PORTAGE_RSYNC_OPTS", "")))
                        for opt in ("--recursive", "--times"):
                                if opt not in rsync_opts:
                                        portage.writemsg(yellow("WARNING:") + " adding required option " + \
@@ -1912,7 +1911,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction):
                        user_name=""
                updatecache_flg=True
                all_rsync_opts = set(rsync_opts)
-               extra_rsync_opts = shlex.split(
+               extra_rsync_opts = portage.util.shlex_split(
                        settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
                all_rsync_opts.update(extra_rsync_opts)
                family = socket.AF_INET

diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py
index e0e66d8ac783e5c56fbab697c24c9c11f87eb422..78ac776f516fe72820488da46fd958281debaba4 100644 (file)
--- a/pym/_emerge/main.py
+++ b/pym/_emerge/main.py
@@ -3,7 +3,6 @@
 # $Id$
 
 import logging
-import shlex
 import signal
 import sys
 import textwrap
@@ -779,7 +778,7 @@ def ionice(settings):
 
        ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND")
        if ionice_cmd:
-               ionice_cmd = shlex.split(ionice_cmd)
+               ionice_cmd = portage.util.shlex_split(ionice_cmd)
        if not ionice_cmd:
                return
 

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 5b18fd05612cf4721f756404a7b089ae5bc1c7e1..df7961b14fae98cdcd95871a5fd1c06f068ad10e 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -3848,9 +3848,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                # no digests because fetch was not called for a specific package
                mydigests = {}
 
-       import shlex
        ro_distdirs = [x for x in \
-               shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
+               util.shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
                if os.path.isdir(x)]
 
        fsmirrors = []
@@ -4408,8 +4407,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                "URI":     loc,
                                                "FILE":    myfile
                                        }
-                                       import shlex
-                                       myfetch = shlex.split(locfetch)
+
+                                       myfetch = util.shlex_split(locfetch)
                                        myfetch = [varexpand(x, mydict=variables) for x in myfetch]
                                        myret = -1
                                        try:

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 52d48a70fade7a61d0ed3d9fb65b8195590906cd..dc1180c5437fcbcae4008356367320236e608b02 100644 (file)
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -40,7 +40,6 @@ from portage.cache.mappings import slot_dict_class
 import codecs
 import os, re, shutil, stat, errno, copy, subprocess
 import logging
-import shlex
 import sys
 from itertools import izip
 
@@ -1809,8 +1808,9 @@ class dblink(object):
 
                self.myroot=myroot
                protect_obj = ConfigProtect(myroot,
-                       shlex.split(mysettings.get("CONFIG_PROTECT", "")),
-                       shlex.split(mysettings.get("CONFIG_PROTECT_MASK", "")))
+                       portage.util.shlex_split(mysettings.get("CONFIG_PROTECT", "")),
+                       portage.util.shlex_split(
+                       mysettings.get("CONFIG_PROTECT_MASK", "")))
                self.updateprotect = protect_obj.updateprotect
                self.isprotected = protect_obj.isprotected
                self._installed_instance = None
@@ -2907,7 +2907,8 @@ class dblink(object):
 
        def _collision_protect(self, srcroot, destroot, mypkglist, mycontents):
                        collision_ignore = set([normalize_path(myignore) for myignore in \
-                               shlex.split(self.settings.get("COLLISION_IGNORE", ""))])
+                               portage.util.shlex_split(
+                               self.settings.get("COLLISION_IGNORE", ""))])
 
                        # For collisions with preserved libraries, the current package
                        # will assume ownership and the libraries will be unregistered.

diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py
index 29e2ff4f871b6ef64106800a42175231a66ecb93..75d997e547d679d9c07f13b24f7c7ba9a609a5b6 100644 (file)
--- a/pym/portage/getbinpkg.py
+++ b/pym/portage/getbinpkg.py
@@ -5,7 +5,7 @@
 
 from portage.output import colorize
 from portage.cache.mappings import slot_dict_class
-import portage.xpak
+import portage
 import HTMLParser
 import sys
 import os
@@ -400,10 +400,10 @@ def file_get(baseurl,dest,conn=None,fcmd=None):
                "URI":     baseurl,
                "FILE":    os.path.basename(baseurl)
        }
-       import shlex
+
        from portage.util import varexpand
        from portage.process import spawn
-       myfetch = shlex.split(fcmd)
+       myfetch = portage.util.shlex_split(fcmd)
        myfetch = [varexpand(x, mydict=variables) for x in myfetch]
        fd_pipes= {
                0:sys.stdin.fileno(),

diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py
index d6e78bfb62c2cde3946d2e9badc74022346b2792..0f82bd1c9b547501cc533ca7a3854e936f28f817 100644 (file)
--- a/pym/portage/sets/dbapi.py
+++ b/pym/portage/sets/dbapi.py
@@ -7,6 +7,7 @@ from portage.dep import Atom
 from portage.localization import _
 from portage.sets.base import PackageSet
 from portage.sets import SetConfigError, get_boolean
+import portage
 
 __all__ = ["CategorySet", "DowngradeSet",
        "EverythingSet", "OwnerSet", "VariableSet"]
@@ -84,9 +85,8 @@ class OwnerSet(PackageSet):
                if not "files" in options:
                        raise SetConfigError(_("no files given"))
 
-               import shlex
                return cls(vardb=trees["vartree"].dbapi,
-                       files=frozenset(shlex.split(options["files"])))
+                       files=frozenset(portage.util.shlex_split(options["files"])))
 
        singleBuilder = classmethod(singleBuilder)
 

diff --git a/pym/portage/sets/libs.py b/pym/portage/sets/libs.py
index 806cb63f6faf7e74a9877625d71fd0e1b619e373..cbaf57c277ec339f5f89612fb0e7a24c6431894e 100644 (file)
--- a/pym/portage/sets/libs.py
+++ b/pym/portage/sets/libs.py
@@ -6,6 +6,7 @@ from portage.localization import _
 from portage.sets.base import PackageSet
 from portage.sets import get_boolean
 from portage.versions import catpkgsplit
+import portage
 
 class LibraryConsumerSet(PackageSet):
        _operations = ["merge", "unmerge"]
@@ -47,8 +48,7 @@ class LibraryFileConsumerSet(LibraryConsumerSet):
                self._setAtoms(self.mapPathsToAtoms(consumers))
 
        def singleBuilder(cls, options, settings, trees):
-               import shlex
-               files = tuple(shlex.split(options.get("files", "")))
+               files = tuple(portage.util.shlex_split(options.get("files", "")))
                if not files:
                        raise SetConfigError(_("no files given"))
                debug = get_boolean(options, "debug", False)

diff --git a/pym/portage/util.py b/pym/portage/util.py
index 5b860218d21e4921d5e6a4fde118f9422aee83d4..0a8035de6b3db74b0dbd981a0dfd6312b33ae104 100644 (file)
--- a/pym/portage/util.py
+++ b/pym/portage/util.py
@@ -350,6 +350,19 @@ def writedict(mydict,myfilename,writekey=True):
                return 0
        return 1
 
+def shlex_split(s):
+       """
+       This is equivalent to shlex.split but it temporarily encodes unicode
+       strings to bytes since shlex.split() doesn't handle unicode strings.
+       """
+       is_unicode = isinstance(s, unicode)
+       if is_unicode:
+               s = s.encode('utf_8', 'replace')
+       rval = shlex.split(s)
+       if is_unicode:
+               rval = [unicode(x, encoding='utf_8', errors='replace') for x in rval]
+       return rval
+
 class _tolerant_shlex(shlex.shlex):
        def sourcehook(self, newfile):
                try:

diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py
index 6003deeac1bac006741c6e7b9b6bbdb7878e2b6d..9ea5ff46705aeb77904a4bab137866d1700ce6ba 100644 (file)
--- a/pym/repoman/utilities.py
+++ b/pym/repoman/utilities.py
@@ -238,7 +238,7 @@ def format_qa_output(formatter, stats, fails, dofull, dofail, options, qawarning
 def editor_is_executable(editor):
        """
        Given an EDITOR string, validate that it refers to
-       an executable. This uses shlex.split() to split the
+       an executable. This uses shlex_split() to split the
        first component and do a PATH lookup if necessary.
 
        @param editor: An EDITOR value from the environment.
@@ -246,8 +246,7 @@ def editor_is_executable(editor):
        @rtype: bool
        @returns: True if an executable is found, False otherwise.
        """
-       import shlex
-       editor_split = shlex.split(editor)
+       editor_split = util.shlex_split(editor)
        if not editor_split:
                return False
        filename = editor_split[0]