Python 3.1: pass bytes to Popen

For Python 3.1, it's possible to pass bytes to Popen as long as the
executable has an absolute path, since otherwise we would trigger a
TypeError in os._execvp (see http://bugs.python.org/issue8513).

Note that Python <=3.1.3 produces the following message:

  TypeError: expected an object with the buffer interface

Later 3.1.x releases produce a different message:

  TypeError: startswith first arg must be bytes or a tuple of bytes, not str

The difference in messages is due to os.path.join() implementation
changes, but both errors are triggered by the same underlying bug in
os._execvp() which was fixed by using fsencode() in this hunk:

--- a/Lib/os.py
+++ b/Lib/os.py
@@ -355,7 +355,11 @@ def _execvpe(file, args, env=None):
         return
     last_exc = saved_exc = None
     saved_tb = None
-    for dir in get_exec_path(env):
+    path_list = get_exec_path(env)
+    if name != 'nt':
+        file = fsencode(file)
+        path_list = map(fsencode, path_list)
+    for dir in path_list:
         fullname = path.join(dir, file)

diff --git a/bin/install.py b/bin/install.py
index cce68c34e20bbee7303248077adf6c9443e1fe92..02083a18178c850a6bcd6928d6996f1adf50e94d 100755 (executable)
--- a/bin/install.py
+++ b/bin/install.py
@@ -241,12 +241,10 @@ def main(args):
        cmdline = [install_binary]
        cmdline += args
 
-       if sys.hexversion >= 0x3020000:
+       if sys.hexversion >= 0x3000000:
                # We can't trust that the filesystem encoding (locale dependent)
                # correctly matches the arguments, so use surrogateescape to
                # pass through the original argv bytes for Python 3.
-               # Since Python <3.2 does not support bytes in Popen args, trust
-               # the locale in that case.
                fs_encoding = sys.getfilesystemencoding()
                cmdline = [x.encode(fs_encoding, 'surrogateescape') for x in cmdline]
                files = [x.encode(fs_encoding, 'surrogateescape') for x in files]

diff --git a/bin/repoman b/bin/repoman
index ff481d77638509f033b1e0eac26d00c7861ea547..457da097efb0af39c7081c0378c700f9c0b5a923 100755 (executable)
--- a/bin/repoman
+++ b/bin/repoman
@@ -754,11 +754,19 @@ def repoman_getstatusoutput(cmd):
        customized unicode handling (see bug #310789) and without the shell.
        """
        args = portage.util.shlex_split(cmd)
+
+       if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
+               not os.path.isabs(args[0]):
+               # Python 3.1 _execvp throws TypeError for non-absolute executable
+               # path passed as bytes (see http://bugs.python.org/issue8513).
+               fullname = find_binary(args[0])
+               if fullname is None:
+                       raise portage.exception.CommandNotFound(args[0])
+               args[0] = fullname
+
        encoding = _encodings['fs']
-       if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-               # Python 3.1 does not support bytes in Popen args.
-               args = [_unicode_encode(x,
-                       encoding=encoding, errors='strict') for x in args]
+       args = [_unicode_encode(x,
+               encoding=encoding, errors='strict') for x in args]
        proc = subprocess.Popen(args, stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT)
        output = portage._unicode_decode(proc.communicate()[0],
@@ -778,11 +786,19 @@ class repoman_popen(portage.proxy.objectproxy.ObjectProxy):
 
        def __init__(self, cmd):
                args = portage.util.shlex_split(cmd)
+
+               if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
+                       not os.path.isabs(args[0]):
+                       # Python 3.1 _execvp throws TypeError for non-absolute executable
+                       # path passed as bytes (see http://bugs.python.org/issue8513).
+                       fullname = find_binary(args[0])
+                       if fullname is None:
+                               raise portage.exception.CommandNotFound(args[0])
+                       args[0] = fullname
+
                encoding = _encodings['fs']
-               if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                       # Python 3.1 does not support bytes in Popen args.
-                       args = [_unicode_encode(x,
-                               encoding=encoding, errors='strict') for x in args]
+               args = [_unicode_encode(x,
+                       encoding=encoding, errors='strict') for x in args]
                proc = subprocess.Popen(args, stdout=subprocess.PIPE)
                object.__setattr__(self, '_proc', proc)
                object.__setattr__(self, '_stdout',
@@ -2763,15 +2779,17 @@ else:
                        portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd),
                                noiselevel=-1)
                else:
-                       if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000):
-                               # Python 3.1 produces the following TypeError if raw bytes are
-                               # passed to subprocess.call():
-                               #   File "/usr/lib/python3.1/subprocess.py", line 646, in __init__
-                               #     errread, errwrite)
-                               #   File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child
-                               #     raise child_exception
-                               # TypeError: expected an object with the buffer interface
-                               add_cmd = [_unicode_encode(arg) for arg in add_cmd]
+
+                       if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
+                               not os.path.isabs(add_cmd[0]):
+                               # Python 3.1 _execvp throws TypeError for non-absolute executable
+                               # path passed as bytes (see http://bugs.python.org/issue8513).
+                               fullname = find_binary(add_cmd[0])
+                               if fullname is None:
+                                       raise portage.exception.CommandNotFound(add_cmd[0])
+                               add_cmd[0] = fullname
+
+                       add_cmd = [_unicode_encode(arg) for arg in add_cmd]
                        retcode = subprocess.call(add_cmd)
                        if retcode != os.EX_OK:
                                logging.error(
@@ -2919,10 +2937,18 @@ else:
                else:
                        # Encode unicode manually for bug #310789.
                        gpgcmd = portage.util.shlex_split(gpgcmd)
-                       if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                               # Python 3.1 does not support bytes in Popen args.
-                               gpgcmd = [_unicode_encode(arg,
-                                       encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
+
+                       if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
+                               not os.path.isabs(gpgcmd[0]):
+                               # Python 3.1 _execvp throws TypeError for non-absolute executable
+                               # path passed as bytes (see http://bugs.python.org/issue8513).
+                               fullname = find_binary(gpgcmd[0])
+                               if fullname is None:
+                                       raise portage.exception.CommandNotFound(gpgcmd[0])
+                               gpgcmd[0] = fullname
+
+                       gpgcmd = [_unicode_encode(arg,
+                               encoding=_encodings['fs'], errors='strict') for arg in gpgcmd]
                        rValue = subprocess.call(gpgcmd)
                        if rValue == os.EX_OK:
                                os.rename(filename + ".asc", filename)

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index d6da9f74473d8b9fb516f545d29dde034908f6cb..85be13e61b516897c7f0c17edb812b6a128aae4f 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -427,11 +427,18 @@ if platform.system() in ('FreeBSD',):
                                cmd.append(opts)
                        cmd.append('%o' % (flags,))
                        cmd.append(path)
+
+                       if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000:
+                               # Python 3.1 _execvp throws TypeError for non-absolute executable
+                               # path passed as bytes (see http://bugs.python.org/issue8513).
+                               fullname = process.find_binary(cmd[0])
+                               if fullname is None:
+                                       raise exception.CommandNotFound(cmd[0])
+                               cmd[0] = fullname
+
                        encoding = _encodings['fs']
-                       if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                               # Python 3.1 does not support bytes in Popen args.
-                               cmd = [_unicode_encode(x, encoding=encoding, errors='strict')
-                                       for x in cmd]
+                       cmd = [_unicode_encode(x, encoding=encoding, errors='strict')
+                               for x in cmd]
                        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
                        output = proc.communicate()[0]
@@ -635,10 +642,8 @@ if VERSION == 'HEAD':
                                        "if [ -n \"`git diff-index --name-only --diff-filter=M HEAD`\" ] ; " + \
                                        "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; " + \
                                        "exit 0") % _shell_quote(PORTAGE_BASE_PATH)]
-                               if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                                       # Python 3.1 does not support bytes in Popen args.
-                                       cmd = [_unicode_encode(x, encoding=encoding, errors='strict')
-                                               for x in cmd]
+                               cmd = [_unicode_encode(x, encoding=encoding, errors='strict')
+                                       for x in cmd]
                                proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
                                output = _unicode_decode(proc.communicate()[0], encoding=encoding)

diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index 46412e3f1f6ba8f385f1b78fa54f45f378722a2c..66c41c1a2879b751f46ae512e0a01a4d0d2974c9 100644 (file)
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -431,10 +431,17 @@ class FetchTask(CompositeTask):
                args = [portage.util.varexpand(x, mydict=variables)
                        for x in args]
 
-               if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                       # Python 3.1 does not support bytes in Popen args.
-                       args = [_unicode_encode(x,
-                               encoding=_encodings['fs'], errors='strict') for x in args]
+               if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
+                       not os.path.isabs(args[0]):
+                       # Python 3.1 _execvp throws TypeError for non-absolute executable
+                       # path passed as bytes (see http://bugs.python.org/issue8513).
+                       fullname = portage.process.find_binary(args[0])
+                       if fullname is None:
+                               raise portage.exception.CommandNotFound(args[0])
+                       args[0] = fullname
+
+               args = [_unicode_encode(x,
+                       encoding=_encodings['fs'], errors='strict') for x in args]
 
                null_fd = os.open(os.devnull, os.O_RDONLY)
                fetcher = PopenProcess(background=self.background,

diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index cd663e76726291f1c540b2b1a254aefe822aecbf..cd1572e8b95568ab2ee044bfc1e7016490546699 100644 (file)
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -1,5 +1,5 @@
 # checksum.py -- core Portage functionality
-# Copyright 1998-2012 Gentoo Foundation
+# Copyright 1998-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import portage
@@ -165,10 +165,8 @@ hashfunc_map["size"] = getsize
 prelink_capable = False
 if os.path.exists(PRELINK_BINARY):
        cmd = [PRELINK_BINARY, "--version"]
-       if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-               # Python 3.1 does not support bytes in Popen args.
-               cmd = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
-                       for x in cmd]
+       cmd = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
+               for x in cmd]
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT)
        proc.communicate()

diff --git a/pym/portage/data.py b/pym/portage/data.py
index 422dea22f66e3916ef88743b8c8579bce8473718..caf4752ac2b04849ee99ef13d9d2eff379956e8a 100644 (file)
--- a/pym/portage/data.py
+++ b/pym/portage/data.py
@@ -139,12 +139,20 @@ def _get_global(k):
                        # grp.getgrall() since it is known to trigger spurious
                        # SIGPIPE problems with nss_ldap.
                        cmd = ["id", "-G", _portage_username]
+
+                       if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000:
+                               # Python 3.1 _execvp throws TypeError for non-absolute executable
+                               # path passed as bytes (see http://bugs.python.org/issue8513).
+                               fullname = portage.process.find_binary(cmd[0])
+                               if fullname is None:
+                                       globals()[k] = v
+                                       _initialized_globals.add(k)
+                                       return v
+                               cmd[0] = fullname
+
                        encoding = portage._encodings['content']
-                       if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                               # Python 3.1 does not support bytes in Popen args.
-                               cmd = [portage._unicode_encode(x,
-                                       encoding=encoding, errors='strict')
-                                       for x in cmd]
+                       cmd = [portage._unicode_encode(x,
+                               encoding=encoding, errors='strict') for x in cmd]
                        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
                        myoutput = proc.communicate()[0]

diff --git a/pym/portage/dispatch_conf.py b/pym/portage/dispatch_conf.py
index 570bd8c464a7ee29a1e369174d9708b2e5dbcd00..79daa9f7773e3050f31451a0a8b0ae8c3ff839be 100644 (file)
--- a/pym/portage/dispatch_conf.py
+++ b/pym/portage/dispatch_conf.py
@@ -1,5 +1,5 @@
 # archive_conf.py -- functionality common to archive-conf and dispatch-conf
-# Copyright 2003-2012 Gentoo Foundation
+# Copyright 2003-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 
@@ -31,9 +31,17 @@ def diffstatusoutput(cmd, file1, file2):
     # Use Popen to emulate getstatusoutput(), since getstatusoutput() may
     # raise a UnicodeDecodeError which makes the output inaccessible.
     args = shlex_split(cmd % (file1, file2))
-    if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-        # Python 3.1 does not support bytes in Popen args.
-        args = [portage._unicode_encode(x, errors='strict') for x in args]
+
+    if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
+        not os.path.isabs(args[0]):
+        # Python 3.1 _execvp throws TypeError for non-absolute executable
+        # path passed as bytes (see http://bugs.python.org/issue8513).
+        fullname = portage.process.find_binary(args[0])
+        if fullname is None:
+            raise portage.exception.CommandNotFound(args[0])
+        args[0] = fullname
+
+    args = [portage._unicode_encode(x, errors='strict') for x in args]
     proc = subprocess.Popen(args,
         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     output = portage._unicode_decode(proc.communicate()[0])

diff --git a/pym/portage/tests/lint/test_bash_syntax.py b/pym/portage/tests/lint/test_bash_syntax.py
index 0d7d35a6d11a84ea0d5c0353314f248a09fe6c62..fdbb6fe8812bd43c6f5c1ad05c950306a379cefb 100644 (file)
--- a/pym/portage/tests/lint/test_bash_syntax.py
+++ b/pym/portage/tests/lint/test_bash_syntax.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2012 Gentoo Foundation
+# Copyright 2010-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from itertools import chain
@@ -43,10 +43,8 @@ class BashSyntaxTestCase(TestCase):
                                if line[:2] == '#!' and \
                                        'bash' in line:
                                        cmd = [BASH_BINARY, "-n", x]
-                                       if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                                               # Python 3.1 does not support bytes in Popen args.
-                                               cmd = [_unicode_encode(x,
-                                                       encoding=_encodings['fs'], errors='strict') for x in cmd]
+                                       cmd = [_unicode_encode(x,
+                                               encoding=_encodings['fs'], errors='strict') for x in cmd]
                                        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                                stderr=subprocess.STDOUT)
                                        output = _unicode_decode(proc.communicate()[0],

diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py
index 7c16b7bae67a60a18e5e425cc873a04cbe345d7f..69459b827678d2a1b851747006d2f5abdd90d821 100644 (file)
--- a/pym/portage/util/__init__.py
+++ b/pym/portage/util/__init__.py
@@ -1698,10 +1698,17 @@ def find_updated_config_files(target_root, config_protect):
                                                os.path.split(x.rstrip(os.path.sep))
                        mycommand += " ! -name '.*~' ! -iname '.*.bak' -print0"
                        cmd = shlex_split(mycommand)
-                       if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                               # Python 3.1 does not support bytes in Popen args.
-                               cmd = [_unicode_encode(arg, encoding=encoding, errors='strict')
-                                       for arg in cmd]
+
+                       if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000:
+                               # Python 3.1 _execvp throws TypeError for non-absolute executable
+                               # path passed as bytes (see http://bugs.python.org/issue8513).
+                               fullname = portage.process.find_binary(cmd[0])
+                               if fullname is None:
+                                       raise portage.exception.CommandNotFound(cmd[0])
+                               cmd[0] = fullname
+
+                       cmd = [_unicode_encode(arg, encoding=encoding, errors='strict')
+                               for arg in cmd]
                        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
                        output = _unicode_decode(proc.communicate()[0], encoding=encoding)

diff --git a/pym/portage/util/_desktop_entry.py b/pym/portage/util/_desktop_entry.py
index 24bf15431ce6825d37e5ebcab5319aa32a1e5407..a46d5828d3a38384aafd8effa2eaccfaecd5f9ae 100644 (file)
--- a/pym/portage/util/_desktop_entry.py
+++ b/pym/portage/util/_desktop_entry.py
@@ -1,4 +1,4 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import io
@@ -11,6 +11,7 @@ try:
 except ImportError:
        from ConfigParser import Error as ConfigParserError, RawConfigParser
 
+import portage
 from portage import _encodings, _unicode_encode, _unicode_decode
 from portage.util import writemsg
 
@@ -52,9 +53,16 @@ _ignored_errors = (
 
 def validate_desktop_entry(path):
        args = ["desktop-file-validate", path]
-       if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-               # Python 3.1 does not support bytes in Popen args.
-               args = [_unicode_encode(x, errors='strict') for x in args]
+
+       if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000:
+               # Python 3.1 _execvp throws TypeError for non-absolute executable
+               # path passed as bytes (see http://bugs.python.org/issue8513).
+               fullname = portage.process.find_binary(args[0])
+               if fullname is None:
+                       raise portage.exception.CommandNotFound(args[0])
+               args[0] = fullname
+
+       args = [_unicode_encode(x, errors='strict') for x in args]
        proc = subprocess.Popen(args,
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        output_lines = _unicode_decode(proc.communicate()[0]).splitlines()

diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py
index 990090c5f253ea7dccd95708094c33c9407e9a2b..f62e42830baf57f7da23c851ea02d4005fb33dc8 100644 (file)
--- a/pym/repoman/utilities.py
+++ b/pym/repoman/utilities.py
@@ -92,9 +92,7 @@ def detect_vcs_conflicts(options, vcs):
                # Use Popen instead of getstatusoutput(), in order to avoid
                # unicode handling problems (see bug #310789).
                args = [BASH_BINARY, "-c", cmd]
-               if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
-                       # Python 3.1 does not support bytes in Popen args.
-                       args = [_unicode_encode(x) for x in args]
+               args = [_unicode_encode(x) for x in args]
                proc = subprocess.Popen(args, stdout=subprocess.PIPE,
                        stderr=subprocess.STDOUT)
                out = _unicode_decode(proc.communicate()[0])