From: Zac Medico Date: Sat, 22 Jun 2013 19:33:50 +0000 (-0700) Subject: Python 3.1: pass bytes to Popen X-Git-Tag: v2.2.0_alpha185~12 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=dd8ead757b3dcfc1b84037b28f215db3ecc6b800;p=portage.git Python 3.1: pass bytes to Popen For Python 3.1, it's possible to pass bytes to Popen as long as the executable has an absolute path, since otherwise we would trigger a TypeError in os._execvp (see http://bugs.python.org/issue8513). Note that Python <=3.1.3 produces the following message: TypeError: expected an object with the buffer interface Later 3.1.x releases produce a different message: TypeError: startswith first arg must be bytes or a tuple of bytes, not str The difference in messages is due to os.path.join() implementation changes, but both errors are triggered by the same underlying bug in os._execvp() which was fixed by using fsencode() in this hunk: --- a/Lib/os.py +++ b/Lib/os.py @@ -355,7 +355,11 @@ def _execvpe(file, args, env=None): return last_exc = saved_exc = None saved_tb = None - for dir in get_exec_path(env): + path_list = get_exec_path(env) + if name != 'nt': + file = fsencode(file) + path_list = map(fsencode, path_list) + for dir in path_list: fullname = path.join(dir, file) --- diff --git a/bin/install.py b/bin/install.py index cce68c34e..02083a181 100755 --- a/bin/install.py +++ b/bin/install.py @@ -241,12 +241,10 @@ def main(args): cmdline = [install_binary] cmdline += args - if sys.hexversion >= 0x3020000: + if sys.hexversion >= 0x3000000: # We can't trust that the filesystem encoding (locale dependent) # correctly matches the arguments, so use surrogateescape to # pass through the original argv bytes for Python 3. - # Since Python <3.2 does not support bytes in Popen args, trust - # the locale in that case. fs_encoding = sys.getfilesystemencoding() cmdline = [x.encode(fs_encoding, 'surrogateescape') for x in cmdline] files = [x.encode(fs_encoding, 'surrogateescape') for x in files] diff --git a/bin/repoman b/bin/repoman index ff481d776..457da097e 100755 --- a/bin/repoman +++ b/bin/repoman @@ -754,11 +754,19 @@ def repoman_getstatusoutput(cmd): customized unicode handling (see bug #310789) and without the shell. """ args = portage.util.shlex_split(cmd) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + encoding = _encodings['fs'] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [_unicode_encode(x, - encoding=encoding, errors='strict') for x in args] + args = [_unicode_encode(x, + encoding=encoding, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0], @@ -778,11 +786,19 @@ class repoman_popen(portage.proxy.objectproxy.ObjectProxy): def __init__(self, cmd): args = portage.util.shlex_split(cmd) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + encoding = _encodings['fs'] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [_unicode_encode(x, - encoding=encoding, errors='strict') for x in args] + args = [_unicode_encode(x, + encoding=encoding, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE) object.__setattr__(self, '_proc', proc) object.__setattr__(self, '_stdout', @@ -2763,15 +2779,17 @@ else: portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd), noiselevel=-1) else: - if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000): - # Python 3.1 produces the following TypeError if raw bytes are - # passed to subprocess.call(): - # File "/usr/lib/python3.1/subprocess.py", line 646, in __init__ - # errread, errwrite) - # File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child - # raise child_exception - # TypeError: expected an object with the buffer interface - add_cmd = [_unicode_encode(arg) for arg in add_cmd] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(add_cmd[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(add_cmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(add_cmd[0]) + add_cmd[0] = fullname + + add_cmd = [_unicode_encode(arg) for arg in add_cmd] retcode = subprocess.call(add_cmd) if retcode != os.EX_OK: logging.error( @@ -2919,10 +2937,18 @@ else: else: # Encode unicode manually for bug #310789. gpgcmd = portage.util.shlex_split(gpgcmd) - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - gpgcmd = [_unicode_encode(arg, - encoding=_encodings['fs'], errors='strict') for arg in gpgcmd] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(gpgcmd[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = find_binary(gpgcmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(gpgcmd[0]) + gpgcmd[0] = fullname + + gpgcmd = [_unicode_encode(arg, + encoding=_encodings['fs'], errors='strict') for arg in gpgcmd] rValue = subprocess.call(gpgcmd) if rValue == os.EX_OK: os.rename(filename + ".asc", filename) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index d6da9f744..85be13e61 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -427,11 +427,18 @@ if platform.system() in ('FreeBSD',): cmd.append(opts) cmd.append('%o' % (flags,)) cmd.append(path) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = process.find_binary(cmd[0]) + if fullname is None: + raise exception.CommandNotFound(cmd[0]) + cmd[0] = fullname + encoding = _encodings['fs'] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - cmd = [_unicode_encode(x, encoding=encoding, errors='strict') - for x in cmd] + cmd = [_unicode_encode(x, encoding=encoding, errors='strict') + for x in cmd] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = proc.communicate()[0] @@ -635,10 +642,8 @@ if VERSION == 'HEAD': "if [ -n \"`git diff-index --name-only --diff-filter=M HEAD`\" ] ; " + \ "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; " + \ "exit 0") % _shell_quote(PORTAGE_BASE_PATH)] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - cmd = [_unicode_encode(x, encoding=encoding, errors='strict') - for x in cmd] + cmd = [_unicode_encode(x, encoding=encoding, errors='strict') + for x in cmd] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = _unicode_decode(proc.communicate()[0], encoding=encoding) diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py index 46412e3f1..66c41c1a2 100644 --- a/pym/portage/_emirrordist/FetchTask.py +++ b/pym/portage/_emirrordist/FetchTask.py @@ -431,10 +431,17 @@ class FetchTask(CompositeTask): args = [portage.util.varexpand(x, mydict=variables) for x in args] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [_unicode_encode(x, - encoding=_encodings['fs'], errors='strict') for x in args] + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [_unicode_encode(x, + encoding=_encodings['fs'], errors='strict') for x in args] null_fd = os.open(os.devnull, os.O_RDONLY) fetcher = PopenProcess(background=self.background, diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index cd663e767..cd1572e8b 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -1,5 +1,5 @@ # checksum.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage @@ -165,10 +165,8 @@ hashfunc_map["size"] = getsize prelink_capable = False if os.path.exists(PRELINK_BINARY): cmd = [PRELINK_BINARY, "--version"] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - cmd = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') - for x in cmd] + cmd = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') + for x in cmd] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) proc.communicate() diff --git a/pym/portage/data.py b/pym/portage/data.py index 422dea22f..caf4752ac 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -139,12 +139,20 @@ def _get_global(k): # grp.getgrall() since it is known to trigger spurious # SIGPIPE problems with nss_ldap. cmd = ["id", "-G", _portage_username] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(cmd[0]) + if fullname is None: + globals()[k] = v + _initialized_globals.add(k) + return v + cmd[0] = fullname + encoding = portage._encodings['content'] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - cmd = [portage._unicode_encode(x, - encoding=encoding, errors='strict') - for x in cmd] + cmd = [portage._unicode_encode(x, + encoding=encoding, errors='strict') for x in cmd] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) myoutput = proc.communicate()[0] diff --git a/pym/portage/dispatch_conf.py b/pym/portage/dispatch_conf.py index 570bd8c46..79daa9f77 100644 --- a/pym/portage/dispatch_conf.py +++ b/pym/portage/dispatch_conf.py @@ -1,5 +1,5 @@ # archive_conf.py -- functionality common to archive-conf and dispatch-conf -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -31,9 +31,17 @@ def diffstatusoutput(cmd, file1, file2): # Use Popen to emulate getstatusoutput(), since getstatusoutput() may # raise a UnicodeDecodeError which makes the output inaccessible. args = shlex_split(cmd % (file1, file2)) - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [portage._unicode_encode(x, errors='strict') for x in args] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [portage._unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0]) diff --git a/pym/portage/tests/lint/test_bash_syntax.py b/pym/portage/tests/lint/test_bash_syntax.py index 0d7d35a6d..fdbb6fe88 100644 --- a/pym/portage/tests/lint/test_bash_syntax.py +++ b/pym/portage/tests/lint/test_bash_syntax.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from itertools import chain @@ -43,10 +43,8 @@ class BashSyntaxTestCase(TestCase): if line[:2] == '#!' and \ 'bash' in line: cmd = [BASH_BINARY, "-n", x] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - cmd = [_unicode_encode(x, - encoding=_encodings['fs'], errors='strict') for x in cmd] + cmd = [_unicode_encode(x, + encoding=_encodings['fs'], errors='strict') for x in cmd] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = _unicode_decode(proc.communicate()[0], diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index 7c16b7bae..69459b827 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -1698,10 +1698,17 @@ def find_updated_config_files(target_root, config_protect): os.path.split(x.rstrip(os.path.sep)) mycommand += " ! -name '.*~' ! -iname '.*.bak' -print0" cmd = shlex_split(mycommand) - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - cmd = [_unicode_encode(arg, encoding=encoding, errors='strict') - for arg in cmd] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(cmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(cmd[0]) + cmd[0] = fullname + + cmd = [_unicode_encode(arg, encoding=encoding, errors='strict') + for arg in cmd] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = _unicode_decode(proc.communicate()[0], encoding=encoding) diff --git a/pym/portage/util/_desktop_entry.py b/pym/portage/util/_desktop_entry.py index 24bf15431..a46d5828d 100644 --- a/pym/portage/util/_desktop_entry.py +++ b/pym/portage/util/_desktop_entry.py @@ -1,4 +1,4 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io @@ -11,6 +11,7 @@ try: except ImportError: from ConfigParser import Error as ConfigParserError, RawConfigParser +import portage from portage import _encodings, _unicode_encode, _unicode_decode from portage.util import writemsg @@ -52,9 +53,16 @@ _ignored_errors = ( def validate_desktop_entry(path): args = ["desktop-file-validate", path] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [_unicode_encode(x, errors='strict') for x in args] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [_unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output_lines = _unicode_decode(proc.communicate()[0]).splitlines() diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py index 990090c5f..f62e42830 100644 --- a/pym/repoman/utilities.py +++ b/pym/repoman/utilities.py @@ -92,9 +92,7 @@ def detect_vcs_conflicts(options, vcs): # Use Popen instead of getstatusoutput(), in order to avoid # unicode handling problems (see bug #310789). args = [BASH_BINARY, "-c", cmd] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [_unicode_encode(x) for x in args] + args = [_unicode_encode(x) for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out = _unicode_decode(proc.communicate()[0])