test: Fix UTF-8 JSON tests in Python 3

test_expect_equal_json uses json.tool from the system Python.  While
Python 2 wasn't picky about the encoding of stdin, Python 3 decodes
stdin strictly according to the environment.  Since we set LC_ALL=C
for the tests, Python 3's json.tool was assuming stdin would be in
ASCII and aborting when it couldn't decode the UTF-8 characters from
some of the JSON tests.  This patch sets the PYTHONIOENCODING
environment variable to utf-8 when invoking json.tool to override
Python's default encoding choice.

diff --git a/test/test-lib.sh b/test/test-lib.sh
index f1697856426225f2df4de7a6ca9c5c8a384333e5..948752613dda3e2cf5711b0b477176651c592d60 100644 (file)
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -534,8 +534,13 @@ test_expect_equal_file ()
 # canonicalized before diff'ing.  If an argument cannot be parsed, it
 # is used unchanged so that there's something to diff against.
 test_expect_equal_json () {
-    output=$(echo "$1" | python -mjson.tool || echo "$1")
-    expected=$(echo "$2" | python -mjson.tool || echo "$2")
+    # The test suite forces LC_ALL=C, but this causes Python 3 to
+    # decode stdin as ASCII.  We need to read JSON in UTF-8, so
+    # override Python's stdio encoding defaults.
+    output=$(echo "$1" | PYTHONIOENCODING=utf-8 python -mjson.tool \
+        || echo "$1")
+    expected=$(echo "$2" | PYTHONIOENCODING=utf-8 python -mjson.tool \
+        || echo "$2")
     shift 2
     test_expect_equal "$output" "$expected" "$@"
 }