MtimeDB: add JSON read/write

Support serialization as JSON instead of pickle, so that
/var/cache/edb/mtimedb is human readable/writable, for those rare
cases where it may be useful. Currently, pickle is still used for
writes. The plan is to migrate to JSON after JSON read has been
supported for some time.

diff --git a/pym/portage/util/mtimedb.py b/pym/portage/util/mtimedb.py
index 0a27166930fa3d86c296f0557af5b638ccf0aa7a..4ccea2438e0dbabfee6fc9936c30ad16eb436228 100644 (file)
--- a/pym/portage/util/mtimedb.py
+++ b/pym/portage/util/mtimedb.py
@@ -10,13 +10,32 @@ except ImportError:
        import pickle
 
 import errno
+import io
+import json
+import sys
+
 import portage
+from portage import _encodings
+from portage import _unicode_decode
 from portage import _unicode_encode
 from portage.data import portage_gid, uid
 from portage.localization import _
 from portage.util import apply_secpass_permissions, atomic_ofstream, writemsg
 
 class MtimeDB(dict):
+
+       # Enable this after JSON read has been supported for some time.
+       _json_write = False
+
+       _json_write_opts = {
+               "ensure_ascii": False,
+               "indent": "\t",
+               "sort_keys": True
+       }
+       if sys.hexversion < 0x3020000:
+               # indent only supports int number of spaces
+               _json_write_opts["indent"] = 4
+
        def __init__(self, filename):
                dict.__init__(self)
                self.filename = filename
@@ -24,28 +43,45 @@ class MtimeDB(dict):
 
        def _load(self, filename):
                f = None
+               content = None
                try:
                        f = open(_unicode_encode(filename), 'rb')
-                       mypickle = pickle.Unpickler(f)
-                       try:
-                               mypickle.find_global = None
-                       except AttributeError:
-                               # TODO: If py3k, override Unpickler.find_class().
-                               pass
-                       d = mypickle.load()
-               except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e:
-                       if isinstance(e, EnvironmentError) and \
-                               getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES):
+                       content = f.read()
+               except EnvironmentError as e:
+                       if getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES):
                                pass
                        else:
                                writemsg(_("!!! Error loading '%s': %s\n") % \
-                                       (filename, str(e)), noiselevel=-1)
-                       del e
-                       d = {}
+                                       (filename, e), noiselevel=-1)
                finally:
                        if f is not None:
                                f.close()
 
+               d = None
+               if content:
+                       try:
+                               mypickle = pickle.Unpickler(io.BytesIO(content))
+                               try:
+                                       mypickle.find_global = None
+                               except AttributeError:
+                                       # TODO: If py3k, override Unpickler.find_class().
+                                       pass
+                               d = mypickle.load()
+                       except SystemExit:
+                               raise
+                       except Exception as e:
+                               try:
+                                       d = json.loads(_unicode_decode(content,
+                                               encoding=_encodings['repo.content'], errors='strict'))
+                               except SystemExit:
+                                       raise
+                               except Exception:
+                                       writemsg(_("!!! Error loading '%s': %s\n") % \
+                                               (filename, e), noiselevel=-1)
+
+               if d is None:
+                       d = {}
+
                if "old" in d:
                        d["updates"] = d["old"]
                        del d["old"]
@@ -80,7 +116,12 @@ class MtimeDB(dict):
                        except EnvironmentError:
                                pass
                        else:
-                               pickle.dump(d, f, protocol=2)
+                               if self._json_write:
+                                       f.write(_unicode_encode(
+                                               json.dumps(d, **self._json_write_opts),
+                                               encoding=_encodings['repo.content'], errors='strict'))
+                               else:
+                                       pickle.dump(d, f, protocol=2)
                                f.close()
                                apply_secpass_permissions(self.filename,
                                        uid=uid, gid=portage_gid, mode=0o644)