More Python 3 fixes, mostly about string/byte/unicode handling.

diff --git a/h5config/storage/__init__.py b/h5config/storage/__init__.py
index 00d6ccade0817ea7a7e0a9069a05b860d6898bd6..48028d4bc0388c64e57d21d67d58d266decbc97d 100644 (file)
--- a/h5config/storage/__init__.py
+++ b/h5config/storage/__init__.py
@@ -17,6 +17,8 @@
 
 import os as _os
 import os.path as _os_path
 
 import os as _os
 import os.path as _os_path

+import sys as _sys
+import types as _types

 
 
 class Storage (object):
 
 
 class Storage (object):


@@ -54,3 +56,10 @@ class FileStorage (Storage):
         dirname = _os_path.dirname(filename)
         if dirname and not _os_path.isdir(dirname):
             _os.makedirs(dirname)
         dirname = _os_path.dirname(filename)
         if dirname and not _os_path.isdir(dirname):
             _os.makedirs(dirname)

+
+
+def is_string(x):
+    if _sys.version_info >= (3,):
+        return isinstance(x, (bytes, str))
+    else:  # Python 2 compatibility
+        return isinstance(x, _types.StringTypes)

diff --git a/h5config/storage/hdf5.py b/h5config/storage/hdf5.py
index 79e968200741299b030f7af040cb53c0dea47580..39e6418d313201630455561a656bf0ae9a7eca54 100644 (file)
--- a/h5config/storage/hdf5.py
+++ b/h5config/storage/hdf5.py
@@ -19,7 +19,7 @@
 """
 
 import os.path as _os_path
 """
 
 import os.path as _os_path

-import types as _types
+import sys as _sys

 
 import h5py as _h5py
 import numpy as _numpy
 
 import h5py as _h5py
 import numpy as _numpy


@@ -27,6 +27,7 @@ import numpy as _numpy
 from .. import LOG as _LOG
 from .. import config as _config
 from . import FileStorage as _FileStorage
 from .. import LOG as _LOG
 from .. import config as _config
 from . import FileStorage as _FileStorage

+from . import is_string as _is_string

 
 
 def pprint_HDF5(*args, **kwargs):
 
 
 def pprint_HDF5(*args, **kwargs):


@@ -49,7 +50,7 @@ def _pformat_hdf5(cwg, depth=0):
     lines = []
     lines.append('  '*depth + cwg.name)
     depth += 1
     lines = []
     lines.append('  '*depth + cwg.name)
     depth += 1

-    for key,value in cwg.iteritems():
+    for key,value in cwg.items():

         if isinstance(value, _h5py.Group):
             lines.extend(_pformat_hdf5(value, depth))
         elif isinstance(value, _h5py.Dataset):
         if isinstance(value, _h5py.Group):
             lines.extend(_pformat_hdf5(value, depth))
         elif isinstance(value, _h5py.Dataset):


@@ -217,8 +218,15 @@ class HDF5_Storage (_FileStorage):
                                     v = list(v)
                                 except TypeError:
                                     v = []
                                     v = list(v)
                                 except TypeError:
                                     v = []

-                            else:
-                                v = str(v) # array('abc', dtype='|S3') -> 'abc'
+                                if _sys.version_info >= (3,):
+                                    for i,v_ in enumerate(v):
+                                        if isinstance(v_, bytes):
+                                            v[i] = str(v_, 'utf-8')
+                            else:  # array('abc', dtype='|S3') -> 'abc'
+                                if _sys.version_info >= (3,):
+                                    v = str(v, 'utf-8')
+                                else:
+                                    v = str(v)

                         elif isinstance(s, _config.IntegerSetting):
                             v = int(v)  # array(3, dtpe='int32') -> 3
                         elif isinstance(s, _config.FloatSetting):
                         elif isinstance(s, _config.IntegerSetting):
                             v = int(v)  # array(3, dtpe='int32') -> 3
                         elif isinstance(s, _config.FloatSetting):


@@ -227,7 +235,7 @@ class HDF5_Storage (_FileStorage):
                             raise NotImplementedError(type(s))
                         elif isinstance(s, _config.ListSetting):
                             v = list(v)  # convert from numpy array
                             raise NotImplementedError(type(s))
                         elif isinstance(s, _config.ListSetting):
                             v = list(v)  # convert from numpy array

-                    if isinstance(v, _types.StringTypes):
+                    if _is_string(v):

                         # convert back from None, etc.
                         v = s.convert_from_text(v)
                     config[s.name] = v
                         # convert back from None, etc.
                         v = s.convert_from_text(v)
                     config[s.name] = v


@@ -269,11 +277,22 @@ class HDF5_Storage (_FileStorage):
                         continue
                 if value is None:  # not set yet, or invalid
                     value = s.convert_to_text(config[s.name])
                         continue
                 if value is None:  # not set yet, or invalid
                     value = s.convert_to_text(config[s.name])

+                if _sys.version_info >= (3,):  # convert strings to bytes/
+                    if isinstance(value, str):
+                        value = value.encode('utf-8')
+                    elif isinstance(value, list):
+                        value = list(value)  # shallow copy
+                        for i,v in enumerate(value):
+                            if isinstance(v, str):
+                                value[i] = v.encode('utf-8')

                 try:
                     del group[s.name]
                 except KeyError:
                     pass
                 try:
                     del group[s.name]
                 except KeyError:
                     pass

-                group[s.name] = value
+                try:
+                    group[s.name] = value
+                except TypeError:
+                    raise ValueError((value, type(value)))

         finally:
             if f:
                 f.close()
         finally:
             if f:
                 f.close()

diff --git a/h5config/storage/yaml.py b/h5config/storage/yaml.py
index 9b61c6ecec157eb693513a784c62536bfa56327b..27403d7349b0ea4c5e9e1cf78e55b30bb55ae45a 100644 (file)
--- a/h5config/storage/yaml.py
+++ b/h5config/storage/yaml.py
@@ -21,23 +21,23 @@
 from __future__ import absolute_import
 
 import os.path as _os_path
 from __future__ import absolute_import
 
 import os.path as _os_path

-import types as _types

 
 import yaml as _yaml  # global PyYAML module
 
 from .. import LOG as _LOG
 from .. import config as _config
 from . import FileStorage as _FileStorage
 
 import yaml as _yaml  # global PyYAML module
 
 from .. import LOG as _LOG
 from .. import config as _config
 from . import FileStorage as _FileStorage

+from . import is_string as _is_string

 
 
 class _YAMLDumper (_yaml.SafeDumper):
     def represent_bool(self, data):
         "Use yes/no instead of the default true/false"
         if data:
 
 
 class _YAMLDumper (_yaml.SafeDumper):
     def represent_bool(self, data):
         "Use yes/no instead of the default true/false"
         if data:

-            value = u'yes'
+            value = 'yes'

         else:
         else:

-            value = u'no'
-        return self.represent_scalar(u'tag:yaml.org,2002:bool', value)
+            value = 'no'
+        return self.represent_scalar('tag:yaml.org,2002:bool', value)

 
 
 _YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool)
 
 
 _YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool)


@@ -109,14 +109,14 @@ class YAML_Storage (_FileStorage):
     @staticmethod
     def _from_dict(config, data):
         settings = dict([(s.name, s) for s in config.settings])
     @staticmethod
     def _from_dict(config, data):
         settings = dict([(s.name, s) for s in config.settings])

-        for key,value in data.iteritems():
+        for key,value in data.items():

             setting = settings[key]
             if isinstance(setting, (_config.BooleanSetting,
                                     _config.NumericSetting,
                                     _config.ListSetting,
                                     _config.IntegerListSetting,
                                     _config.FloatListSetting)):
             setting = settings[key]
             if isinstance(setting, (_config.BooleanSetting,
                                     _config.NumericSetting,
                                     _config.ListSetting,
                                     _config.IntegerListSetting,
                                     _config.FloatListSetting)):

-                if isinstance(value, _types.StringTypes):
+                if _is_string(value):

                     # older versions of h5config
                     value = s.convert_from_text(value)
                 v = value
                     # older versions of h5config
                     value = s.convert_from_text(value)
                 v = value


@@ -144,7 +144,7 @@ class YAML_Storage (_FileStorage):
     def _to_dict(config):
         data = {}
         settings = dict([(s.name, s) for s in config.settings])
     def _to_dict(config):
         data = {}
         settings = dict([(s.name, s) for s in config.settings])

-        for key,value in config.iteritems():
+        for key,value in config.items():

             if key in settings:
                 setting = settings[key]
                 if isinstance(setting, (_config.BooleanSetting,
             if key in settings:
                 setting = settings[key]
                 if isinstance(setting, (_config.BooleanSetting,

diff --git a/h5config/test.py b/h5config/test.py
index ac22c107d7fb3950bdbd9bf84c3043229d0f4245..3a509b7f34475385319a1881a628c198bb25bfb6 100644 (file)
--- a/h5config/test.py
+++ b/h5config/test.py
@@ -22,7 +22,7 @@ probably be empty or not exist.
 
 >>> import os
 >>> import tempfile
 
 >>> import os
 >>> import tempfile

->>> from storage.hdf5 import pprint_HDF5, HDF5_Storage
+>>> from h5config.storage.hdf5 import pprint_HDF5, HDF5_Storage

 
 >>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='h5config-')
 >>> os.close(fd)
 
 >>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='h5config-')
 >>> os.close(fd)