From 6bec535f32069c69e7a5567347f7acf8dbb030c2 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 30 Mar 2012 00:13:56 -0400 Subject: [PATCH] More Python 3 fixes, mostly about string/byte/unicode handling. --- h5config/storage/__init__.py | 9 +++++++++ h5config/storage/hdf5.py | 31 +++++++++++++++++++++++++------ h5config/storage/yaml.py | 14 +++++++------- h5config/test.py | 2 +- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/h5config/storage/__init__.py b/h5config/storage/__init__.py index 00d6cca..48028d4 100644 --- a/h5config/storage/__init__.py +++ b/h5config/storage/__init__.py @@ -17,6 +17,8 @@ import os as _os import os.path as _os_path +import sys as _sys +import types as _types class Storage (object): @@ -54,3 +56,10 @@ class FileStorage (Storage): dirname = _os_path.dirname(filename) if dirname and not _os_path.isdir(dirname): _os.makedirs(dirname) + + +def is_string(x): + if _sys.version_info >= (3,): + return isinstance(x, (bytes, str)) + else: # Python 2 compatibility + return isinstance(x, _types.StringTypes) diff --git a/h5config/storage/hdf5.py b/h5config/storage/hdf5.py index 79e9682..39e6418 100644 --- a/h5config/storage/hdf5.py +++ b/h5config/storage/hdf5.py @@ -19,7 +19,7 @@ """ import os.path as _os_path -import types as _types +import sys as _sys import h5py as _h5py import numpy as _numpy @@ -27,6 +27,7 @@ import numpy as _numpy from .. import LOG as _LOG from .. import config as _config from . import FileStorage as _FileStorage +from . import is_string as _is_string def pprint_HDF5(*args, **kwargs): @@ -49,7 +50,7 @@ def _pformat_hdf5(cwg, depth=0): lines = [] lines.append(' '*depth + cwg.name) depth += 1 - for key,value in cwg.iteritems(): + for key,value in cwg.items(): if isinstance(value, _h5py.Group): lines.extend(_pformat_hdf5(value, depth)) elif isinstance(value, _h5py.Dataset): @@ -217,8 +218,15 @@ class HDF5_Storage (_FileStorage): v = list(v) except TypeError: v = [] - else: - v = str(v) # array('abc', dtype='|S3') -> 'abc' + if _sys.version_info >= (3,): + for i,v_ in enumerate(v): + if isinstance(v_, bytes): + v[i] = str(v_, 'utf-8') + else: # array('abc', dtype='|S3') -> 'abc' + if _sys.version_info >= (3,): + v = str(v, 'utf-8') + else: + v = str(v) elif isinstance(s, _config.IntegerSetting): v = int(v) # array(3, dtpe='int32') -> 3 elif isinstance(s, _config.FloatSetting): @@ -227,7 +235,7 @@ class HDF5_Storage (_FileStorage): raise NotImplementedError(type(s)) elif isinstance(s, _config.ListSetting): v = list(v) # convert from numpy array - if isinstance(v, _types.StringTypes): + if _is_string(v): # convert back from None, etc. v = s.convert_from_text(v) config[s.name] = v @@ -269,11 +277,22 @@ class HDF5_Storage (_FileStorage): continue if value is None: # not set yet, or invalid value = s.convert_to_text(config[s.name]) + if _sys.version_info >= (3,): # convert strings to bytes/ + if isinstance(value, str): + value = value.encode('utf-8') + elif isinstance(value, list): + value = list(value) # shallow copy + for i,v in enumerate(value): + if isinstance(v, str): + value[i] = v.encode('utf-8') try: del group[s.name] except KeyError: pass - group[s.name] = value + try: + group[s.name] = value + except TypeError: + raise ValueError((value, type(value))) finally: if f: f.close() diff --git a/h5config/storage/yaml.py b/h5config/storage/yaml.py index 9b61c6e..27403d7 100644 --- a/h5config/storage/yaml.py +++ b/h5config/storage/yaml.py @@ -21,23 +21,23 @@ from __future__ import absolute_import import os.path as _os_path -import types as _types import yaml as _yaml # global PyYAML module from .. import LOG as _LOG from .. import config as _config from . import FileStorage as _FileStorage +from . import is_string as _is_string class _YAMLDumper (_yaml.SafeDumper): def represent_bool(self, data): "Use yes/no instead of the default true/false" if data: - value = u'yes' + value = 'yes' else: - value = u'no' - return self.represent_scalar(u'tag:yaml.org,2002:bool', value) + value = 'no' + return self.represent_scalar('tag:yaml.org,2002:bool', value) _YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool) @@ -109,14 +109,14 @@ class YAML_Storage (_FileStorage): @staticmethod def _from_dict(config, data): settings = dict([(s.name, s) for s in config.settings]) - for key,value in data.iteritems(): + for key,value in data.items(): setting = settings[key] if isinstance(setting, (_config.BooleanSetting, _config.NumericSetting, _config.ListSetting, _config.IntegerListSetting, _config.FloatListSetting)): - if isinstance(value, _types.StringTypes): + if _is_string(value): # older versions of h5config value = s.convert_from_text(value) v = value @@ -144,7 +144,7 @@ class YAML_Storage (_FileStorage): def _to_dict(config): data = {} settings = dict([(s.name, s) for s in config.settings]) - for key,value in config.iteritems(): + for key,value in config.items(): if key in settings: setting = settings[key] if isinstance(setting, (_config.BooleanSetting, diff --git a/h5config/test.py b/h5config/test.py index ac22c10..3a509b7 100644 --- a/h5config/test.py +++ b/h5config/test.py @@ -22,7 +22,7 @@ probably be empty or not exist. >>> import os >>> import tempfile ->>> from storage.hdf5 import pprint_HDF5, HDF5_Storage +>>> from h5config.storage.hdf5 import pprint_HDF5, HDF5_Storage >>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='h5config-') >>> os.close(fd) -- 2.26.2