"""The basic h5config classes
"""
+import copy as _copy
+
class Setting (object):
"A named setting with arbitrart text values."
return ret.strip()
def convert_from_text(self, value):
- return value
+ return value or None
def convert_to_text(self, value):
- return value
+ return value or ''
class ChoiceSetting (Setting):
>>> s.help()
'Default: no. Choices: yes, no'
"""
- def __init__(self, **kwargs):
- assert 'choices' not in kwargs
- if 'default' not in kwargs:
- kwargs['default'] = False
+ def __init__(self, default=False, **kwargs):
super(BooleanSetting, self).__init__(
- choices=[('yes', True), ('no', False)], **kwargs)
+ choices=[('yes', True), ('no', False)], default=default, **kwargs)
class NumericSetting (Setting):
>>> s.convert_to_text(13)
'13'
"""
- _default_value = 0
-
- def __init__(self, **kwargs):
- if 'default' not in kwargs:
- kwargs['default'] = self._default_value
- super(NumericSetting, self).__init__(**kwargs)
+ def __init__(self, default=0, **kwargs):
+ super(NumericSetting, self).__init__(default=default, **kwargs)
def convert_to_text(self, value):
return str(value)
>>> s.convert_from_text('8')
8
"""
- _default_value = 1
+ def __init__(self, default=1, **kwargs):
+ super(IntegerSetting, self).__init__(default=default, **kwargs)
def _convert_from_text(self, value):
return int(value)
... print 'caught a ValueError'
caught a ValueError
"""
- _default_value = 1.0
+ def __init__(self, default=1.0, **kwargs):
+ super(FloatSetting, self).__init__(default=default, **kwargs)
def _convert_from_text(self, value):
return float(value)
>>> s.convert_from_text('')
[]
"""
- def __init__(self, **kwargs):
- if 'default' not in kwargs:
- kwargs['default'] = []
- super(FloatListSetting, self).__init__(**kwargs)
+ def __init__(self, default=[], **kwargs):
+ super(FloatListSetting, self).__init__(default=default, **kwargs)
def _convert_from_text(self, value):
if value is None:
def convert_from_text(self, value):
if value is None:
return None
- elif value == '':
+ elif value in ['', []]:
return []
return [self._convert_from_text(x) for x in value.split(',')]
return ', '.join([str(x) for x in value])
+class ConfigSetting (Setting):
+ """A setting that holds a pointer to a child `Config` class
+
+ This allows you to nest `Config`\s, which is a useful way to
+ contain complexity. In order to save such a config, the backend
+ must be able to handle hierarchical storage (possibly via
+ references).
+
+ For example, a configurable AFM may contain a configurable piezo
+ scanner, as well as a configurable stepper motor.
+ """
+ def __init__(self, config_class=None, **kwargs):
+ super(ConfigSetting, self).__init__(**kwargs)
+ self.config_class = config_class
+
+
+class ConfigListSetting (ConfigSetting):
+ """A setting that holds a list of child `Config` classes
+
+ For example, a piezo scanner with several axes.
+ """
+ def __init__(self, default=[], **kwargs):
+ super(ConfigListSetting, self).__init__(**kwargs)
+
+
class Config (dict):
"A class with a list `._keys` of `Setting`\s."
settings = []
- def __init__(self):
+ def __init__(self, storage=None):
+ super(Config, self).__init__()
+ self.clear()
+ self._storage = storage
+
+ def __repr__(self):
+ return '<{} {}>'.format(self.__class__.__name__, id(self))
+
+ def clear(self):
+ super(Config, self).clear()
for s in self.settings:
- self[s.name] = s.default
+ # copy to avoid ambiguity with mutable defaults
+ self[s.name] = _copy.deepcopy(s.default)
+
+ def load(self, merge=False, **kwargs):
+ self._storage.load(config=self, merge=merge, **kwargs)
- def dump(self, help=False):
+ def save(self, merge=False, **kwargs):
+ self._storage.save(config=self, merge=merge, **kwargs)
+
+ def dump(self, help=False, prefix=''):
"""Return all settings and their values as a string
>>> class MyConfig (Config):
lines = []
for setting in self.settings:
name = setting.name
+ value = self[name]
+ if isinstance(setting, ConfigSetting):
+ if value is not None:
+ lines.append(value.dump(help=help, prefix=prefix+' '))
+ continue
+ elif isinstance(setting, ConfigListSetting):
+ if value:
+ for config in value:
+ lines.append(
+ config.dump(help=help, prefix=prefix+' '))
+ continue
value_string = setting.convert_to_text(self[name])
if help:
help_string = '\t({})'.format(setting.help())
help_string = ''
lines.append('{}: {}{}'.format(name, value_string, help_string))
return '\n'.join(lines)
-
-
-class BackedConfig (Config):
- "A `Config` instance with some kind of storage interface"
- def load(self):
- raise NotImplementedError()
-
- def save(self):
- raise NotImplementedError()
+++ /dev/null
-# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
-#
-# This file is part of h5config.
-#
-# h5config is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation, either version 3 of the License, or (at your
-# option) any later version.
-#
-# h5config is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with h5config. If not, see <http://www.gnu.org/licenses/>.
-
-"""HDF5 backend implementation
-"""
-
-import h5py as _h5py
-
-from . import LOG as _LOG
-from . import config as _config
-
-
-def pprint_HDF5(*args, **kwargs):
- print pformat_HDF5(*args, **kwargs)
-
-def pformat_HDF5(filename, group='/'):
- f = _h5py.File(filename, 'r')
- cwg = f[group]
- return '\n'.join(_pformat_hdf5(cwg))
-
-def _pformat_hdf5(cwg, depth=0):
- lines = []
- lines.append(' '*depth + cwg.name)
- depth += 1
- for key,value in cwg.iteritems():
- if isinstance(value, _h5py.Group):
- lines.extend(_pformat_hdf5(value, depth))
- elif isinstance(value, _h5py.Dataset):
- lines.append(' '*depth + str(value))
- lines.append(' '*(depth+1) + str(value[...]))
- else:
- lines.append(' '*depth + str(value))
- return lines
-
-def h5_create_group(cwg, path):
- "Create the group where the settings are stored (if necessary)."
- if path == '/':
- return cwg
- gpath = ['']
- for group in path.strip('/').split('/'):
- gpath.append(group)
- if group not in cwg.keys():
- _LOG.debug('creating group %s in %s'
- % ('/'.join(gpath), cwg.file))
- cwg.create_group(group)
- cwg = cwg[group]
- return cwg
-
-
-class _HDF5Config (_config.BackedConfig):
- """Mixin to back a `_Config` class with an HDF5 file.
-
- TODO: Special handling for Choice (enums), FloatList (arrays), etc.?
-
- The `.save` and `.load` methods have an optional `group` argument
- that allows you to save and load settings from an externally
- opened HDF5 file. This can make it easier to stash several
- related `_Config` classes in a single file. For example
-
- >>> import os
- >>> import tempfile
- >>> from .test import HDF5_TestConfig
- >>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='pypiezo-')
- >>> os.close(fd)
-
- >>> f = _h5py.File(filename, 'a')
- >>> c = HDF5_TestConfig(filename='untouched_file.h5',
- ... group='/untouched/group')
- >>> c['syslog'] = True
- >>> group = f.create_group('base')
- >>> c.save(group)
- >>> pprint_HDF5(filename)
- /
- /base
- <HDF5 dataset "age": shape (), type "|S3">
- 1.3
- <HDF5 dataset "alive": shape (), type "|S2">
- no
- <HDF5 dataset "bids": shape (), type "|S11">
- 5.4, 3.2, 1
- <HDF5 dataset "daisies": shape (), type "|S2">
- 13
- <HDF5 dataset "species": shape (), type "|S14">
- Norwegian Blue
- >>> d = HDF5_TestConfig(filename='untouched_file.h5',
- ... group='/untouched/group')
- >>> d.load(group)
- >>> d['alive']
- False
-
- >>> f.close()
- >>> os.remove(filename)
- """
- def __init__(self, filename=None, group='/', **kwargs):
- super(_HDF5Config, self).__init__(**kwargs)
- self.filename = filename
- assert group.startswith('/'), group
- if not group.endswith('/'):
- group += '/'
- self.group = group
- self._file_checked = False
-
- def _check_file(self):
- if self._file_checked:
- return
- self._setup_file()
- self._file_checked = True
-
- def _setup_file(self):
- f = _h5py.File(self.filename, 'a')
- cwg = f # current working group
- h5_create_group(cwg, self.group)
- f.close()
-
- def dump(self, help=False, from_file=False):
- """Return the relevant group in `self.filename` as a string
-
- Extends the base :meth:`dump` by adding the `from_file`
- option. If `from_file` is true, dump all entries that
- currently exist in the relevant group, rather than listing all
- settings defined in the instance dictionary.
- """
- if from_file:
- self._check_file()
- f = _h5py.File(self.filename, 'r')
- cwg = f[self.group]
- lines = []
- settings = dict([(s.name, s) for s in self.settings])
- for key,value in cwg.iteritems():
- if help and key in settings:
- help_string = '\t(%s)' % settings[key].help()
- else:
- help_string = ''
- lines.append('%s: %s%s' % (key, value[...], help_string))
- return '\n'.join(lines)
- return super(_HDF5Config, self).dump(help=help)
-
- def load(self, group=None):
- if group is None:
- self._check_file()
- f = _h5py.File(self.filename, 'r')
- group = f[self.group]
- else:
- f = None
- for s in self.settings:
- if s.name not in group.keys():
- continue
- self[s.name] = s.convert_from_text(group[s.name][...])
- if f:
- f.close()
-
- def save(self, group=None):
- if group is None:
- self._check_file()
- f = _h5py.File(self.filename, 'a')
- group = f[self.group]
- else:
- f = None
- for s in self.settings:
- value = s.convert_to_text(self[s.name])
- try:
- del group[s.name]
- except KeyError:
- pass
- group[s.name] = value
- if f:
- f.close()
--- /dev/null
+# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
+#
+# This file is part of h5config.
+#
+# h5config is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# h5config is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with h5config. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Storage (object):
+ "A storage bakend for loading and saving `Config` instances"
+ def load(self, config, merge=False, **kwargs):
+ if merge:
+ self.clear()
+ self._load(config=config, **kwargs)
+ config._storage = self
+
+ def _load(self, config, **kwargs):
+ raise NotImplementedError()
+
+ def save(self, config, merge=False, **kwargs):
+ if merge:
+ self.clear()
+ self._save(config=config, **kwargs)
+ config._storage = self
+
+ def _save(self, config, **kwargs):
+ raise NotImplementedError()
+
+
+class FileStorage (Storage):
+ "`Config` storage backend by a single file"
+ extension = None
+
+ def __init__(self, filename=None):
+ self._filename = filename
--- /dev/null
+# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
+#
+# This file is part of h5config.
+#
+# h5config is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# h5config is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with h5config. If not, see <http://www.gnu.org/licenses/>.
+
+"""HDF5 backend implementation
+"""
+
+import h5py as _h5py
+
+from .. import LOG as _LOG
+from .. import config as _config
+from . import FileStorage as _FileStorage
+
+
+def pprint_HDF5(*args, **kwargs):
+ print pformat_HDF5(*args, **kwargs)
+
+def pformat_HDF5(filename, group='/'):
+ with _h5py.File(filename, 'r') as f:
+ cwg = f[group]
+ ret = '\n'.join(_pformat_hdf5(cwg))
+ return ret
+
+def _pformat_hdf5(cwg, depth=0):
+ lines = []
+ lines.append(' '*depth + cwg.name)
+ depth += 1
+ for key,value in cwg.iteritems():
+ if isinstance(value, _h5py.Group):
+ lines.extend(_pformat_hdf5(value, depth))
+ elif isinstance(value, _h5py.Dataset):
+ lines.append(' '*depth + str(value))
+ lines.append(' '*(depth+1) + str(value[...]))
+ else:
+ lines.append(' '*depth + str(value))
+ return lines
+
+def h5_create_group(cwg, path, force=False):
+ "Create the group where the settings are stored (if necessary)."
+ if path == '/':
+ return cwg
+ gpath = ['']
+ for group in path.strip('/').split('/'):
+ gpath.append(group)
+ if group not in cwg.keys():
+ _LOG.debug('creating group {} in {}'.format(
+ '/'.join(gpath), cwg.file))
+ cwg.create_group(group)
+ _cwg = cwg[group]
+ if isinstance(_cwg, _h5py.Dataset):
+ if force:
+ _LOG.info('overwrite {} in {} ({}) with a group'.format(
+ '/'.join(gpath), _cwg.file, _cwg))
+ del cwg[group]
+ _cwg = cwg.create_group(group)
+ else:
+ raise ValueError(_cwg)
+ cwg = _cwg
+ return cwg
+
+
+class HDF5_Storage (_FileStorage):
+ """Back a `Config` class with an HDF5 file.
+
+ TODO: Special handling for Choice (enums), FloatList (arrays), etc.?
+
+ The `.save` and `.load` methods have an optional `group` argument
+ that allows you to save and load settings from an externally
+ opened HDF5 file. This can make it easier to stash several
+ related `Config` classes in a single file. For example
+
+ >>> import os
+ >>> import tempfile
+ >>> from ..test import TestConfig
+ >>> fd,filename = tempfile.mkstemp(
+ ... suffix='.'+HDF5_Storage.extension, prefix='pypiezo-')
+ >>> os.close(fd)
+
+ >>> f = _h5py.File(filename, 'a')
+ >>> c = TestConfig(storage=HDF5_Storage(
+ ... filename='untouched_file.h5', group='/untouched/group'))
+ >>> c['alive'] = True
+ >>> group = f.create_group('base')
+ >>> c.save(group=group)
+ >>> pprint_HDF5(filename) # doctest: +REPORT_UDIFF
+ /
+ /base
+ <HDF5 dataset "age": shape (), type "|S3">
+ 1.3
+ <HDF5 dataset "alive": shape (), type "|S3">
+ yes
+ <HDF5 dataset "bids": shape (), type "|S11">
+ 5.4, 3.2, 1
+ <HDF5 dataset "children": shape (), type "|S1">
+ <BLANKLINE>
+ <HDF5 dataset "daisies": shape (), type "|S2">
+ 13
+ <HDF5 dataset "name": shape (), type "|S1">
+ <BLANKLINE>
+ <HDF5 dataset "species": shape (), type "|S14">
+ Norwegian Blue
+ <HDF5 dataset "spouse": shape (), type "|S1">
+ <BLANKLINE>
+ >>> c.clear()
+ >>> c['alive']
+ False
+ >>> c.load(group=group)
+ >>> c['alive']
+ True
+
+ >>> f.close()
+ >>> os.remove(filename)
+ """
+ extension = 'h5'
+
+ def __init__(self, group='/', **kwargs):
+ super(HDF5_Storage, self).__init__(**kwargs)
+ assert group.startswith('/'), group
+ if not group.endswith('/'):
+ group += '/'
+ self.group = group
+ self._file_checked = False
+
+ def _check_file(self):
+ if self._file_checked:
+ return
+ self._setup_file()
+ self._file_checked = True
+
+ def _setup_file(self):
+ with _h5py.File(self._filename, 'a') as f:
+ cwg = f # current working group
+ h5_create_group(cwg, self.group)
+
+ def _load(self, config, group=None):
+ f = None
+ try:
+ if group is None:
+ self._check_file()
+ f = _h5py.File(self._filename, 'r')
+ group = f[self.group]
+ for s in config.settings:
+ if s.name not in group.keys():
+ continue
+ elif isinstance(s, _config.ConfigListSetting):
+ try:
+ cwg = h5_create_group(group, s.name)
+ except ValueError:
+ pass
+ else:
+ value = []
+ for i in sorted(int(x) for x in cwg.keys()):
+ instance = s.config_class()
+ try:
+ _cwg = h5_create_group(cwg, str(i))
+ except ValueError:
+ pass
+ else:
+ self._load(config=instance, group=_cwg)
+ value.append(instance)
+ config[s.name] = value
+ elif isinstance(s, _config.ConfigSetting):
+ try:
+ cwg = h5_create_group(group, s.name)
+ except ValueError:
+ pass
+ else:
+ if not config[s.name]:
+ config[s.name] = s.config_class()
+ self._load(config=config[s.name], group=cwg)
+ else:
+ config[s.name] = s.convert_from_text(group[s.name][...])
+ finally:
+ if f:
+ f.close()
+
+ def _save(self, config, group=None):
+ f = None
+ try:
+ if group is None:
+ self._check_file()
+ f = _h5py.File(self._filename, 'a')
+ group = f[self.group]
+ for s in config.settings:
+ if isinstance(s, _config.ConfigListSetting):
+ configs = config[s.name]
+ if configs:
+ cwg = h5_create_group(group, s.name, force=True)
+ for i,cfg in enumerate(configs):
+ _cwg = h5_create_group(cwg, str(i), force=True)
+ self._save(config=cfg, group=_cwg)
+ continue
+ elif isinstance(s, _config.ConfigSetting):
+ cfg = config[s.name]
+ if cfg:
+ cwg = h5_create_group(group, s.name, force=True)
+ self._save(config=cfg, group=cwg)
+ continue
+ value = s.convert_to_text(config[s.name])
+ try:
+ del group[s.name]
+ except KeyError:
+ pass
+ group[s.name] = value
+ finally:
+ if f:
+ f.close()
--- /dev/null
+# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
+#
+# This file is part of h5config.
+#
+# h5config is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# h5config is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with h5config. If not, see <http://www.gnu.org/licenses/>.
+
+"""HDF5 backend implementation
+"""
+
+from __future__ import absolute_import
+
+import os.path as _os_path
+
+import yaml as _yaml # global PyYAML module
+
+from .. import LOG as _LOG
+from .. import config as _config
+from . import FileStorage as _FileStorage
+
+
+class _YAMLDumper (_yaml.SafeDumper):
+ def represent_bool(self, data):
+ "Use yes/no instead of the default true/false"
+ if data:
+ value = u'yes'
+ else:
+ value = u'no'
+ return self.represent_scalar(u'tag:yaml.org,2002:bool', value)
+
+
+_YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool)
+
+
+class YAML_Storage (_FileStorage):
+ """Back a `Config` class with a YAML file.
+
+ TODO: Special handling for Choice (enums), etc.?
+
+ >>> import os
+ >>> from ..test import TestConfig
+ >>> import os.path
+ >>> import tempfile
+ >>> fd,filename = tempfile.mkstemp(
+ ... suffix='.'+YAML_Storage.extension, prefix='pypiezo-')
+ >>> os.close(fd)
+
+ >>> c = TestConfig(storage=YAML_Storage(filename=filename))
+ >>> c.load()
+
+ Saving writes all the config values to disk.
+
+ >>> c['alive'] = True
+ >>> c.save()
+ >>> print open(filename, 'r').read() # doctest: +REPORT_UDIFF
+ age: 1.3
+ alive: yes
+ bids:
+ - 5.4
+ - 3.2
+ - 1
+ children: ''
+ daisies: 13
+ name: ''
+ species: Norwegian Blue
+ spouse: ''
+ <BLANKLINE>
+
+ Loading reads the config files from disk.
+
+ >>> c.clear()
+ >>> c['alive']
+ False
+ >>> c.load()
+ >>> c['alive']
+ True
+
+ Cleanup our temporary config file.
+
+ >>> os.remove(filename)
+ """
+ extension = 'yaml'
+ dumper = _YAMLDumper
+
+ def _load(self, config):
+ if not _os_path.exists(self._filename):
+ open(self._filename, 'a').close()
+ with open(self._filename, 'r') as f:
+ data = _yaml.safe_load(f)
+ if data == None:
+ return # empty file
+ return self._from_dict(config, data)
+
+ @staticmethod
+ def _from_dict(config, data):
+ settings = dict([(s.name, s) for s in config.settings])
+ for key,value in data.iteritems():
+ setting = settings[key]
+ if isinstance(setting, (_config.BooleanSetting,
+ _config.NumericSetting,
+ _config.FloatListSetting)):
+ v = value
+ elif isinstance(setting, _config.ConfigListSetting) and value:
+ values = []
+ for v in value:
+ values.append(YAML_Storage._from_dict(
+ setting.config_class(), v))
+ v = values
+ elif isinstance(setting, _config.ConfigSetting) and value:
+ v = YAML_Storage._from_dict(setting.config_class(), value)
+ else:
+ v = setting.convert_from_text(value)
+ config[key] = v
+ return config
+
+ def _save(self, config):
+ data = self._to_dict(config)
+ with open(self._filename, 'w') as f:
+ _yaml.dump(data, stream=f, Dumper=self.dumper,
+ default_flow_style=False)
+
+ @staticmethod
+ def _to_dict(config):
+ data = {}
+ settings = dict([(s.name, s) for s in config.settings])
+ for key,value in config.iteritems():
+ if key in settings:
+ setting = settings[key]
+ if isinstance(setting, (_config.BooleanSetting,
+ _config.NumericSetting,
+ _config.FloatListSetting)):
+ v = value
+ elif isinstance(setting, _config.ConfigListSetting) and value:
+ values = []
+ for v in value:
+ values.append(YAML_Storage._to_dict(v))
+ v = values
+ elif isinstance(setting, _config.ConfigSetting) and value:
+ v = YAML_Storage._to_dict(value)
+ else:
+ v = setting.convert_to_text(value)
+ data[key] = v
+ return data
"""Define a test config object using all the setting types
-Ensure that `build_backend_classes` worked properly for this module:
-
->>> print '\\n'.join([obj for obj in sorted(locals().keys())
-... if obj.endswith('Config')
-... and not obj.startswith('_')])
-HDF5_TestConfig
-TestConfig
-YAML_TestConfig
-
-
-The first time you use them, the file they create will probably be
-empty or not exist.
+The first time you a storage backend, the file it creates will
+probably be empty or not exist.
>>> import os
>>> import tempfile
->>> from hdf5 import pprint_HDF5
+>>> from storage.hdf5 import pprint_HDF5, HDF5_Storage
>>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='h5config-')
>>> os.close(fd)
->>> c = HDF5_TestConfig(filename=filename, group='/base')
+>>> c = TestConfig(storage=HDF5_Storage(filename=filename, group='/base'))
>>> c.load()
Loading will create a stub group group if it hadn't existed before.
>>> pprint_HDF5(filename)
/
/base
->>> print c.dump(from_file=True)
-<BLANKLINE>
Saving fills in all the config values.
no
<HDF5 dataset "bids": shape (), type "|S11">
5.4, 3.2, 1
+ <HDF5 dataset "children": shape (), type "|S1">
+<BLANKLINE>
<HDF5 dataset "daisies": shape (), type "|S2">
13
+ <HDF5 dataset "name": shape (), type "|S1">
+<BLANKLINE>
<HDF5 dataset "species": shape (), type "|S14">
Norwegian Blue
->>> print c.dump(from_file=True)
-age: 1.3
-alive: no
-bids: 5.4, 3.2, 1
-daisies: 13
-species: Norwegian Blue
+ <HDF5 dataset "spouse": shape (), type "|S1">
+<BLANKLINE>
If you want more details, you can dump with help strings.
->>> print c.dump(help=True, from_file=True) # doctest: +NORMALIZE_WHITESPACE
-age: 1.3 (Parrot age in years Default: 1.3.)
-alive: no (The parrot is alive. Default: no. Choices: yes, no)
-bids: 5.4, 3.2, 1 (Prices offered for parrot. Default: 5.4, 3.2, 1.)
-daisies: 13 (Number of daisies pushed up by the parrot.
- Default: 13.)
-species: Norwegian Blue (Type of parrot. Default: Norwegian Blue.
- Choices: Norwegian Blue, Macaw)
+>>> print c.dump(help=True) # doctest: +REPORT_UDIFF, +NORMALIZE_WHITESPACE
+name: (The parrot's name. Default: .)
+species: Norwegian Blue (Type of parrot. Default: Norwegian Blue.
+ Choices: Norwegian Blue, Macaw)
+alive: no (The parrot is alive. Default: no. Choices: yes, no)
+daisies: 13 (Number of daisies pushed up by the parrot.
+ Default: 13.)
+age: 1.3 (Parrot age in years Default: 1.3.)
+bids: 5.4, 3.2, 1 (Prices offered for parrot. Default: 5.4, 3.2, 1.)
+spouse: (This parrot's significant other. Default: .)
+children: (This parrot's children. Default: .)
As you can see from the `age` setting, settings also support `None`,
even if they have numeric types.
from . import LOG as _LOG
from . import config as _config
-from . import util as _util
-from .hdf5 import _HDF5Config
-from .yaml import _YAMLConfig
-
-
-_ALTERNATIVES = { # alternative settings for testing
- 'species': 1,
- 'alive': True,
- 'daisies': None,
- 'age': None,
- 'bids': [],
- }
+from .storage import FileStorage as _FileStorage
+from .storage.hdf5 import HDF5_Storage
+from .storage.yaml import YAML_Storage
class TestConfig (_config.Config):
"Test all the setting types for the h5config module"
settings = [
+ _config.Setting(
+ name='name',
+ help="The parrot's name."),
_config.ChoiceSetting(
name='species',
help='Type of parrot.',
name='bids',
help='Prices offered for parrot.',
default=[5.4, 3.2, 1]),
+ _config.ConfigSetting(
+ name='spouse',
+ help="This parrot's significant other."),
+ _config.ConfigListSetting(
+ name='children',
+ help="This parrot's children."),
]
+# must define self-references after completing the TestConfig class
+for s in TestConfig.settings:
+ if s.name in ['spouse', 'children']:
+ s.config_class = TestConfig
+
-_util.build_backend_classes(_sys.modules[__name__])
+def _alternative_test_config(name):
+ ret = TestConfig()
+ ret['name'] = name
+ return ret
+_ALTERNATIVES = { # alternative settings for testing
+ 'name': 'Captain Flint',
+ 'species': 1,
+ 'alive': True,
+ 'daisies': None,
+ 'age': None,
+ 'bids': [],
+ 'spouse': _alternative_test_config(name='Lory'),
+ 'children': [_alternative_test_config(name=n)
+ for n in ['Washington Post', 'Eli Yale']],
+ }
+# TODO: share children with spouse to test references
-def test(test_class=None):
- if test_class is None:
- classes = []
- for name,obj in sorted(globals().items()):
- try:
- if issubclass(obj, TestConfig):
- classes.append(obj)
- except TypeError:
- pass
- for class_ in classes:
- test(test_class=class_)
+def test(storage=None):
+ if storage is None:
+ storage = [HDF5_Storage, YAML_Storage]
+ for s in storage:
+ test(storage=s)
return
- _LOG.debug('testing {}'.format(test_class))
- _basic_tests(test_class)
- if issubclass(test_class, (_HDF5Config, _YAMLConfig)):
- _uses_file_tests(test_class)
+ _LOG.debug('testing {}'.format(storage))
+ _basic_tests(storage)
+ if issubclass(storage, _FileStorage):
+ _file_storage_tests(storage)
-def _basic_tests(test_class):
+def _basic_tests(storage):
pass
-def _uses_file_tests(test_class):
- fd,filename = _tempfile.mkstemp(suffix='.tmp', prefix='h5config-')
+def _file_storage_tests(storage):
+ fd,filename = _tempfile.mkstemp(
+ suffix='.'+storage.extension, prefix='h5config-')
_os.close(fd)
- kwargs = {'filename': filename}
try:
- x = test_class(**kwargs)
- y = test_class(**kwargs)
- x.dump()
- x.save()
- y.load()
- nd = list(_non_defaults(y))
+ c = TestConfig(storage=storage(filename=filename))
+ c.dump()
+ c.save()
+ c.load()
+ nd = list(_non_defaults(c))
assert not nd, nd
for key,value in _ALTERNATIVES.items():
- x[key] = value
- x.save()
- y.load()
- nd = dict(_non_defaults(y))
- assert nd == _ALTERNATIVES, nd
+ c[key] = value
+ c.save()
+ na = dict(_non_alternatives(c))
+ assert not na, na
+ c.clear()
+ nd = list(_non_defaults(c))
+ assert not nd, nd
+ c.load()
+ na = dict(_non_alternatives(c))
+ assert not na, na
finally:
_os.remove(filename)
-def _non_defaults(test_instance):
+def _non_defaults(config):
for setting in TestConfig.settings:
- value = test_instance[setting.name]
+ value = config[setting.name]
if value != setting.default:
yield (setting.name, value)
+
+def _non_alternatives(config, alternatives=None):
+ if alternatives is None:
+ alternatives = _ALTERNATIVES
+ for setting in TestConfig.settings:
+ value = config[setting.name]
+ alt = alternatives[setting.name]
+ if value != alt:
+ _LOG.error('{} value missmatch: {} vs {}'.format(
+ setting.name, value, alt))
+ yield (setting.name, value)
+ elif type(value) != type(alt):
+ _LOG.error('{} type missmatch: {} vs {}'.format(
+ setting.name, type(value), type(alt)))
+ yield (setting.name, value)
from . import config as _config
from . import log as _log
-from . import util as _util
+from .storage.hdf5 import HDF5_Storage as _HDF5_Storage
+from .storage.yaml import YAML_Storage as _YAML_Storage
class PackageConfig (_config.Config):
`LOG` instance. If you create this instance on your own (for
example, to work around bootstrapping issues), just pass your
instance in as `logger` when you initialize this class.
-
- Because this class occasionally replaces itself, you should always
- work with the version in the package namespace and not with a
- local reference (which may be stale).
-
- Things get a bit interesting because we want to configure the
- package from an internal class. This leads to TODO
"""
- _backed_subclasses = ()
+ possible_storage = [_HDF5_Storage, _YAML_Storage]
settings = [
_config.ChoiceSetting(
name='log-level',
if 'LOG' not in dir(namespace):
namespace.LOG = logger
- def _name(self):
- "Find this instance's name in the bound namespace"
- for attr_name in dir(self._namespace):
- attr = getattr(self._namespace, attr_name, None)
- if id(attr) == id(self):
- return attr_name
- raise IndexError('{} not found in the {} namespace'.format(
- self, self._namespace))
-
- def _replace_self(self, new_config):
- self._logger.debug('replacing {} package config {} with {}'.format(
- self._package_name, self, new_config))
- new_config.setup()
- name = self._name()
- setattr(self._namespace, name, new_config)
-
def setup(self):
self._logger.setLevel(self['log-level'])
if self['syslog']:
def clear(self):
"Replace self with a non-backed version with default settings."
- replacement = self._clear_class(
- package_name=self._package_name,
- namespace=self._namespace,
- logger = self._logger)
- self._replace_self(replacement)
+ super(PackageConfig, self).clear()
+ self._storage = None
def _base_paths(self):
user_basepath = _os_path.join(
self._logger.info('looking for package config file')
basepaths = self._base_paths()
for basepath in basepaths:
- for extension,config in self._backed_subclasses:
- filename = basepath + extension
+ for storage in self.possible_storage:
+ filename = '{}.{}'.format(basepath, storage.extension)
if _os_path.exists(filename):
self._logger.info(
'base_config file found at {}'.format(filename))
- replacement = config(
- filename=filename,
- package_name=self._package_name,
- namespace=self._namespace,
- logger = self._logger)
- replacement.load()
- self._replace_self(replacement)
+ self._storage = storage(filename=filename)
+ self.load()
+ self.setup()
return
else:
self._logger.debug(
'no base_config file at {}'.format(filename))
# create (but don't save) the preferred file
basepath = basepaths[0]
- extension,config = self._backed_subclasses[0]
- filename = basepath + extension
+ storage = self.possible_storage[0]
+ filename = '{}.{}'.format(basepath, storage.extension)
self._logger.info('new base_config file at {}'.format(filename))
- replacement = config(
- filename=filename,
- package_name=self._package_name,
- namespace=self._namespace,
- logger = self._logger)
- self._replace_self(replacement)
-
-
-PackageConfig._clear_class = PackageConfig
-
-
-_util.build_backend_classes(_sys.modules[__name__])
-
-PackageConfig._backed_subclasses = [
- ('.h5', HDF5_PackageConfig),
- ('.yaml', YAML_PackageConfig)
- ]
+ self._storage = storage(filename=filename)
+ self.load()
+ self.setup()
+++ /dev/null
-# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
-#
-# This file is part of h5config.
-#
-# h5config is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation, either version 3 of the License, or (at your
-# option) any later version.
-#
-# h5config is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with h5config. If not, see <http://www.gnu.org/licenses/>.
-
-"""Utilities for building your own config classes
-"""
-
-import os.path as _os_path
-
-from . import LOG as _LOG
-from . import config as _config
-from .hdf5 import _HDF5Config
-from .yaml import _YAMLConfig
-
-
-def build_backend_classes(namespace, objects=None):
- """Define HDF5- and YAML-backed subclasses of the basic `Config` types
-
- At the end of your class-defining module, you should build the
- particular backends. You'll need to pass in a namespace to attach
- the new classes to. I generally use `sys.modules[__name__]` to
- attach the classes to the current module. By default
- `build_backend_classes` will search for base `Config` classes in
- the namespace you pass it. If that doesn't work for you, you can
- explicitly pass it an iterable of `name, object)` tuples.
- """
- if objects is None:
- objects = [(name,getattr(namespace, name, None))
- for name in dir(namespace)]
- for name,obj in objects:
- if (obj != _config.Config and
- type(obj) == type and
- issubclass(obj, _config.Config) and
- not issubclass(obj, _config.BackedConfig)):
- for prefix,base in [('HDF5', _HDF5Config), ('YAML', _YAMLConfig)]:
- _LOG.debug(
- 'creating {} backend for {} in the {} namespace'.format(
- prefix, name, namespace))
- name_ = '%s_%s' % (prefix, name)
- bases = (base, obj)
- dict_ = {}
- class_ = type(name_, bases, dict_)
- setattr(namespace, name_, class_)
+++ /dev/null
-# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
-#
-# This file is part of h5config.
-#
-# h5config is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation, either version 3 of the License, or (at your
-# option) any later version.
-#
-# h5config is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with h5config. If not, see <http://www.gnu.org/licenses/>.
-
-"""HDF5 backend implementation
-"""
-
-from __future__ import absolute_import
-
-import os.path as _os_path
-
-import yaml as _yaml # global PyYAML module
-
-from . import LOG as _LOG
-from . import config as _config
-
-
-class _YAMLDumper (_yaml.SafeDumper):
- def represent_bool(self, data):
- "Use yes/no instead of the default true/false"
- if data:
- value = u'yes'
- else:
- value = u'no'
- return self.represent_scalar(u'tag:yaml.org,2002:bool', value)
-
-
-_YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool)
-
-
-class _YAMLConfig (_config.BackedConfig):
- """Mixin to back a `Config` class with a YAML file.
-
- TODO: Special handling for Choice (enums), FloatList (arrays), etc.?
-
-
- >>> import os
- >>> from .test import YAML_TestConfig
- >>> import os.path
- >>> import tempfile
- >>> fd,filename = tempfile.mkstemp(suffix='.yaml', prefix='pypiezo-')
- >>> os.close(fd)
-
- >>> c = YAML_TestConfig(filename=filename)
- >>> c.load()
-
- Saving writes all the config values to disk.
-
- >>> c['syslog'] = True
- >>> c.save()
- >>> print open(c.filename, 'r').read()
- age: '1.3'
- alive: no
- bids: 5.4, 3.2, 1
- daisies: '13'
- species: Norwegian Blue
- <BLANKLINE>
-
- Loading reads the config files from disk.
-
- >>> c = YAML_TestConfig(filename=filename)
- >>> c.load()
- >>> print c.dump()
- species: Norwegian Blue
- alive: no
- daisies: 13
- age: 1.3
- bids: 5.4, 3.2, 1.0
-
- Cleanup our temporary config file.
-
- >>> os.remove(filename)
- """
- dumper = _YAMLDumper
-
- def __init__(self, filename=None, **kwargs):
- super(_YAMLConfig, self).__init__(**kwargs)
- self.filename = filename
-
- def load(self):
- if not _os_path.exists(self.filename):
- open(self.filename, 'a').close()
- with open(self.filename, 'r') as f:
- data = _yaml.safe_load(f)
- if data == None:
- return # empty file
- settings = dict([(s.name, s) for s in self.settings])
- for key,value in data.iteritems():
- setting = settings[key]
- if isinstance(setting, _config.BooleanSetting):
- v = value
- else:
- v = setting.convert_from_text(value)
- self[key] = v
-
- def save(self):
- data = {}
- settings = dict([(s.name, s) for s in self.settings])
- for key,value in self.iteritems():
- if key in settings:
- setting = settings[key]
- if isinstance(setting, _config.BooleanSetting):
- v = value
- else:
- v = setting.convert_to_text(value)
- data[key] = v
- with open(self.filename, 'w') as f:
- _yaml.dump(data, stream=f, Dumper=self.dumper,
- default_flow_style=False)
'Topic :: Scientific/Engineering',
'Topic :: Software Development :: Libraries :: Python Modules',
],
- packages=['h5config'],
+ packages=[package_name, '{}.storage'.format(package_name)],
provides=['{} ({})'.format(package_name, __version__)],
)
projects / h5config.git / commitdiff