From ae159d096e4a9c7e2c93ba586067b7a230229010 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 29 Jul 2011 00:13:03 -0400 Subject: [PATCH] Add ability to nest Configs. I also restructured the storage backends as distinct helper classes rather than Config subclasses. This means that you can use several Storage instances to save a single Config instance (although each Config instance will have a single default Storage instance stored in config._storage). This avoids all the automatic subclassing shenanigans, and makes loading and saving from different backends much easier. --- h5config/config.py | 102 +++++++++++----- h5config/hdf5.py | 181 ---------------------------- h5config/storage/__init__.py | 45 +++++++ h5config/storage/hdf5.py | 220 +++++++++++++++++++++++++++++++++++ h5config/storage/yaml.py | 153 ++++++++++++++++++++++++ h5config/test.py | 174 +++++++++++++++------------ h5config/tools.py | 73 +++--------- h5config/util.py | 56 --------- h5config/yaml.py | 122 ------------------- setup.py | 2 +- 10 files changed, 603 insertions(+), 525 deletions(-) delete mode 100644 h5config/hdf5.py create mode 100644 h5config/storage/__init__.py create mode 100644 h5config/storage/hdf5.py create mode 100644 h5config/storage/yaml.py delete mode 100644 h5config/util.py delete mode 100644 h5config/yaml.py diff --git a/h5config/config.py b/h5config/config.py index 51dc8d9..e8ff023 100644 --- a/h5config/config.py +++ b/h5config/config.py @@ -18,6 +18,8 @@ """The basic h5config classes """ +import copy as _copy + class Setting (object): "A named setting with arbitrart text values." @@ -38,10 +40,10 @@ class Setting (object): return ret.strip() def convert_from_text(self, value): - return value + return value or None def convert_to_text(self, value): - return value + return value or '' class ChoiceSetting (Setting): @@ -105,12 +107,9 @@ class BooleanSetting (ChoiceSetting): >>> s.help() 'Default: no. Choices: yes, no' """ - def __init__(self, **kwargs): - assert 'choices' not in kwargs - if 'default' not in kwargs: - kwargs['default'] = False + def __init__(self, default=False, **kwargs): super(BooleanSetting, self).__init__( - choices=[('yes', True), ('no', False)], **kwargs) + choices=[('yes', True), ('no', False)], default=default, **kwargs) class NumericSetting (Setting): @@ -125,12 +124,8 @@ class NumericSetting (Setting): >>> s.convert_to_text(13) '13' """ - _default_value = 0 - - def __init__(self, **kwargs): - if 'default' not in kwargs: - kwargs['default'] = self._default_value - super(NumericSetting, self).__init__(**kwargs) + def __init__(self, default=0, **kwargs): + super(NumericSetting, self).__init__(default=default, **kwargs) def convert_to_text(self, value): return str(value) @@ -153,7 +148,8 @@ class IntegerSetting (NumericSetting): >>> s.convert_from_text('8') 8 """ - _default_value = 1 + def __init__(self, default=1, **kwargs): + super(IntegerSetting, self).__init__(default=default, **kwargs) def _convert_from_text(self, value): return int(value) @@ -173,7 +169,8 @@ class FloatSetting (NumericSetting): ... print 'caught a ValueError' caught a ValueError """ - _default_value = 1.0 + def __init__(self, default=1.0, **kwargs): + super(FloatSetting, self).__init__(default=default, **kwargs) def _convert_from_text(self, value): return float(value) @@ -194,10 +191,8 @@ class FloatListSetting (Setting): >>> s.convert_from_text('') [] """ - def __init__(self, **kwargs): - if 'default' not in kwargs: - kwargs['default'] = [] - super(FloatListSetting, self).__init__(**kwargs) + def __init__(self, default=[], **kwargs): + super(FloatListSetting, self).__init__(default=default, **kwargs) def _convert_from_text(self, value): if value is None: @@ -207,7 +202,7 @@ class FloatListSetting (Setting): def convert_from_text(self, value): if value is None: return None - elif value == '': + elif value in ['', []]: return [] return [self._convert_from_text(x) for x in value.split(',')] @@ -217,15 +212,56 @@ class FloatListSetting (Setting): return ', '.join([str(x) for x in value]) +class ConfigSetting (Setting): + """A setting that holds a pointer to a child `Config` class + + This allows you to nest `Config`\s, which is a useful way to + contain complexity. In order to save such a config, the backend + must be able to handle hierarchical storage (possibly via + references). + + For example, a configurable AFM may contain a configurable piezo + scanner, as well as a configurable stepper motor. + """ + def __init__(self, config_class=None, **kwargs): + super(ConfigSetting, self).__init__(**kwargs) + self.config_class = config_class + + +class ConfigListSetting (ConfigSetting): + """A setting that holds a list of child `Config` classes + + For example, a piezo scanner with several axes. + """ + def __init__(self, default=[], **kwargs): + super(ConfigListSetting, self).__init__(**kwargs) + + class Config (dict): "A class with a list `._keys` of `Setting`\s." settings = [] - def __init__(self): + def __init__(self, storage=None): + super(Config, self).__init__() + self.clear() + self._storage = storage + + def __repr__(self): + return '<{} {}>'.format(self.__class__.__name__, id(self)) + + def clear(self): + super(Config, self).clear() for s in self.settings: - self[s.name] = s.default + # copy to avoid ambiguity with mutable defaults + self[s.name] = _copy.deepcopy(s.default) + + def load(self, merge=False, **kwargs): + self._storage.load(config=self, merge=merge, **kwargs) - def dump(self, help=False): + def save(self, merge=False, **kwargs): + self._storage.save(config=self, merge=merge, **kwargs) + + def dump(self, help=False, prefix=''): """Return all settings and their values as a string >>> class MyConfig (Config): @@ -260,6 +296,17 @@ class Config (dict): lines = [] for setting in self.settings: name = setting.name + value = self[name] + if isinstance(setting, ConfigSetting): + if value is not None: + lines.append(value.dump(help=help, prefix=prefix+' ')) + continue + elif isinstance(setting, ConfigListSetting): + if value: + for config in value: + lines.append( + config.dump(help=help, prefix=prefix+' ')) + continue value_string = setting.convert_to_text(self[name]) if help: help_string = '\t({})'.format(setting.help()) @@ -267,12 +314,3 @@ class Config (dict): help_string = '' lines.append('{}: {}{}'.format(name, value_string, help_string)) return '\n'.join(lines) - - -class BackedConfig (Config): - "A `Config` instance with some kind of storage interface" - def load(self): - raise NotImplementedError() - - def save(self): - raise NotImplementedError() diff --git a/h5config/hdf5.py b/h5config/hdf5.py deleted file mode 100644 index 996fadf..0000000 --- a/h5config/hdf5.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright (C) 2011 W. Trevor King -# -# This file is part of h5config. -# -# h5config is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# h5config is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with h5config. If not, see . - -"""HDF5 backend implementation -""" - -import h5py as _h5py - -from . import LOG as _LOG -from . import config as _config - - -def pprint_HDF5(*args, **kwargs): - print pformat_HDF5(*args, **kwargs) - -def pformat_HDF5(filename, group='/'): - f = _h5py.File(filename, 'r') - cwg = f[group] - return '\n'.join(_pformat_hdf5(cwg)) - -def _pformat_hdf5(cwg, depth=0): - lines = [] - lines.append(' '*depth + cwg.name) - depth += 1 - for key,value in cwg.iteritems(): - if isinstance(value, _h5py.Group): - lines.extend(_pformat_hdf5(value, depth)) - elif isinstance(value, _h5py.Dataset): - lines.append(' '*depth + str(value)) - lines.append(' '*(depth+1) + str(value[...])) - else: - lines.append(' '*depth + str(value)) - return lines - -def h5_create_group(cwg, path): - "Create the group where the settings are stored (if necessary)." - if path == '/': - return cwg - gpath = [''] - for group in path.strip('/').split('/'): - gpath.append(group) - if group not in cwg.keys(): - _LOG.debug('creating group %s in %s' - % ('/'.join(gpath), cwg.file)) - cwg.create_group(group) - cwg = cwg[group] - return cwg - - -class _HDF5Config (_config.BackedConfig): - """Mixin to back a `_Config` class with an HDF5 file. - - TODO: Special handling for Choice (enums), FloatList (arrays), etc.? - - The `.save` and `.load` methods have an optional `group` argument - that allows you to save and load settings from an externally - opened HDF5 file. This can make it easier to stash several - related `_Config` classes in a single file. For example - - >>> import os - >>> import tempfile - >>> from .test import HDF5_TestConfig - >>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='pypiezo-') - >>> os.close(fd) - - >>> f = _h5py.File(filename, 'a') - >>> c = HDF5_TestConfig(filename='untouched_file.h5', - ... group='/untouched/group') - >>> c['syslog'] = True - >>> group = f.create_group('base') - >>> c.save(group) - >>> pprint_HDF5(filename) - / - /base - - 1.3 - - no - - 5.4, 3.2, 1 - - 13 - - Norwegian Blue - >>> d = HDF5_TestConfig(filename='untouched_file.h5', - ... group='/untouched/group') - >>> d.load(group) - >>> d['alive'] - False - - >>> f.close() - >>> os.remove(filename) - """ - def __init__(self, filename=None, group='/', **kwargs): - super(_HDF5Config, self).__init__(**kwargs) - self.filename = filename - assert group.startswith('/'), group - if not group.endswith('/'): - group += '/' - self.group = group - self._file_checked = False - - def _check_file(self): - if self._file_checked: - return - self._setup_file() - self._file_checked = True - - def _setup_file(self): - f = _h5py.File(self.filename, 'a') - cwg = f # current working group - h5_create_group(cwg, self.group) - f.close() - - def dump(self, help=False, from_file=False): - """Return the relevant group in `self.filename` as a string - - Extends the base :meth:`dump` by adding the `from_file` - option. If `from_file` is true, dump all entries that - currently exist in the relevant group, rather than listing all - settings defined in the instance dictionary. - """ - if from_file: - self._check_file() - f = _h5py.File(self.filename, 'r') - cwg = f[self.group] - lines = [] - settings = dict([(s.name, s) for s in self.settings]) - for key,value in cwg.iteritems(): - if help and key in settings: - help_string = '\t(%s)' % settings[key].help() - else: - help_string = '' - lines.append('%s: %s%s' % (key, value[...], help_string)) - return '\n'.join(lines) - return super(_HDF5Config, self).dump(help=help) - - def load(self, group=None): - if group is None: - self._check_file() - f = _h5py.File(self.filename, 'r') - group = f[self.group] - else: - f = None - for s in self.settings: - if s.name not in group.keys(): - continue - self[s.name] = s.convert_from_text(group[s.name][...]) - if f: - f.close() - - def save(self, group=None): - if group is None: - self._check_file() - f = _h5py.File(self.filename, 'a') - group = f[self.group] - else: - f = None - for s in self.settings: - value = s.convert_to_text(self[s.name]) - try: - del group[s.name] - except KeyError: - pass - group[s.name] = value - if f: - f.close() diff --git a/h5config/storage/__init__.py b/h5config/storage/__init__.py new file mode 100644 index 0000000..b9d72e5 --- /dev/null +++ b/h5config/storage/__init__.py @@ -0,0 +1,45 @@ +# Copyright (C) 2011 W. Trevor King +# +# This file is part of h5config. +# +# h5config is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# h5config is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with h5config. If not, see . + + +class Storage (object): + "A storage bakend for loading and saving `Config` instances" + def load(self, config, merge=False, **kwargs): + if merge: + self.clear() + self._load(config=config, **kwargs) + config._storage = self + + def _load(self, config, **kwargs): + raise NotImplementedError() + + def save(self, config, merge=False, **kwargs): + if merge: + self.clear() + self._save(config=config, **kwargs) + config._storage = self + + def _save(self, config, **kwargs): + raise NotImplementedError() + + +class FileStorage (Storage): + "`Config` storage backend by a single file" + extension = None + + def __init__(self, filename=None): + self._filename = filename diff --git a/h5config/storage/hdf5.py b/h5config/storage/hdf5.py new file mode 100644 index 0000000..b488c37 --- /dev/null +++ b/h5config/storage/hdf5.py @@ -0,0 +1,220 @@ +# Copyright (C) 2011 W. Trevor King +# +# This file is part of h5config. +# +# h5config is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# h5config is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with h5config. If not, see . + +"""HDF5 backend implementation +""" + +import h5py as _h5py + +from .. import LOG as _LOG +from .. import config as _config +from . import FileStorage as _FileStorage + + +def pprint_HDF5(*args, **kwargs): + print pformat_HDF5(*args, **kwargs) + +def pformat_HDF5(filename, group='/'): + with _h5py.File(filename, 'r') as f: + cwg = f[group] + ret = '\n'.join(_pformat_hdf5(cwg)) + return ret + +def _pformat_hdf5(cwg, depth=0): + lines = [] + lines.append(' '*depth + cwg.name) + depth += 1 + for key,value in cwg.iteritems(): + if isinstance(value, _h5py.Group): + lines.extend(_pformat_hdf5(value, depth)) + elif isinstance(value, _h5py.Dataset): + lines.append(' '*depth + str(value)) + lines.append(' '*(depth+1) + str(value[...])) + else: + lines.append(' '*depth + str(value)) + return lines + +def h5_create_group(cwg, path, force=False): + "Create the group where the settings are stored (if necessary)." + if path == '/': + return cwg + gpath = [''] + for group in path.strip('/').split('/'): + gpath.append(group) + if group not in cwg.keys(): + _LOG.debug('creating group {} in {}'.format( + '/'.join(gpath), cwg.file)) + cwg.create_group(group) + _cwg = cwg[group] + if isinstance(_cwg, _h5py.Dataset): + if force: + _LOG.info('overwrite {} in {} ({}) with a group'.format( + '/'.join(gpath), _cwg.file, _cwg)) + del cwg[group] + _cwg = cwg.create_group(group) + else: + raise ValueError(_cwg) + cwg = _cwg + return cwg + + +class HDF5_Storage (_FileStorage): + """Back a `Config` class with an HDF5 file. + + TODO: Special handling for Choice (enums), FloatList (arrays), etc.? + + The `.save` and `.load` methods have an optional `group` argument + that allows you to save and load settings from an externally + opened HDF5 file. This can make it easier to stash several + related `Config` classes in a single file. For example + + >>> import os + >>> import tempfile + >>> from ..test import TestConfig + >>> fd,filename = tempfile.mkstemp( + ... suffix='.'+HDF5_Storage.extension, prefix='pypiezo-') + >>> os.close(fd) + + >>> f = _h5py.File(filename, 'a') + >>> c = TestConfig(storage=HDF5_Storage( + ... filename='untouched_file.h5', group='/untouched/group')) + >>> c['alive'] = True + >>> group = f.create_group('base') + >>> c.save(group=group) + >>> pprint_HDF5(filename) # doctest: +REPORT_UDIFF + / + /base + + 1.3 + + yes + + 5.4, 3.2, 1 + + + + 13 + + + + Norwegian Blue + + + >>> c.clear() + >>> c['alive'] + False + >>> c.load(group=group) + >>> c['alive'] + True + + >>> f.close() + >>> os.remove(filename) + """ + extension = 'h5' + + def __init__(self, group='/', **kwargs): + super(HDF5_Storage, self).__init__(**kwargs) + assert group.startswith('/'), group + if not group.endswith('/'): + group += '/' + self.group = group + self._file_checked = False + + def _check_file(self): + if self._file_checked: + return + self._setup_file() + self._file_checked = True + + def _setup_file(self): + with _h5py.File(self._filename, 'a') as f: + cwg = f # current working group + h5_create_group(cwg, self.group) + + def _load(self, config, group=None): + f = None + try: + if group is None: + self._check_file() + f = _h5py.File(self._filename, 'r') + group = f[self.group] + for s in config.settings: + if s.name not in group.keys(): + continue + elif isinstance(s, _config.ConfigListSetting): + try: + cwg = h5_create_group(group, s.name) + except ValueError: + pass + else: + value = [] + for i in sorted(int(x) for x in cwg.keys()): + instance = s.config_class() + try: + _cwg = h5_create_group(cwg, str(i)) + except ValueError: + pass + else: + self._load(config=instance, group=_cwg) + value.append(instance) + config[s.name] = value + elif isinstance(s, _config.ConfigSetting): + try: + cwg = h5_create_group(group, s.name) + except ValueError: + pass + else: + if not config[s.name]: + config[s.name] = s.config_class() + self._load(config=config[s.name], group=cwg) + else: + config[s.name] = s.convert_from_text(group[s.name][...]) + finally: + if f: + f.close() + + def _save(self, config, group=None): + f = None + try: + if group is None: + self._check_file() + f = _h5py.File(self._filename, 'a') + group = f[self.group] + for s in config.settings: + if isinstance(s, _config.ConfigListSetting): + configs = config[s.name] + if configs: + cwg = h5_create_group(group, s.name, force=True) + for i,cfg in enumerate(configs): + _cwg = h5_create_group(cwg, str(i), force=True) + self._save(config=cfg, group=_cwg) + continue + elif isinstance(s, _config.ConfigSetting): + cfg = config[s.name] + if cfg: + cwg = h5_create_group(group, s.name, force=True) + self._save(config=cfg, group=cwg) + continue + value = s.convert_to_text(config[s.name]) + try: + del group[s.name] + except KeyError: + pass + group[s.name] = value + finally: + if f: + f.close() diff --git a/h5config/storage/yaml.py b/h5config/storage/yaml.py new file mode 100644 index 0000000..ec06aba --- /dev/null +++ b/h5config/storage/yaml.py @@ -0,0 +1,153 @@ +# Copyright (C) 2011 W. Trevor King +# +# This file is part of h5config. +# +# h5config is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# h5config is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with h5config. If not, see . + +"""HDF5 backend implementation +""" + +from __future__ import absolute_import + +import os.path as _os_path + +import yaml as _yaml # global PyYAML module + +from .. import LOG as _LOG +from .. import config as _config +from . import FileStorage as _FileStorage + + +class _YAMLDumper (_yaml.SafeDumper): + def represent_bool(self, data): + "Use yes/no instead of the default true/false" + if data: + value = u'yes' + else: + value = u'no' + return self.represent_scalar(u'tag:yaml.org,2002:bool', value) + + +_YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool) + + +class YAML_Storage (_FileStorage): + """Back a `Config` class with a YAML file. + + TODO: Special handling for Choice (enums), etc.? + + >>> import os + >>> from ..test import TestConfig + >>> import os.path + >>> import tempfile + >>> fd,filename = tempfile.mkstemp( + ... suffix='.'+YAML_Storage.extension, prefix='pypiezo-') + >>> os.close(fd) + + >>> c = TestConfig(storage=YAML_Storage(filename=filename)) + >>> c.load() + + Saving writes all the config values to disk. + + >>> c['alive'] = True + >>> c.save() + >>> print open(filename, 'r').read() # doctest: +REPORT_UDIFF + age: 1.3 + alive: yes + bids: + - 5.4 + - 3.2 + - 1 + children: '' + daisies: 13 + name: '' + species: Norwegian Blue + spouse: '' + + + Loading reads the config files from disk. + + >>> c.clear() + >>> c['alive'] + False + >>> c.load() + >>> c['alive'] + True + + Cleanup our temporary config file. + + >>> os.remove(filename) + """ + extension = 'yaml' + dumper = _YAMLDumper + + def _load(self, config): + if not _os_path.exists(self._filename): + open(self._filename, 'a').close() + with open(self._filename, 'r') as f: + data = _yaml.safe_load(f) + if data == None: + return # empty file + return self._from_dict(config, data) + + @staticmethod + def _from_dict(config, data): + settings = dict([(s.name, s) for s in config.settings]) + for key,value in data.iteritems(): + setting = settings[key] + if isinstance(setting, (_config.BooleanSetting, + _config.NumericSetting, + _config.FloatListSetting)): + v = value + elif isinstance(setting, _config.ConfigListSetting) and value: + values = [] + for v in value: + values.append(YAML_Storage._from_dict( + setting.config_class(), v)) + v = values + elif isinstance(setting, _config.ConfigSetting) and value: + v = YAML_Storage._from_dict(setting.config_class(), value) + else: + v = setting.convert_from_text(value) + config[key] = v + return config + + def _save(self, config): + data = self._to_dict(config) + with open(self._filename, 'w') as f: + _yaml.dump(data, stream=f, Dumper=self.dumper, + default_flow_style=False) + + @staticmethod + def _to_dict(config): + data = {} + settings = dict([(s.name, s) for s in config.settings]) + for key,value in config.iteritems(): + if key in settings: + setting = settings[key] + if isinstance(setting, (_config.BooleanSetting, + _config.NumericSetting, + _config.FloatListSetting)): + v = value + elif isinstance(setting, _config.ConfigListSetting) and value: + values = [] + for v in value: + values.append(YAML_Storage._to_dict(v)) + v = values + elif isinstance(setting, _config.ConfigSetting) and value: + v = YAML_Storage._to_dict(value) + else: + v = setting.convert_to_text(value) + data[key] = v + return data diff --git a/h5config/test.py b/h5config/test.py index 27eb67c..5734a27 100644 --- a/h5config/test.py +++ b/h5config/test.py @@ -17,27 +17,17 @@ """Define a test config object using all the setting types -Ensure that `build_backend_classes` worked properly for this module: - ->>> print '\\n'.join([obj for obj in sorted(locals().keys()) -... if obj.endswith('Config') -... and not obj.startswith('_')]) -HDF5_TestConfig -TestConfig -YAML_TestConfig - - -The first time you use them, the file they create will probably be -empty or not exist. +The first time you a storage backend, the file it creates will +probably be empty or not exist. >>> import os >>> import tempfile ->>> from hdf5 import pprint_HDF5 +>>> from storage.hdf5 import pprint_HDF5, HDF5_Storage >>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='h5config-') >>> os.close(fd) ->>> c = HDF5_TestConfig(filename=filename, group='/base') +>>> c = TestConfig(storage=HDF5_Storage(filename=filename, group='/base')) >>> c.load() Loading will create a stub group group if it hadn't existed before. @@ -45,8 +35,6 @@ Loading will create a stub group group if it hadn't existed before. >>> pprint_HDF5(filename) / /base ->>> print c.dump(from_file=True) - Saving fills in all the config values. @@ -61,27 +49,30 @@ Saving fills in all the config values. no 5.4, 3.2, 1 + + 13 + + Norwegian Blue ->>> print c.dump(from_file=True) -age: 1.3 -alive: no -bids: 5.4, 3.2, 1 -daisies: 13 -species: Norwegian Blue + + If you want more details, you can dump with help strings. ->>> print c.dump(help=True, from_file=True) # doctest: +NORMALIZE_WHITESPACE -age: 1.3 (Parrot age in years Default: 1.3.) -alive: no (The parrot is alive. Default: no. Choices: yes, no) -bids: 5.4, 3.2, 1 (Prices offered for parrot. Default: 5.4, 3.2, 1.) -daisies: 13 (Number of daisies pushed up by the parrot. - Default: 13.) -species: Norwegian Blue (Type of parrot. Default: Norwegian Blue. - Choices: Norwegian Blue, Macaw) +>>> print c.dump(help=True) # doctest: +REPORT_UDIFF, +NORMALIZE_WHITESPACE +name: (The parrot's name. Default: .) +species: Norwegian Blue (Type of parrot. Default: Norwegian Blue. + Choices: Norwegian Blue, Macaw) +alive: no (The parrot is alive. Default: no. Choices: yes, no) +daisies: 13 (Number of daisies pushed up by the parrot. + Default: 13.) +age: 1.3 (Parrot age in years Default: 1.3.) +bids: 5.4, 3.2, 1 (Prices offered for parrot. Default: 5.4, 3.2, 1.) +spouse: (This parrot's significant other. Default: .) +children: (This parrot's children. Default: .) As you can see from the `age` setting, settings also support `None`, even if they have numeric types. @@ -97,23 +88,17 @@ import tempfile as _tempfile from . import LOG as _LOG from . import config as _config -from . import util as _util -from .hdf5 import _HDF5Config -from .yaml import _YAMLConfig - - -_ALTERNATIVES = { # alternative settings for testing - 'species': 1, - 'alive': True, - 'daisies': None, - 'age': None, - 'bids': [], - } +from .storage import FileStorage as _FileStorage +from .storage.hdf5 import HDF5_Storage +from .storage.yaml import YAML_Storage class TestConfig (_config.Config): "Test all the setting types for the h5config module" settings = [ + _config.Setting( + name='name', + help="The parrot's name."), _config.ChoiceSetting( name='species', help='Type of parrot.', @@ -135,55 +120,94 @@ class TestConfig (_config.Config): name='bids', help='Prices offered for parrot.', default=[5.4, 3.2, 1]), + _config.ConfigSetting( + name='spouse', + help="This parrot's significant other."), + _config.ConfigListSetting( + name='children', + help="This parrot's children."), ] +# must define self-references after completing the TestConfig class +for s in TestConfig.settings: + if s.name in ['spouse', 'children']: + s.config_class = TestConfig + -_util.build_backend_classes(_sys.modules[__name__]) +def _alternative_test_config(name): + ret = TestConfig() + ret['name'] = name + return ret +_ALTERNATIVES = { # alternative settings for testing + 'name': 'Captain Flint', + 'species': 1, + 'alive': True, + 'daisies': None, + 'age': None, + 'bids': [], + 'spouse': _alternative_test_config(name='Lory'), + 'children': [_alternative_test_config(name=n) + for n in ['Washington Post', 'Eli Yale']], + } +# TODO: share children with spouse to test references -def test(test_class=None): - if test_class is None: - classes = [] - for name,obj in sorted(globals().items()): - try: - if issubclass(obj, TestConfig): - classes.append(obj) - except TypeError: - pass - for class_ in classes: - test(test_class=class_) +def test(storage=None): + if storage is None: + storage = [HDF5_Storage, YAML_Storage] + for s in storage: + test(storage=s) return - _LOG.debug('testing {}'.format(test_class)) - _basic_tests(test_class) - if issubclass(test_class, (_HDF5Config, _YAMLConfig)): - _uses_file_tests(test_class) + _LOG.debug('testing {}'.format(storage)) + _basic_tests(storage) + if issubclass(storage, _FileStorage): + _file_storage_tests(storage) -def _basic_tests(test_class): +def _basic_tests(storage): pass -def _uses_file_tests(test_class): - fd,filename = _tempfile.mkstemp(suffix='.tmp', prefix='h5config-') +def _file_storage_tests(storage): + fd,filename = _tempfile.mkstemp( + suffix='.'+storage.extension, prefix='h5config-') _os.close(fd) - kwargs = {'filename': filename} try: - x = test_class(**kwargs) - y = test_class(**kwargs) - x.dump() - x.save() - y.load() - nd = list(_non_defaults(y)) + c = TestConfig(storage=storage(filename=filename)) + c.dump() + c.save() + c.load() + nd = list(_non_defaults(c)) assert not nd, nd for key,value in _ALTERNATIVES.items(): - x[key] = value - x.save() - y.load() - nd = dict(_non_defaults(y)) - assert nd == _ALTERNATIVES, nd + c[key] = value + c.save() + na = dict(_non_alternatives(c)) + assert not na, na + c.clear() + nd = list(_non_defaults(c)) + assert not nd, nd + c.load() + na = dict(_non_alternatives(c)) + assert not na, na finally: _os.remove(filename) -def _non_defaults(test_instance): +def _non_defaults(config): for setting in TestConfig.settings: - value = test_instance[setting.name] + value = config[setting.name] if value != setting.default: yield (setting.name, value) + +def _non_alternatives(config, alternatives=None): + if alternatives is None: + alternatives = _ALTERNATIVES + for setting in TestConfig.settings: + value = config[setting.name] + alt = alternatives[setting.name] + if value != alt: + _LOG.error('{} value missmatch: {} vs {}'.format( + setting.name, value, alt)) + yield (setting.name, value) + elif type(value) != type(alt): + _LOG.error('{} type missmatch: {} vs {}'.format( + setting.name, type(value), type(alt))) + yield (setting.name, value) diff --git a/h5config/tools.py b/h5config/tools.py index f7cd2e4..48c6832 100644 --- a/h5config/tools.py +++ b/h5config/tools.py @@ -31,7 +31,8 @@ import sys as _sys from . import config as _config from . import log as _log -from . import util as _util +from .storage.hdf5 import HDF5_Storage as _HDF5_Storage +from .storage.yaml import YAML_Storage as _YAML_Storage class PackageConfig (_config.Config): @@ -41,15 +42,8 @@ class PackageConfig (_config.Config): `LOG` instance. If you create this instance on your own (for example, to work around bootstrapping issues), just pass your instance in as `logger` when you initialize this class. - - Because this class occasionally replaces itself, you should always - work with the version in the package namespace and not with a - local reference (which may be stale). - - Things get a bit interesting because we want to configure the - package from an internal class. This leads to TODO """ - _backed_subclasses = () + possible_storage = [_HDF5_Storage, _YAML_Storage] settings = [ _config.ChoiceSetting( name='log-level', @@ -80,22 +74,6 @@ class PackageConfig (_config.Config): if 'LOG' not in dir(namespace): namespace.LOG = logger - def _name(self): - "Find this instance's name in the bound namespace" - for attr_name in dir(self._namespace): - attr = getattr(self._namespace, attr_name, None) - if id(attr) == id(self): - return attr_name - raise IndexError('{} not found in the {} namespace'.format( - self, self._namespace)) - - def _replace_self(self, new_config): - self._logger.debug('replacing {} package config {} with {}'.format( - self._package_name, self, new_config)) - new_config.setup() - name = self._name() - setattr(self._namespace, name, new_config) - def setup(self): self._logger.setLevel(self['log-level']) if self['syslog']: @@ -111,11 +89,8 @@ class PackageConfig (_config.Config): def clear(self): "Replace self with a non-backed version with default settings." - replacement = self._clear_class( - package_name=self._package_name, - namespace=self._namespace, - logger = self._logger) - self._replace_self(replacement) + super(PackageConfig, self).clear() + self._storage = None def _base_paths(self): user_basepath = _os_path.join( @@ -130,41 +105,23 @@ class PackageConfig (_config.Config): self._logger.info('looking for package config file') basepaths = self._base_paths() for basepath in basepaths: - for extension,config in self._backed_subclasses: - filename = basepath + extension + for storage in self.possible_storage: + filename = '{}.{}'.format(basepath, storage.extension) if _os_path.exists(filename): self._logger.info( 'base_config file found at {}'.format(filename)) - replacement = config( - filename=filename, - package_name=self._package_name, - namespace=self._namespace, - logger = self._logger) - replacement.load() - self._replace_self(replacement) + self._storage = storage(filename=filename) + self.load() + self.setup() return else: self._logger.debug( 'no base_config file at {}'.format(filename)) # create (but don't save) the preferred file basepath = basepaths[0] - extension,config = self._backed_subclasses[0] - filename = basepath + extension + storage = self.possible_storage[0] + filename = '{}.{}'.format(basepath, storage.extension) self._logger.info('new base_config file at {}'.format(filename)) - replacement = config( - filename=filename, - package_name=self._package_name, - namespace=self._namespace, - logger = self._logger) - self._replace_self(replacement) - - -PackageConfig._clear_class = PackageConfig - - -_util.build_backend_classes(_sys.modules[__name__]) - -PackageConfig._backed_subclasses = [ - ('.h5', HDF5_PackageConfig), - ('.yaml', YAML_PackageConfig) - ] + self._storage = storage(filename=filename) + self.load() + self.setup() diff --git a/h5config/util.py b/h5config/util.py deleted file mode 100644 index 9f1742a..0000000 --- a/h5config/util.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (C) 2011 W. Trevor King -# -# This file is part of h5config. -# -# h5config is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# h5config is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with h5config. If not, see . - -"""Utilities for building your own config classes -""" - -import os.path as _os_path - -from . import LOG as _LOG -from . import config as _config -from .hdf5 import _HDF5Config -from .yaml import _YAMLConfig - - -def build_backend_classes(namespace, objects=None): - """Define HDF5- and YAML-backed subclasses of the basic `Config` types - - At the end of your class-defining module, you should build the - particular backends. You'll need to pass in a namespace to attach - the new classes to. I generally use `sys.modules[__name__]` to - attach the classes to the current module. By default - `build_backend_classes` will search for base `Config` classes in - the namespace you pass it. If that doesn't work for you, you can - explicitly pass it an iterable of `name, object)` tuples. - """ - if objects is None: - objects = [(name,getattr(namespace, name, None)) - for name in dir(namespace)] - for name,obj in objects: - if (obj != _config.Config and - type(obj) == type and - issubclass(obj, _config.Config) and - not issubclass(obj, _config.BackedConfig)): - for prefix,base in [('HDF5', _HDF5Config), ('YAML', _YAMLConfig)]: - _LOG.debug( - 'creating {} backend for {} in the {} namespace'.format( - prefix, name, namespace)) - name_ = '%s_%s' % (prefix, name) - bases = (base, obj) - dict_ = {} - class_ = type(name_, bases, dict_) - setattr(namespace, name_, class_) diff --git a/h5config/yaml.py b/h5config/yaml.py deleted file mode 100644 index 642b7a9..0000000 --- a/h5config/yaml.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (C) 2011 W. Trevor King -# -# This file is part of h5config. -# -# h5config is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# h5config is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with h5config. If not, see . - -"""HDF5 backend implementation -""" - -from __future__ import absolute_import - -import os.path as _os_path - -import yaml as _yaml # global PyYAML module - -from . import LOG as _LOG -from . import config as _config - - -class _YAMLDumper (_yaml.SafeDumper): - def represent_bool(self, data): - "Use yes/no instead of the default true/false" - if data: - value = u'yes' - else: - value = u'no' - return self.represent_scalar(u'tag:yaml.org,2002:bool', value) - - -_YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool) - - -class _YAMLConfig (_config.BackedConfig): - """Mixin to back a `Config` class with a YAML file. - - TODO: Special handling for Choice (enums), FloatList (arrays), etc.? - - - >>> import os - >>> from .test import YAML_TestConfig - >>> import os.path - >>> import tempfile - >>> fd,filename = tempfile.mkstemp(suffix='.yaml', prefix='pypiezo-') - >>> os.close(fd) - - >>> c = YAML_TestConfig(filename=filename) - >>> c.load() - - Saving writes all the config values to disk. - - >>> c['syslog'] = True - >>> c.save() - >>> print open(c.filename, 'r').read() - age: '1.3' - alive: no - bids: 5.4, 3.2, 1 - daisies: '13' - species: Norwegian Blue - - - Loading reads the config files from disk. - - >>> c = YAML_TestConfig(filename=filename) - >>> c.load() - >>> print c.dump() - species: Norwegian Blue - alive: no - daisies: 13 - age: 1.3 - bids: 5.4, 3.2, 1.0 - - Cleanup our temporary config file. - - >>> os.remove(filename) - """ - dumper = _YAMLDumper - - def __init__(self, filename=None, **kwargs): - super(_YAMLConfig, self).__init__(**kwargs) - self.filename = filename - - def load(self): - if not _os_path.exists(self.filename): - open(self.filename, 'a').close() - with open(self.filename, 'r') as f: - data = _yaml.safe_load(f) - if data == None: - return # empty file - settings = dict([(s.name, s) for s in self.settings]) - for key,value in data.iteritems(): - setting = settings[key] - if isinstance(setting, _config.BooleanSetting): - v = value - else: - v = setting.convert_from_text(value) - self[key] = v - - def save(self): - data = {} - settings = dict([(s.name, s) for s in self.settings]) - for key,value in self.iteritems(): - if key in settings: - setting = settings[key] - if isinstance(setting, _config.BooleanSetting): - v = value - else: - v = setting.convert_to_text(value) - data[key] = v - with open(self.filename, 'w') as f: - _yaml.dump(data, stream=f, Dumper=self.dumper, - default_flow_style=False) diff --git a/setup.py b/setup.py index cc03ee0..a315eef 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,6 @@ setup(name=package_name, 'Topic :: Scientific/Engineering', 'Topic :: Software Development :: Libraries :: Python Modules', ], - packages=['h5config'], + packages=[package_name, '{}.storage'.format(package_name)], provides=['{} ({})'.format(package_name, __version__)], ) -- 2.26.2