Add ability to nest Configs.
authorW. Trevor King <wking@drexel.edu>
Fri, 29 Jul 2011 04:13:03 +0000 (00:13 -0400)
committerW. Trevor King <wking@drexel.edu>
Fri, 29 Jul 2011 04:15:54 +0000 (00:15 -0400)
I also restructured the storage backends as distinct helper classes
rather than Config subclasses.  This means that you can use several
Storage instances to save a single Config instance (although each
Config instance will have a single default Storage instance stored in
config._storage).  This avoids all the automatic subclassing
shenanigans, and makes loading and saving from different backends much
easier.

h5config/config.py
h5config/hdf5.py [deleted file]
h5config/storage/__init__.py [new file with mode: 0644]
h5config/storage/hdf5.py [new file with mode: 0644]
h5config/storage/yaml.py [new file with mode: 0644]
h5config/test.py
h5config/tools.py
h5config/util.py [deleted file]
h5config/yaml.py [deleted file]
setup.py

index 51dc8d9abfe50d63f9ed72292754786ef48eafb4..e8ff023f456c760841b13639c67afe9da9658411 100644 (file)
@@ -18,6 +18,8 @@
 """The basic h5config classes
 """
 
+import copy as _copy
+
 
 class Setting (object):
     "A named setting with arbitrart text values."
@@ -38,10 +40,10 @@ class Setting (object):
         return ret.strip()
 
     def convert_from_text(self, value):
-        return value
+        return value or None
 
     def convert_to_text(self, value):
-        return value
+        return value or ''
 
 
 class ChoiceSetting (Setting):
@@ -105,12 +107,9 @@ class BooleanSetting (ChoiceSetting):
     >>> s.help()
     'Default: no.  Choices: yes, no'
     """
-    def __init__(self, **kwargs):
-        assert 'choices' not in kwargs
-        if 'default' not in kwargs:
-            kwargs['default'] = False
+    def __init__(self, default=False, **kwargs):
         super(BooleanSetting, self).__init__(
-            choices=[('yes', True), ('no', False)], **kwargs)
+            choices=[('yes', True), ('no', False)], default=default, **kwargs)
 
 
 class NumericSetting (Setting):
@@ -125,12 +124,8 @@ class NumericSetting (Setting):
     >>> s.convert_to_text(13)
     '13'
     """
-    _default_value = 0
-
-    def __init__(self, **kwargs):
-        if 'default' not in kwargs:
-            kwargs['default'] = self._default_value
-        super(NumericSetting, self).__init__(**kwargs)
+    def __init__(self, default=0, **kwargs):
+        super(NumericSetting, self).__init__(default=default, **kwargs)
 
     def convert_to_text(self, value):
         return str(value)
@@ -153,7 +148,8 @@ class IntegerSetting (NumericSetting):
     >>> s.convert_from_text('8')
     8
     """
-    _default_value = 1
+    def __init__(self, default=1, **kwargs):
+        super(IntegerSetting, self).__init__(default=default, **kwargs)
 
     def _convert_from_text(self, value):
         return int(value)
@@ -173,7 +169,8 @@ class FloatSetting (NumericSetting):
     ...     print 'caught a ValueError'
     caught a ValueError
     """
-    _default_value = 1.0
+    def __init__(self, default=1.0, **kwargs):
+        super(FloatSetting, self).__init__(default=default, **kwargs)
 
     def _convert_from_text(self, value):
         return float(value)
@@ -194,10 +191,8 @@ class FloatListSetting (Setting):
     >>> s.convert_from_text('')
     []
     """
-    def __init__(self, **kwargs):
-        if 'default' not in kwargs:
-            kwargs['default'] = []
-        super(FloatListSetting, self).__init__(**kwargs)
+    def __init__(self, default=[], **kwargs):
+        super(FloatListSetting, self).__init__(default=default, **kwargs)
 
     def _convert_from_text(self, value):
         if value is None:
@@ -207,7 +202,7 @@ class FloatListSetting (Setting):
     def convert_from_text(self, value):
         if value is None:
             return None
-        elif value == '':
+        elif value in ['', []]:
             return []
         return [self._convert_from_text(x) for x in value.split(',')]
 
@@ -217,15 +212,56 @@ class FloatListSetting (Setting):
         return ', '.join([str(x) for x in value])
 
 
+class ConfigSetting (Setting):
+    """A setting that holds a pointer to a child `Config` class
+
+    This allows you to nest `Config`\s, which is a useful way to
+    contain complexity.  In order to save such a config, the backend
+    must be able to handle hierarchical storage (possibly via
+    references).
+
+    For example, a configurable AFM may contain a configurable piezo
+    scanner, as well as a configurable stepper motor.
+    """
+    def __init__(self, config_class=None, **kwargs):
+        super(ConfigSetting, self).__init__(**kwargs)
+        self.config_class = config_class
+
+
+class ConfigListSetting (ConfigSetting):
+    """A setting that holds a list of child `Config` classes
+
+    For example, a piezo scanner with several axes.
+    """
+    def __init__(self, default=[], **kwargs):
+        super(ConfigListSetting, self).__init__(**kwargs)
+
+
 class Config (dict):
     "A class with a list `._keys` of `Setting`\s."
     settings = []
 
-    def __init__(self):
+    def __init__(self, storage=None):
+        super(Config, self).__init__()
+        self.clear()
+        self._storage = storage
+
+    def __repr__(self):
+        return '<{} {}>'.format(self.__class__.__name__, id(self))
+
+    def clear(self):
+        super(Config, self).clear()
         for s in self.settings:
-            self[s.name] = s.default
+            # copy to avoid ambiguity with mutable defaults
+            self[s.name] = _copy.deepcopy(s.default)
+
+    def load(self, merge=False, **kwargs):
+        self._storage.load(config=self, merge=merge, **kwargs)
 
-    def dump(self, help=False):
+    def save(self, merge=False, **kwargs):
+        self._storage.save(config=self, merge=merge, **kwargs)
+
+    def dump(self, help=False, prefix=''):
         """Return all settings and their values as a string
 
         >>> class MyConfig (Config):
@@ -260,6 +296,17 @@ class Config (dict):
         lines = []
         for setting in self.settings:
             name = setting.name
+            value = self[name]
+            if isinstance(setting, ConfigSetting):
+                if value is not None:
+                    lines.append(value.dump(help=help, prefix=prefix+'  '))
+                    continue
+            elif isinstance(setting, ConfigListSetting):
+                if value:
+                    for config in value:
+                        lines.append(
+                            config.dump(help=help, prefix=prefix+'  '))
+                    continue
             value_string = setting.convert_to_text(self[name])
             if help:
                 help_string = '\t({})'.format(setting.help())
@@ -267,12 +314,3 @@ class Config (dict):
                 help_string = ''
             lines.append('{}: {}{}'.format(name, value_string, help_string))
         return '\n'.join(lines)
-
-
-class BackedConfig (Config):
-    "A `Config` instance with some kind of storage interface"
-    def load(self):
-        raise NotImplementedError()
-
-    def save(self):
-        raise NotImplementedError()
diff --git a/h5config/hdf5.py b/h5config/hdf5.py
deleted file mode 100644 (file)
index 996fadf..0000000
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
-#
-# This file is part of h5config.
-#
-# h5config is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation, either version 3 of the License, or (at your
-# option) any later version.
-#
-# h5config is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with h5config.  If not, see <http://www.gnu.org/licenses/>.
-
-"""HDF5 backend implementation
-"""
-
-import h5py as _h5py
-
-from . import LOG as _LOG
-from . import config as _config
-
-
-def pprint_HDF5(*args, **kwargs):
-    print pformat_HDF5(*args, **kwargs)
-
-def pformat_HDF5(filename, group='/'):
-    f = _h5py.File(filename, 'r')
-    cwg = f[group]
-    return '\n'.join(_pformat_hdf5(cwg))
-
-def _pformat_hdf5(cwg, depth=0):
-    lines = []
-    lines.append('  '*depth + cwg.name)
-    depth += 1
-    for key,value in cwg.iteritems():
-        if isinstance(value, _h5py.Group):
-            lines.extend(_pformat_hdf5(value, depth))
-        elif isinstance(value, _h5py.Dataset):
-            lines.append('  '*depth + str(value))
-            lines.append('  '*(depth+1) + str(value[...]))
-        else:
-            lines.append('  '*depth + str(value))
-    return lines
-
-def h5_create_group(cwg, path):
-    "Create the group where the settings are stored (if necessary)."
-    if path == '/':
-        return cwg
-    gpath = ['']
-    for group in path.strip('/').split('/'):
-        gpath.append(group)
-        if group not in cwg.keys():
-            _LOG.debug('creating group %s in %s'
-                       % ('/'.join(gpath), cwg.file))
-            cwg.create_group(group)
-        cwg = cwg[group]
-    return cwg
-
-
-class _HDF5Config (_config.BackedConfig):
-    """Mixin to back a `_Config` class with an HDF5 file.
-
-    TODO: Special handling for Choice (enums), FloatList (arrays), etc.?
-
-    The `.save` and `.load` methods have an optional `group` argument
-    that allows you to save and load settings from an externally
-    opened HDF5 file.  This can make it easier to stash several
-    related `_Config` classes in a single file.  For example
-
-    >>> import os
-    >>> import tempfile
-    >>> from .test import HDF5_TestConfig
-    >>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='pypiezo-')
-    >>> os.close(fd)
-
-    >>> f = _h5py.File(filename, 'a')
-    >>> c = HDF5_TestConfig(filename='untouched_file.h5',
-    ...                     group='/untouched/group')
-    >>> c['syslog'] = True
-    >>> group = f.create_group('base')
-    >>> c.save(group)
-    >>> pprint_HDF5(filename)
-    /
-      /base
-        <HDF5 dataset "age": shape (), type "|S3">
-          1.3
-        <HDF5 dataset "alive": shape (), type "|S2">
-          no
-        <HDF5 dataset "bids": shape (), type "|S11">
-          5.4, 3.2, 1
-        <HDF5 dataset "daisies": shape (), type "|S2">
-          13
-        <HDF5 dataset "species": shape (), type "|S14">
-          Norwegian Blue
-    >>> d = HDF5_TestConfig(filename='untouched_file.h5',
-    ...                     group='/untouched/group')
-    >>> d.load(group)
-    >>> d['alive']
-    False
-
-    >>> f.close()
-    >>> os.remove(filename)
-    """
-    def __init__(self, filename=None, group='/', **kwargs):
-        super(_HDF5Config, self).__init__(**kwargs)
-        self.filename = filename
-        assert group.startswith('/'), group
-        if not group.endswith('/'):
-            group += '/'
-        self.group = group
-        self._file_checked = False
-
-    def _check_file(self):
-        if self._file_checked:
-            return
-        self._setup_file()
-        self._file_checked = True
-
-    def _setup_file(self):
-        f = _h5py.File(self.filename, 'a')
-        cwg = f  # current working group
-        h5_create_group(cwg, self.group)
-        f.close()
-
-    def dump(self, help=False, from_file=False):
-        """Return the relevant group in `self.filename` as a string
-
-        Extends the base :meth:`dump` by adding the `from_file`
-        option.  If `from_file` is true, dump all entries that
-        currently exist in the relevant group, rather than listing all
-        settings defined in the instance dictionary.
-        """
-        if from_file:
-            self._check_file()
-            f = _h5py.File(self.filename, 'r')
-            cwg = f[self.group]
-            lines = []
-            settings = dict([(s.name, s) for s in self.settings])
-            for key,value in cwg.iteritems():
-                if help and key in settings:
-                    help_string = '\t(%s)' % settings[key].help()
-                else:
-                    help_string = ''
-                lines.append('%s: %s%s' % (key, value[...], help_string))
-            return '\n'.join(lines)
-        return super(_HDF5Config, self).dump(help=help)
-
-    def load(self, group=None):
-        if group is None:
-            self._check_file()
-            f = _h5py.File(self.filename, 'r')
-            group = f[self.group]
-        else:
-            f = None
-        for s in self.settings:
-            if s.name not in group.keys():
-                continue
-            self[s.name] = s.convert_from_text(group[s.name][...])
-        if f:
-            f.close()
-
-    def save(self, group=None):
-        if group is None:
-            self._check_file()
-            f = _h5py.File(self.filename, 'a')
-            group = f[self.group]
-        else:
-            f = None
-        for s in self.settings:
-            value = s.convert_to_text(self[s.name])
-            try:
-                del group[s.name]
-            except KeyError:
-                pass
-            group[s.name] = value
-        if f:
-            f.close()
diff --git a/h5config/storage/__init__.py b/h5config/storage/__init__.py
new file mode 100644 (file)
index 0000000..b9d72e5
--- /dev/null
@@ -0,0 +1,45 @@
+# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
+#
+# This file is part of h5config.
+#
+# h5config is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# h5config is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with h5config.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class Storage (object):
+    "A storage bakend for loading and saving `Config` instances"
+    def load(self, config, merge=False, **kwargs):
+        if merge:
+            self.clear()
+        self._load(config=config, **kwargs)
+        config._storage = self
+
+    def _load(self, config, **kwargs):
+        raise NotImplementedError()
+
+    def save(self, config, merge=False, **kwargs):
+        if merge:
+            self.clear()
+        self._save(config=config, **kwargs)
+        config._storage = self
+
+    def _save(self, config, **kwargs):
+        raise NotImplementedError()
+
+
+class FileStorage (Storage):
+    "`Config` storage backend by a single file"
+    extension = None
+
+    def __init__(self, filename=None):
+        self._filename = filename
diff --git a/h5config/storage/hdf5.py b/h5config/storage/hdf5.py
new file mode 100644 (file)
index 0000000..b488c37
--- /dev/null
@@ -0,0 +1,220 @@
+# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
+#
+# This file is part of h5config.
+#
+# h5config is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# h5config is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with h5config.  If not, see <http://www.gnu.org/licenses/>.
+
+"""HDF5 backend implementation
+"""
+
+import h5py as _h5py
+
+from .. import LOG as _LOG
+from .. import config as _config
+from . import FileStorage as _FileStorage
+
+
+def pprint_HDF5(*args, **kwargs):
+    print pformat_HDF5(*args, **kwargs)
+
+def pformat_HDF5(filename, group='/'):
+    with _h5py.File(filename, 'r') as f:
+        cwg = f[group]
+        ret = '\n'.join(_pformat_hdf5(cwg))
+    return ret
+
+def _pformat_hdf5(cwg, depth=0):
+    lines = []
+    lines.append('  '*depth + cwg.name)
+    depth += 1
+    for key,value in cwg.iteritems():
+        if isinstance(value, _h5py.Group):
+            lines.extend(_pformat_hdf5(value, depth))
+        elif isinstance(value, _h5py.Dataset):
+            lines.append('  '*depth + str(value))
+            lines.append('  '*(depth+1) + str(value[...]))
+        else:
+            lines.append('  '*depth + str(value))
+    return lines
+
+def h5_create_group(cwg, path, force=False):
+    "Create the group where the settings are stored (if necessary)."
+    if path == '/':
+        return cwg
+    gpath = ['']
+    for group in path.strip('/').split('/'):
+        gpath.append(group)
+        if group not in cwg.keys():
+            _LOG.debug('creating group {} in {}'.format(
+                    '/'.join(gpath), cwg.file))
+            cwg.create_group(group)
+        _cwg = cwg[group]
+        if isinstance(_cwg, _h5py.Dataset):
+            if force:
+                _LOG.info('overwrite {} in {} ({}) with a group'.format(
+                        '/'.join(gpath), _cwg.file, _cwg))
+                del cwg[group]
+                _cwg = cwg.create_group(group)
+            else:
+                raise ValueError(_cwg)
+        cwg = _cwg
+    return cwg
+
+
+class HDF5_Storage (_FileStorage):
+    """Back a `Config` class with an HDF5 file.
+
+    TODO: Special handling for Choice (enums), FloatList (arrays), etc.?
+
+    The `.save` and `.load` methods have an optional `group` argument
+    that allows you to save and load settings from an externally
+    opened HDF5 file.  This can make it easier to stash several
+    related `Config` classes in a single file.  For example
+
+    >>> import os
+    >>> import tempfile
+    >>> from ..test import TestConfig
+    >>> fd,filename = tempfile.mkstemp(
+    ...     suffix='.'+HDF5_Storage.extension, prefix='pypiezo-')
+    >>> os.close(fd)
+
+    >>> f = _h5py.File(filename, 'a')
+    >>> c = TestConfig(storage=HDF5_Storage(
+    ...     filename='untouched_file.h5', group='/untouched/group'))
+    >>> c['alive'] = True
+    >>> group = f.create_group('base')
+    >>> c.save(group=group)
+    >>> pprint_HDF5(filename)  # doctest: +REPORT_UDIFF
+    /
+      /base
+        <HDF5 dataset "age": shape (), type "|S3">
+          1.3
+        <HDF5 dataset "alive": shape (), type "|S3">
+          yes
+        <HDF5 dataset "bids": shape (), type "|S11">
+          5.4, 3.2, 1
+        <HDF5 dataset "children": shape (), type "|S1">
+    <BLANKLINE>
+        <HDF5 dataset "daisies": shape (), type "|S2">
+          13
+        <HDF5 dataset "name": shape (), type "|S1">
+    <BLANKLINE>
+        <HDF5 dataset "species": shape (), type "|S14">
+          Norwegian Blue
+        <HDF5 dataset "spouse": shape (), type "|S1">
+    <BLANKLINE>
+    >>> c.clear()
+    >>> c['alive']
+    False
+    >>> c.load(group=group)
+    >>> c['alive']
+    True
+
+    >>> f.close()
+    >>> os.remove(filename)
+    """
+    extension = 'h5'
+
+    def __init__(self, group='/', **kwargs):
+        super(HDF5_Storage, self).__init__(**kwargs)
+        assert group.startswith('/'), group
+        if not group.endswith('/'):
+            group += '/'
+        self.group = group
+        self._file_checked = False
+
+    def _check_file(self):
+        if self._file_checked:
+            return
+        self._setup_file()
+        self._file_checked = True
+
+    def _setup_file(self):
+        with _h5py.File(self._filename, 'a') as f:
+            cwg = f  # current working group
+            h5_create_group(cwg, self.group)
+
+    def _load(self, config, group=None):
+        f = None
+        try:
+            if group is None:
+                self._check_file()
+                f = _h5py.File(self._filename, 'r')
+                group = f[self.group]
+            for s in config.settings:
+                if s.name not in group.keys():
+                    continue
+                elif isinstance(s, _config.ConfigListSetting):
+                    try:
+                        cwg = h5_create_group(group, s.name)
+                    except ValueError:
+                        pass
+                    else:
+                        value = []
+                        for i in sorted(int(x) for x in cwg.keys()):
+                            instance = s.config_class()
+                            try:
+                                _cwg = h5_create_group(cwg, str(i))
+                            except ValueError:
+                                pass
+                            else:
+                                self._load(config=instance, group=_cwg)
+                                value.append(instance)
+                        config[s.name] = value
+                elif isinstance(s, _config.ConfigSetting):
+                    try:
+                        cwg = h5_create_group(group, s.name)
+                    except ValueError:
+                        pass
+                    else:
+                        if not config[s.name]:
+                            config[s.name] = s.config_class()
+                        self._load(config=config[s.name], group=cwg)
+                else:
+                    config[s.name] = s.convert_from_text(group[s.name][...])
+        finally:
+            if f:
+                f.close()
+
+    def _save(self, config, group=None):
+        f = None
+        try:
+            if group is None:
+                self._check_file()
+                f = _h5py.File(self._filename, 'a')
+                group = f[self.group]
+            for s in config.settings:
+                if isinstance(s, _config.ConfigListSetting):
+                    configs = config[s.name]
+                    if configs:
+                        cwg = h5_create_group(group, s.name, force=True)
+                        for i,cfg in enumerate(configs):
+                            _cwg = h5_create_group(cwg, str(i), force=True)
+                            self._save(config=cfg, group=_cwg)
+                        continue
+                elif isinstance(s, _config.ConfigSetting):
+                    cfg = config[s.name]
+                    if cfg:
+                        cwg = h5_create_group(group, s.name, force=True)
+                        self._save(config=cfg, group=cwg)
+                        continue
+                value = s.convert_to_text(config[s.name])
+                try:
+                    del group[s.name]
+                except KeyError:
+                    pass
+                group[s.name] = value
+        finally:
+            if f:
+                f.close()
diff --git a/h5config/storage/yaml.py b/h5config/storage/yaml.py
new file mode 100644 (file)
index 0000000..ec06aba
--- /dev/null
@@ -0,0 +1,153 @@
+# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
+#
+# This file is part of h5config.
+#
+# h5config is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# h5config is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with h5config.  If not, see <http://www.gnu.org/licenses/>.
+
+"""HDF5 backend implementation
+"""
+
+from __future__ import absolute_import
+
+import os.path as _os_path
+
+import yaml as _yaml  # global PyYAML module
+
+from .. import LOG as _LOG
+from .. import config as _config
+from . import FileStorage as _FileStorage
+
+
+class _YAMLDumper (_yaml.SafeDumper):
+    def represent_bool(self, data):
+        "Use yes/no instead of the default true/false"
+        if data:
+            value = u'yes'
+        else:
+            value = u'no'
+        return self.represent_scalar(u'tag:yaml.org,2002:bool', value)
+
+
+_YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool)
+
+
+class YAML_Storage (_FileStorage):
+    """Back a `Config` class with a YAML file.
+
+    TODO: Special handling for Choice (enums), etc.?
+
+    >>> import os
+    >>> from ..test import TestConfig
+    >>> import os.path
+    >>> import tempfile
+    >>> fd,filename = tempfile.mkstemp(
+    ...     suffix='.'+YAML_Storage.extension, prefix='pypiezo-')
+    >>> os.close(fd)
+
+    >>> c = TestConfig(storage=YAML_Storage(filename=filename))
+    >>> c.load()
+
+    Saving writes all the config values to disk.
+
+    >>> c['alive'] = True
+    >>> c.save()
+    >>> print open(filename, 'r').read()  # doctest: +REPORT_UDIFF
+    age: 1.3
+    alive: yes
+    bids:
+    - 5.4
+    - 3.2
+    - 1
+    children: ''
+    daisies: 13
+    name: ''
+    species: Norwegian Blue
+    spouse: ''
+    <BLANKLINE>
+
+    Loading reads the config files from disk.
+
+    >>> c.clear()
+    >>> c['alive']
+    False
+    >>> c.load()
+    >>> c['alive']
+    True
+
+    Cleanup our temporary config file.
+
+    >>> os.remove(filename)
+    """
+    extension = 'yaml'
+    dumper = _YAMLDumper
+
+    def _load(self, config):
+        if not _os_path.exists(self._filename):
+            open(self._filename, 'a').close()
+        with open(self._filename, 'r') as f:
+            data = _yaml.safe_load(f)
+        if data == None:
+            return  # empty file
+        return self._from_dict(config, data)
+
+    @staticmethod
+    def _from_dict(config, data):
+        settings = dict([(s.name, s) for s in config.settings])
+        for key,value in data.iteritems():
+            setting = settings[key]
+            if isinstance(setting, (_config.BooleanSetting,
+                                    _config.NumericSetting,
+                                    _config.FloatListSetting)):
+                v = value
+            elif isinstance(setting, _config.ConfigListSetting) and value:
+                values = []
+                for v in value:
+                    values.append(YAML_Storage._from_dict(
+                            setting.config_class(), v))
+                v = values
+            elif isinstance(setting, _config.ConfigSetting) and value:
+                v = YAML_Storage._from_dict(setting.config_class(), value)
+            else:
+                v = setting.convert_from_text(value)
+            config[key] = v
+        return config
+
+    def _save(self, config):
+        data = self._to_dict(config)
+        with open(self._filename, 'w') as f:
+            _yaml.dump(data, stream=f, Dumper=self.dumper,
+                       default_flow_style=False)
+
+    @staticmethod
+    def _to_dict(config):
+        data = {}
+        settings = dict([(s.name, s) for s in config.settings])
+        for key,value in config.iteritems():
+            if key in settings:
+                setting = settings[key]
+                if isinstance(setting, (_config.BooleanSetting,
+                                        _config.NumericSetting,
+                                        _config.FloatListSetting)):
+                    v = value
+                elif isinstance(setting, _config.ConfigListSetting) and value:
+                    values = []
+                    for v in value:
+                        values.append(YAML_Storage._to_dict(v))
+                    v = values
+                elif isinstance(setting, _config.ConfigSetting) and value:
+                    v = YAML_Storage._to_dict(value)
+                else:
+                    v = setting.convert_to_text(value)
+                data[key] = v
+        return data
index 27eb67c23fd400604ef587972fc0388e66177e19..5734a27274f318ad30bab330962d1c82117728cc 100644 (file)
 
 """Define a test config object using all the setting types
 
-Ensure that `build_backend_classes` worked properly for this module:
-
->>> print '\\n'.join([obj for obj in sorted(locals().keys())
-...                   if obj.endswith('Config')
-...                   and not obj.startswith('_')])
-HDF5_TestConfig
-TestConfig
-YAML_TestConfig
-
-
-The first time you use them, the file they create will probably be
-empty or not exist.
+The first time you a storage backend, the file it creates will
+probably be empty or not exist.
 
 >>> import os
 >>> import tempfile
->>> from hdf5 import pprint_HDF5
+>>> from storage.hdf5 import pprint_HDF5, HDF5_Storage
 
 >>> fd,filename = tempfile.mkstemp(suffix='.h5', prefix='h5config-')
 >>> os.close(fd)
 
->>> c = HDF5_TestConfig(filename=filename, group='/base')
+>>> c = TestConfig(storage=HDF5_Storage(filename=filename, group='/base'))
 >>> c.load()
 
 Loading will create a stub group group if it hadn't existed before.
@@ -45,8 +35,6 @@ Loading will create a stub group group if it hadn't existed before.
 >>> pprint_HDF5(filename)
 /
   /base
->>> print c.dump(from_file=True)
-<BLANKLINE>
 
 Saving fills in all the config values.
 
@@ -61,27 +49,30 @@ Saving fills in all the config values.
       no
     <HDF5 dataset "bids": shape (), type "|S11">
       5.4, 3.2, 1
+    <HDF5 dataset "children": shape (), type "|S1">
+<BLANKLINE>
     <HDF5 dataset "daisies": shape (), type "|S2">
       13
+    <HDF5 dataset "name": shape (), type "|S1">
+<BLANKLINE>
     <HDF5 dataset "species": shape (), type "|S14">
       Norwegian Blue
->>> print c.dump(from_file=True)
-age: 1.3
-alive: no
-bids: 5.4, 3.2, 1
-daisies: 13
-species: Norwegian Blue
+    <HDF5 dataset "spouse": shape (), type "|S1">
+<BLANKLINE>
 
 If you want more details, you can dump with help strings.
 
->>> print c.dump(help=True, from_file=True)  # doctest: +NORMALIZE_WHITESPACE
-age: 1.3                (Parrot age in years  Default: 1.3.)
-alive: no               (The parrot is alive.  Default: no.  Choices: yes, no)
-bids: 5.4, 3.2, 1       (Prices offered for parrot.  Default: 5.4, 3.2, 1.)
-daisies: 13             (Number of daisies pushed up by the parrot.
-                         Default: 13.)
-species: Norwegian Blue (Type of parrot.  Default: Norwegian Blue.
-                         Choices: Norwegian Blue, Macaw)
+>>> print c.dump(help=True)  # doctest: +REPORT_UDIFF, +NORMALIZE_WHITESPACE
+name:                    (The parrot's name.  Default: .)
+species: Norwegian Blue  (Type of parrot.  Default: Norwegian Blue.
+                          Choices: Norwegian Blue, Macaw)
+alive: no                (The parrot is alive.  Default: no.  Choices: yes, no)
+daisies: 13              (Number of daisies pushed up by the parrot.
+                          Default: 13.)
+age: 1.3                 (Parrot age in years  Default: 1.3.)
+bids: 5.4, 3.2, 1        (Prices offered for parrot.  Default: 5.4, 3.2, 1.)
+spouse:                  (This parrot's significant other.  Default: .)
+children:                (This parrot's children.  Default: .)
 
 As you can see from the `age` setting, settings also support `None`,
 even if they have numeric types.
@@ -97,23 +88,17 @@ import tempfile as _tempfile
 
 from . import LOG as _LOG
 from . import config as _config
-from . import util as _util
-from .hdf5 import _HDF5Config
-from .yaml import _YAMLConfig
-
-
-_ALTERNATIVES = {  # alternative settings for testing
-    'species': 1,
-    'alive': True,
-    'daisies': None,
-    'age': None,
-    'bids': [],
-    }
+from .storage import FileStorage as _FileStorage
+from .storage.hdf5 import HDF5_Storage
+from .storage.yaml import YAML_Storage
 
 
 class TestConfig (_config.Config):
     "Test all the setting types for the h5config module"
     settings = [
+        _config.Setting(
+            name='name',
+            help="The parrot's name."),
         _config.ChoiceSetting(
             name='species',
             help='Type of parrot.',
@@ -135,55 +120,94 @@ class TestConfig (_config.Config):
             name='bids',
             help='Prices offered for parrot.',
             default=[5.4, 3.2, 1]),
+        _config.ConfigSetting(
+            name='spouse',
+            help="This parrot's significant other."),
+        _config.ConfigListSetting(
+            name='children',
+            help="This parrot's children."),
         ]
 
+# must define self-references after completing the TestConfig class
+for s in TestConfig.settings:
+    if s.name in ['spouse', 'children']:
+        s.config_class = TestConfig
+
 
-_util.build_backend_classes(_sys.modules[__name__])
+def _alternative_test_config(name):
+    ret = TestConfig()
+    ret['name'] = name
+    return ret
 
+_ALTERNATIVES = {  # alternative settings for testing
+    'name': 'Captain Flint',
+    'species': 1,
+    'alive': True,
+    'daisies': None,
+    'age': None,
+    'bids': [],
+    'spouse': _alternative_test_config(name='Lory'),
+    'children': [_alternative_test_config(name=n)
+                 for n in ['Washington Post', 'Eli Yale']],
+    }
+# TODO: share children with spouse to test references
 
-def test(test_class=None):
-    if test_class is None:
-        classes = []
-        for name,obj in sorted(globals().items()):
-            try:
-                if issubclass(obj, TestConfig):
-                    classes.append(obj)
-            except TypeError:
-                pass
-        for class_ in classes:
-            test(test_class=class_)
+def test(storage=None):
+    if storage is None:
+        storage = [HDF5_Storage, YAML_Storage]
+        for s in storage:
+            test(storage=s)
         return
-    _LOG.debug('testing {}'.format(test_class))
-    _basic_tests(test_class)
-    if issubclass(test_class, (_HDF5Config, _YAMLConfig)):
-        _uses_file_tests(test_class)
+    _LOG.debug('testing {}'.format(storage))
+    _basic_tests(storage)
+    if issubclass(storage, _FileStorage):
+        _file_storage_tests(storage)
 
-def _basic_tests(test_class):
+def _basic_tests(storage):
     pass
 
-def _uses_file_tests(test_class):
-    fd,filename = _tempfile.mkstemp(suffix='.tmp', prefix='h5config-')
+def _file_storage_tests(storage):
+    fd,filename = _tempfile.mkstemp(
+        suffix='.'+storage.extension, prefix='h5config-')
     _os.close(fd)
-    kwargs = {'filename': filename}
     try:
-        x = test_class(**kwargs)
-        y = test_class(**kwargs)
-        x.dump()
-        x.save()
-        y.load()
-        nd = list(_non_defaults(y))
+        c = TestConfig(storage=storage(filename=filename))
+        c.dump()
+        c.save()
+        c.load()
+        nd = list(_non_defaults(c))
         assert not nd, nd
         for key,value in _ALTERNATIVES.items():
-            x[key] = value
-        x.save()
-        y.load()
-        nd = dict(_non_defaults(y))
-        assert nd == _ALTERNATIVES, nd
+            c[key] = value
+        c.save()
+        na = dict(_non_alternatives(c))
+        assert not na, na
+        c.clear()
+        nd = list(_non_defaults(c))
+        assert not nd, nd
+        c.load()
+        na = dict(_non_alternatives(c))
+        assert not na, na
     finally:
         _os.remove(filename)
 
-def _non_defaults(test_instance):
+def _non_defaults(config):
     for setting in TestConfig.settings:
-        value = test_instance[setting.name]
+        value = config[setting.name]
         if value != setting.default:
             yield (setting.name, value)
+
+def _non_alternatives(config, alternatives=None):
+    if alternatives is None:
+        alternatives = _ALTERNATIVES
+    for setting in TestConfig.settings:
+        value = config[setting.name]
+        alt = alternatives[setting.name]
+        if value != alt:
+            _LOG.error('{} value missmatch: {} vs {}'.format(
+                    setting.name, value, alt))
+            yield (setting.name, value)
+        elif type(value) != type(alt):
+            _LOG.error('{} type missmatch: {} vs {}'.format(
+                    setting.name, type(value), type(alt)))
+            yield (setting.name, value)
index f7cd2e414ea8db310c467bc983d477712d4e25fd..48c683263282df7141679786f1d242f4743e06b1 100644 (file)
@@ -31,7 +31,8 @@ import sys as _sys
 
 from . import config as _config
 from . import log as _log
-from . import util as _util
+from .storage.hdf5 import HDF5_Storage as _HDF5_Storage
+from .storage.yaml import YAML_Storage as _YAML_Storage
 
 
 class PackageConfig (_config.Config):
@@ -41,15 +42,8 @@ class PackageConfig (_config.Config):
     `LOG` instance.  If you create this instance on your own (for
     example, to work around bootstrapping issues), just pass your
     instance in as `logger` when you initialize this class.
-
-    Because this class occasionally replaces itself, you should always
-    work with the version in the package namespace and not with a
-    local reference (which may be stale).
-
-    Things get a bit interesting because we want to configure the
-    package from an internal class.  This leads to TODO
     """
-    _backed_subclasses = ()
+    possible_storage = [_HDF5_Storage, _YAML_Storage]
     settings = [
         _config.ChoiceSetting(
             name='log-level',
@@ -80,22 +74,6 @@ class PackageConfig (_config.Config):
         if 'LOG' not in dir(namespace):
             namespace.LOG = logger
 
-    def _name(self):
-        "Find this instance's name in the bound namespace"
-        for attr_name in dir(self._namespace):
-            attr = getattr(self._namespace, attr_name, None)
-            if id(attr) == id(self):
-                return attr_name
-        raise IndexError('{} not found in the {} namespace'.format(
-                self, self._namespace))
-
-    def _replace_self(self, new_config):
-        self._logger.debug('replacing {} package config {} with {}'.format(
-                self._package_name, self, new_config))
-        new_config.setup()
-        name = self._name()
-        setattr(self._namespace, name, new_config)
-
     def setup(self):
         self._logger.setLevel(self['log-level'])
         if self['syslog']:
@@ -111,11 +89,8 @@ class PackageConfig (_config.Config):
 
     def clear(self):
         "Replace self with a non-backed version with default settings."
-        replacement = self._clear_class(
-                package_name=self._package_name,
-                namespace=self._namespace,
-                logger = self._logger)
-        self._replace_self(replacement)
+        super(PackageConfig, self).clear()
+        self._storage = None
 
     def _base_paths(self):
         user_basepath = _os_path.join(
@@ -130,41 +105,23 @@ class PackageConfig (_config.Config):
         self._logger.info('looking for package config file')
         basepaths = self._base_paths()
         for basepath in basepaths:
-            for extension,config in self._backed_subclasses:
-                filename = basepath + extension
+            for storage in self.possible_storage:
+                filename = '{}.{}'.format(basepath, storage.extension)
                 if _os_path.exists(filename):
                     self._logger.info(
                         'base_config file found at {}'.format(filename))
-                    replacement = config(
-                        filename=filename,
-                        package_name=self._package_name,
-                        namespace=self._namespace,
-                        logger = self._logger)
-                    replacement.load()
-                    self._replace_self(replacement)
+                    self._storage = storage(filename=filename)
+                    self.load()
+                    self.setup()
                     return
                 else:
                     self._logger.debug(
                         'no base_config file at {}'.format(filename))
         # create (but don't save) the preferred file
         basepath = basepaths[0]
-        extension,config = self._backed_subclasses[0]
-        filename = basepath + extension
+        storage = self.possible_storage[0]
+        filename = '{}.{}'.format(basepath, storage.extension)
         self._logger.info('new base_config file at {}'.format(filename))
-        replacement = config(
-            filename=filename,
-            package_name=self._package_name,
-            namespace=self._namespace,
-            logger = self._logger)
-        self._replace_self(replacement)
-
-
-PackageConfig._clear_class = PackageConfig
-
-
-_util.build_backend_classes(_sys.modules[__name__])
-
-PackageConfig._backed_subclasses = [
-    ('.h5', HDF5_PackageConfig),
-    ('.yaml', YAML_PackageConfig)
-    ]
+        self._storage = storage(filename=filename)
+        self.load()
+        self.setup()
diff --git a/h5config/util.py b/h5config/util.py
deleted file mode 100644 (file)
index 9f1742a..0000000
+++ /dev/null
@@ -1,56 +0,0 @@
-# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
-#
-# This file is part of h5config.
-#
-# h5config is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation, either version 3 of the License, or (at your
-# option) any later version.
-#
-# h5config is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with h5config.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Utilities for building your own config classes
-"""
-
-import os.path as _os_path
-
-from . import LOG as _LOG
-from . import config as _config
-from .hdf5 import _HDF5Config
-from .yaml import _YAMLConfig
-
-
-def build_backend_classes(namespace, objects=None):
-    """Define HDF5- and YAML-backed subclasses of the basic `Config` types
-
-    At the end of your class-defining module, you should build the
-    particular backends.  You'll need to pass in a namespace to attach
-    the new classes to.  I generally use `sys.modules[__name__]` to
-    attach the classes to the current module.  By default
-    `build_backend_classes` will search for base `Config` classes in
-    the namespace you pass it.  If that doesn't work for you, you can
-    explicitly pass it an iterable of `name, object)` tuples.
-    """
-    if objects is None:
-        objects = [(name,getattr(namespace, name, None))
-                   for name in dir(namespace)]
-    for name,obj in objects:
-        if (obj != _config.Config and
-            type(obj) == type and
-            issubclass(obj, _config.Config) and
-            not issubclass(obj, _config.BackedConfig)):
-            for prefix,base in [('HDF5', _HDF5Config), ('YAML', _YAMLConfig)]:
-                _LOG.debug(
-                    'creating {} backend for {} in the {} namespace'.format(
-                        prefix, name, namespace))
-                name_ = '%s_%s' % (prefix, name)
-                bases = (base, obj)
-                dict_ = {}
-                class_ = type(name_, bases, dict_)
-                setattr(namespace, name_, class_)
diff --git a/h5config/yaml.py b/h5config/yaml.py
deleted file mode 100644 (file)
index 642b7a9..0000000
+++ /dev/null
@@ -1,122 +0,0 @@
-# Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
-#
-# This file is part of h5config.
-#
-# h5config is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation, either version 3 of the License, or (at your
-# option) any later version.
-#
-# h5config is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with h5config.  If not, see <http://www.gnu.org/licenses/>.
-
-"""HDF5 backend implementation
-"""
-
-from __future__ import absolute_import
-
-import os.path as _os_path
-
-import yaml as _yaml  # global PyYAML module
-
-from . import LOG as _LOG
-from . import config as _config
-
-
-class _YAMLDumper (_yaml.SafeDumper):
-    def represent_bool(self, data):
-        "Use yes/no instead of the default true/false"
-        if data:
-            value = u'yes'
-        else:
-            value = u'no'
-        return self.represent_scalar(u'tag:yaml.org,2002:bool', value)
-
-
-_YAMLDumper.add_representer(bool, _YAMLDumper.represent_bool)
-
-
-class _YAMLConfig (_config.BackedConfig):
-    """Mixin to back a `Config` class with a YAML file.
-
-    TODO: Special handling for Choice (enums), FloatList (arrays), etc.?
-
-
-    >>> import os
-    >>> from .test import YAML_TestConfig
-    >>> import os.path
-    >>> import tempfile
-    >>> fd,filename = tempfile.mkstemp(suffix='.yaml', prefix='pypiezo-')
-    >>> os.close(fd)
-
-    >>> c = YAML_TestConfig(filename=filename)
-    >>> c.load()
-
-    Saving writes all the config values to disk.
-
-    >>> c['syslog'] = True
-    >>> c.save()
-    >>> print open(c.filename, 'r').read()
-    age: '1.3'
-    alive: no
-    bids: 5.4, 3.2, 1
-    daisies: '13'
-    species: Norwegian Blue
-    <BLANKLINE>
-
-    Loading reads the config files from disk.
-
-    >>> c = YAML_TestConfig(filename=filename)
-    >>> c.load()
-    >>> print c.dump()
-    species: Norwegian Blue
-    alive: no
-    daisies: 13
-    age: 1.3
-    bids: 5.4, 3.2, 1.0
-
-    Cleanup our temporary config file.
-
-    >>> os.remove(filename)
-    """
-    dumper = _YAMLDumper
-
-    def __init__(self, filename=None, **kwargs):
-        super(_YAMLConfig, self).__init__(**kwargs)
-        self.filename = filename
-
-    def load(self):
-        if not _os_path.exists(self.filename):
-            open(self.filename, 'a').close()
-        with open(self.filename, 'r') as f:
-            data = _yaml.safe_load(f)
-        if data == None:
-            return  # empty file
-        settings = dict([(s.name, s) for s in self.settings])
-        for key,value in data.iteritems():
-            setting = settings[key]
-            if isinstance(setting, _config.BooleanSetting):
-                v = value
-            else:
-                v = setting.convert_from_text(value)
-            self[key] = v
-
-    def save(self):
-        data = {}
-        settings = dict([(s.name, s) for s in self.settings])
-        for key,value in self.iteritems():
-            if key in settings:
-                setting = settings[key]
-                if isinstance(setting, _config.BooleanSetting):
-                    v = value
-                else:
-                    v = setting.convert_to_text(value)
-                data[key] = v
-        with open(self.filename, 'w') as f:
-            _yaml.dump(data, stream=f, Dumper=self.dumper,
-                       default_flow_style=False)
index cc03ee0a90356a5f41f559291714b360f051674a..a315eefe62456bd5bf3d4eea9f522e978646e3d8 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -48,6 +48,6 @@ setup(name=package_name,
         'Topic :: Scientific/Engineering',
         'Topic :: Software Development :: Libraries :: Python Modules',
         ],
-      packages=['h5config'],
+      packages=[package_name, '{}.storage'.format(package_name)],
       provides=['{} ({})'.format(package_name, __version__)],
       )