h5config/storage/hdf5.py

   1 # Copyright (C) 2011 W. Trevor King <wking@drexel.edu>
   2 #
   3 # This file is part of h5config.
   4 #
   5 # h5config is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the
   7 # Free Software Foundation, either version 3 of the License, or (at your
   8 # option) any later version.
   9 #
  10 # h5config is distributed in the hope that it will be useful, but
  11 # WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with h5config.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """HDF5 backend implementation
  19 """
  20
  21 import types as _types
  22
  23 import h5py as _h5py
  24 import numpy as _numpy
  25
  26 from .. import LOG as _LOG
  27 from .. import config as _config
  28 from . import FileStorage as _FileStorage
  29
  30
  31 def pprint_HDF5(*args, **kwargs):
  32     print pformat_HDF5(*args, **kwargs)
  33
  34 def pformat_HDF5(filename, group='/'):
  35     with _h5py.File(filename, 'r') as f:
  36         cwg = f[group]
  37         ret = '\n'.join(_pformat_hdf5(cwg))
  38     return ret
  39
  40 def _pformat_hdf5(cwg, depth=0):
  41     lines = []
  42     lines.append('  '*depth + cwg.name)
  43     depth += 1
  44     for key,value in cwg.iteritems():
  45         if isinstance(value, _h5py.Group):
  46             lines.extend(_pformat_hdf5(value, depth))
  47         elif isinstance(value, _h5py.Dataset):
  48             lines.append('  '*depth + str(value))
  49             lines.append('  '*(depth+1) + str(value[...]))
  50         else:
  51             lines.append('  '*depth + str(value))
  52     return lines
  53
  54 def h5_create_group(cwg, path, force=False):
  55     "Create the group where the settings are stored (if necessary)."
  56     if path == '/':
  57         return cwg
  58     gpath = ['']
  59     for group in path.strip('/').split('/'):
  60         gpath.append(group)
  61         if group not in cwg.keys():
  62             _LOG.debug('creating group {} in {}'.format(
  63                     '/'.join(gpath), cwg.file))
  64             cwg.create_group(group)
  65         _cwg = cwg[group]
  66         if isinstance(_cwg, _h5py.Dataset):
  67             if force:
  68                 _LOG.info('overwrite {} in {} ({}) with a group'.format(
  69                         '/'.join(gpath), _cwg.file, _cwg))
  70                 del cwg[group]
  71                 _cwg = cwg.create_group(group)
  72             else:
  73                 raise ValueError(_cwg)
  74         cwg = _cwg
  75     return cwg
  76
  77
  78 class HDF5_Storage (_FileStorage):
  79     """Back a `Config` class with an HDF5 file.
  80
  81     The `.save` and `.load` methods have an optional `group` argument
  82     that allows you to save and load settings from an externally
  83     opened HDF5 file.  This can make it easier to stash several
  84     related `Config` classes in a single file.  For example
  85
  86     >>> import os
  87     >>> import tempfile
  88     >>> from ..test import TestConfig
  89     >>> fd,filename = tempfile.mkstemp(
  90     ...     suffix='.'+HDF5_Storage.extension, prefix='pypiezo-')
  91     >>> os.close(fd)
  92
  93     >>> f = _h5py.File(filename, 'a')
  94     >>> c = TestConfig(storage=HDF5_Storage(
  95     ...     filename='untouched_file.h5', group='/untouched/group'))
  96     >>> c['alive'] = True
  97     >>> group = f.create_group('base')
  98     >>> c.save(group=group)
  99     >>> pprint_HDF5(filename)  # doctest: +REPORT_UDIFF, +ELLIPSIS
 100     /
 101       /base
 102         <HDF5 dataset "age": shape (), type "<f8">
 103           1.3
 104         <HDF5 dataset "alive": shape (), type "|b1">
 105           True
 106         <HDF5 dataset "bids": shape (3,), type "<f8">
 107           [ 5.4  3.2  1. ]
 108         <HDF5 dataset "children": shape (), type "|S1">
 109     <BLANKLINE>
 110         <HDF5 dataset "claws": shape (2,), type "<i8">
 111           [1 2]
 112         <HDF5 dataset "daisies": shape (), type "<i...">
 113           13
 114         <HDF5 dataset "name": shape (), type "|S1">
 115     <BLANKLINE>
 116         <HDF5 dataset "species": shape (), type "|S14">
 117           Norwegian Blue
 118         <HDF5 dataset "spouse": shape (), type "|S1">
 119     <BLANKLINE>
 120         <HDF5 dataset "words": shape (2,), type "|S7">
 121           ['cracker' 'wants']
 122     >>> c.clear()
 123     >>> c['alive']
 124     False
 125     >>> c.load(group=group)
 126     >>> c['alive']
 127     True
 128
 129     >>> f.close()
 130     >>> os.remove(filename)
 131     """
 132     extension = 'h5'
 133
 134     def __init__(self, group='/', **kwargs):
 135         super(HDF5_Storage, self).__init__(**kwargs)
 136         assert group.startswith('/'), group
 137         if not group.endswith('/'):
 138             group += '/'
 139         self.group = group
 140         self._file_checked = False
 141
 142     def _check_file(self):
 143         if self._file_checked:
 144             return
 145         self._setup_file()
 146         self._file_checked = True
 147
 148     def _setup_file(self):
 149         self._create_basedir(filename=self._filename)
 150         with _h5py.File(self._filename, 'a') as f:
 151             cwg = f  # current working group
 152             h5_create_group(cwg, self.group)
 153
 154     def _load(self, config, group=None):
 155         f = None
 156         try:
 157             if group is None:
 158                 self._check_file()
 159                 f = _h5py.File(self._filename, 'r')
 160                 group = f[self.group]
 161             for s in config.settings:
 162                 if s.name not in group.keys():
 163                     continue
 164                 if isinstance(s, _config.ConfigListSetting):
 165                     try:
 166                         cwg = h5_create_group(group, s.name)
 167                     except ValueError:
 168                         pass
 169                     else:
 170                         value = []
 171                         for i in sorted(int(x) for x in cwg.keys()):
 172                             instance = s.config_class()
 173                             try:
 174                                 _cwg = h5_create_group(cwg, str(i))
 175                             except ValueError:
 176                                 pass
 177                             else:
 178                                 self._load(config=instance, group=_cwg)
 179                                 value.append(instance)
 180                         config[s.name] = value
 181                 elif isinstance(s, _config.ConfigSetting):
 182                     try:
 183                         cwg = h5_create_group(group, s.name)
 184                     except ValueError:
 185                         pass
 186                     else:
 187                         if not config[s.name]:
 188                             config[s.name] = s.config_class()
 189                         self._load(config=config[s.name], group=cwg)
 190                 else:
 191                     try:
 192                         v = group[s.name][...]
 193                     except Exception, e:
 194                         _LOG.error('Could not access {}/{}: {}'.format(
 195                                 group.name, s.name, e))
 196                         raise
 197                     if isinstance(v, _numpy.ndarray):
 198                         if isinstance(s, _config.BooleanSetting):
 199                             v = bool(v)  # array(True, dtype=bool) -> True
 200                         elif v.dtype.type == _numpy.string_:
 201                             if isinstance(s, _config.ListSetting):
 202                                 try:
 203                                     v = list(v)
 204                                 except TypeError:
 205                                     v = []
 206                             else:
 207                                 v = str(v) # array('abc', dtype='|S3') -> 'abc'
 208                         elif isinstance(s, _config.IntegerSetting):
 209                             v = int(v)  # array(3, dtpe='int32') -> 3
 210                         elif isinstance(s, _config.FloatSetting):
 211                             v = float(v)  # array(1.2, dtype='float64') -> 1.2
 212                         elif isinstance(s, _config.NumericSetting):
 213                             raise NotImplementedError(type(s))
 214                         elif isinstance(s, _config.ListSetting):
 215                             v = list(v)  # convert from numpy array
 216                     if isinstance(v, _types.StringTypes):
 217                         # convert back from None, etc.
 218                         v = s.convert_from_text(v)
 219                     config[s.name] = v
 220         finally:
 221             if f:
 222                 f.close()
 223
 224     def _save(self, config, group=None):
 225         f = None
 226         try:
 227             if group is None:
 228                 self._check_file()
 229                 f = _h5py.File(self._filename, 'a')
 230                 group = f[self.group]
 231             for s in config.settings:
 232                 value = None
 233                 if isinstance(s, (_config.BooleanSetting,
 234                                   _config.NumericSetting,
 235                                   _config.ListSetting)):
 236                     value = config[s.name]
 237                     if value in [None, []]:
 238                         value = s.convert_to_text(value)
 239                 elif isinstance(s, _config.ConfigListSetting):
 240                     configs = config[s.name]
 241                     if configs:
 242                         cwg = h5_create_group(group, s.name, force=True)
 243                         for i,cfg in enumerate(configs):
 244                             _cwg = h5_create_group(cwg, str(i), force=True)
 245                             self._save(config=cfg, group=_cwg)
 246                         continue
 247                 elif isinstance(s, _config.ConfigSetting):
 248                     cfg = config[s.name]
 249                     if cfg:
 250                         cwg = h5_create_group(group, s.name, force=True)
 251                         self._save(config=cfg, group=cwg)
 252                         continue
 253                 if value is None:  # not set yet, or invalid
 254                     value = s.convert_to_text(config[s.name])
 255                 try:
 256                     del group[s.name]
 257                 except KeyError:
 258                     pass
 259                 group[s.name] = value
 260         finally:
 261             if f:
 262                 f.close()