hooke/util/yaml.py

   1 # Copyright (C) 2010-2012 W. Trevor King <wking@drexel.edu>
   2 #
   3 # This file is part of Hooke.
   4 #
   5 # Hooke is free software: you can redistribute it and/or modify it
   6 # under the terms of the GNU Lesser General Public License as
   7 # published by the Free Software Foundation, either version 3 of the
   8 # License, or (at your option) any later version.
   9 #
  10 # Hooke is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General
  13 # Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Lesser General Public
  16 # License along with Hooke.  If not, see
  17 # <http://www.gnu.org/licenses/>.
  18
  19 """Add representers to YAML to support Hooke.
  20
  21 Without introspection, YAML cannot decide how to save some
  22 objects.  By refusing to save these objects, we obviously loose
  23 that information, so make sure the things you drop are either
  24 stored somewhere else or not important.
  25
  26 >>> import yaml
  27 >>> a = numpy.array([1,2,3])
  28 >>> print yaml.dump(a)
  29 null
  30 ...
  31 <BLANKLINE>
  32
  33 The default behavior is to crash.
  34
  35 >>> yaml.Dumper.yaml_representers.pop(numpy.ndarray)  # doctest: +ELLIPSIS
  36 <function none_representer at 0x...>
  37 >>> print yaml.dump(a)  # doctest: +REPORT_UDIFF
  38 !!python/object/apply:numpy.core.multiarray._reconstruct
  39 args:
  40 - !!python/name:numpy.ndarray ''
  41 - !!python/tuple [0]
  42 - b
  43 state: !!python/tuple
  44 - 1
  45 - !!python/tuple [3]
  46 - !!python/object/apply:numpy.dtype
  47   args: [i4, 0, 1]
  48   state: !!python/tuple [3, <, null, null, null, -1, -1, 0]
  49 - false
  50 - "\\x01\\0\\0\\0\\x02\\0\\0\\0\\x03\\0\\0\\0"
  51 <BLANKLINE>
  52
  53 Hmm, at one point that crashed like this::
  54
  55     Traceback (most recent call last):
  56       ...
  57         if data in [None, ()]:
  58     TypeError: data type not understood
  59
  60 Must be because of the other representers I've loaded since.
  61
  62 Restore the representer for future tests.
  63
  64 >>> yaml.add_representer(numpy.ndarray, none_representer)
  65
  66 We also avoid !!python/unicode tags by sacrificing the string/unicode
  67 distinction.
  68
  69 >>> yaml.dump('ascii', allow_unicode=True)
  70 'ascii\\n...\\n'
  71 >>> yaml.dump(u'ascii', allow_unicode=True)
  72 'ascii\\n...\\n'
  73 >>> a = yaml.dump(u'Fran\\xe7ois', allow_unicode=True)
  74 >>> a
  75 'Fran\\xc3\\xa7ois\\n...\\n'
  76 >>> unicode(a, 'utf-8')
  77 u'Fran\\xe7ois\\n...\\n'
  78 """
  79
  80 from __future__ import absolute_import
  81 import copy_reg
  82 import sys
  83 import types
  84
  85 import numpy
  86 import yaml
  87 import yaml.constructor
  88 from yaml.constructor import ConstructorError
  89 import yaml.representer
  90
  91 from ..curve import Data, Curve
  92 from ..playlist import FilePlaylist
  93
  94
  95 DATA_INFO_TAG = u'!hooke.curve.DataInfo'
  96
  97
  98 if False: # YAML dump debugging code
  99     """To help isolate data types etc. that give YAML problems.
 100
 101     This is usually caused by external C modules (e.g. numpy) that
 102     define new types (e.g. numpy.ndarray) which YAML cannot inspect.
 103     """
 104     def ignore_aliases(data):
 105         print data, repr(data), type(data), repr(type(data))
 106         sys.stdout.flush()
 107         if data in [None, ()]:
 108             return True
 109         if isinstance(data, (str, unicode, bool, int, float)):
 110             return True
 111     yaml.representer.SafeRepresenter.ignore_aliases = staticmethod(
 112         ignore_aliases)
 113 else:
 114     # Avoid error with
 115     #   numpy.dtype(numpy.int32) in [None, ()]
 116     # See
 117     #   http://projects.scipy.org/numpy/ticket/1001
 118     def ignore_aliases(data):
 119         try:
 120             if data in [None, ()]:
 121                 return True
 122             if isinstance(data, (str, unicode, bool, int, float)):
 123                 return True
 124         except TypeError, e:
 125             pass
 126     yaml.representer.SafeRepresenter.ignore_aliases = staticmethod(
 127         ignore_aliases)
 128
 129 def unicode_representer(dumper, data):
 130     return dumper.represent_scalar(u'tag:yaml.org,2002:str', data)
 131 yaml.add_representer(unicode, unicode_representer)
 132
 133 def none_representer(dumper, data):
 134     return dumper.represent_none(None)
 135 yaml.add_representer(numpy.ndarray, none_representer)
 136
 137 def bool_representer(dumper, data):
 138     return dumper.represent_bool(data)
 139 yaml.add_representer(numpy.bool_, bool_representer)
 140
 141 def int_representer(dumper, data):
 142     return dumper.represent_int(data)
 143 yaml.add_representer(numpy.int32, int_representer)
 144 yaml.add_representer(numpy.dtype(numpy.int32), int_representer)
 145
 146 def long_representer(dumper, data):
 147     return dumper.represent_long(data)
 148 yaml.add_representer(numpy.int64, int_representer)
 149
 150 def float_representer(dumper, data):
 151     return dumper.represent_float(data)
 152 yaml.add_representer(numpy.float32, float_representer)
 153 yaml.add_representer(numpy.float64, float_representer)
 154
 155 def data_representer(dumper, data):
 156     info = dict(data.info)
 157     for key in info.keys():
 158         if key.startswith('raw '):
 159             del(info[key])
 160     return dumper.represent_mapping(DATA_INFO_TAG, info)
 161 yaml.add_representer(Data, data_representer)
 162
 163 def data_constructor(loader, node):
 164     info = loader.construct_mapping(node)
 165     return Data(shape=(0,0), dtype=numpy.float32, info=info)
 166 yaml.add_constructor(DATA_INFO_TAG, data_constructor)
 167
 168 def object_representer(dumper, data):
 169     cls = type(data)
 170     if cls in copy_reg.dispatch_table:
 171         reduce = copy_reg.dispatch_table[cls](data)
 172     elif hasattr(data, '__reduce_ex__'):
 173         reduce = data.__reduce_ex__(2)
 174     elif hasattr(data, '__reduce__'):
 175         reduce = data.__reduce__()
 176     else:
 177         raise RepresenterError("cannot represent object: %r" % data)
 178     reduce = (list(reduce)+[None]*5)[:5]
 179     function, args, state, listitems, dictitems = reduce
 180     args = list(args)
 181     if state is None:
 182         state = {}
 183     if isinstance(state, dict) and '_default_attrs' in state:
 184         for key in state['_default_attrs']:
 185             if key in state and state[key] == state['_default_attrs'][key]:
 186                 del(state[key])
 187         del(state['_default_attrs'])
 188     if listitems is not None:
 189         listitems = list(listitems)
 190     if dictitems is not None:
 191         dictitems = dict(dictitems)
 192     if function.__name__ == '__newobj__':
 193         function = args[0]
 194         args = args[1:]
 195         tag = u'tag:yaml.org,2002:python/object/new:'
 196         newobj = True
 197     else:
 198         tag = u'tag:yaml.org,2002:python/object/apply:'
 199         newobj = False
 200     function_name = u'%s.%s' % (function.__module__, function.__name__)
 201     if not args and not listitems and not dictitems \
 202             and isinstance(state, dict) and newobj:
 203         return dumper.represent_mapping(
 204                 u'tag:yaml.org,2002:python/object:'+function_name, state)
 205     if not listitems and not dictitems  \
 206             and isinstance(state, dict) and not state:
 207         return dumper.represent_sequence(tag+function_name, args)
 208     value = {}
 209     if args:
 210         value['args'] = args
 211     if state or not isinstance(state, dict):
 212         value['state'] = state
 213     if listitems:
 214         value['listitems'] = listitems
 215     if dictitems:
 216         value['dictitems'] = dictitems
 217     return dumper.represent_mapping(tag+function_name, value)
 218 yaml.add_representer(FilePlaylist, object_representer)
 219 yaml.add_representer(Curve, object_representer)
 220
 221
 222 # Monkey patch PyYAML bug 159.
 223 #   Yaml failed to restore loops in objects when __setstate__ is defined
 224 #   http://pyyaml.org/ticket/159
 225 # With viktor.x.voroshylo@jpmchase.com's patch
 226 def construct_object(self, node, deep=False):
 227     if deep:
 228         old_deep = self.deep_construct
 229         self.deep_construct = True
 230     if node in self.constructed_objects:
 231         return self.constructed_objects[node]
 232     if node in self.recursive_objects:
 233         obj = self.recursive_objects[node]
 234         if obj is None :
 235             raise ConstructorError(None, None,
 236                  "found unconstructable recursive node", node.start_mark)
 237         return obj
 238     self.recursive_objects[node] = None
 239     constructor = None
 240     tag_suffix = None
 241     if node.tag in self.yaml_constructors:
 242         constructor = self.yaml_constructors[node.tag]
 243     else:
 244         for tag_prefix in self.yaml_multi_constructors:
 245             if node.tag.startswith(tag_prefix):
 246                 tag_suffix = node.tag[len(tag_prefix):]
 247                 constructor = self.yaml_multi_constructors[tag_prefix]
 248                 break
 249         else:
 250             if None in self.yaml_multi_constructors:
 251                 tag_suffix = node.tag
 252                 constructor = self.yaml_multi_constructors[None]
 253             elif None in self.yaml_constructors:
 254                 constructor = self.yaml_constructors[None]
 255             elif isinstance(node, ScalarNode):
 256                 constructor = self.__class__.construct_scalar
 257             elif isinstance(node, SequenceNode):
 258                 constructor = self.__class__.construct_sequence
 259             elif isinstance(node, MappingNode):
 260                 constructor = self.__class__.construct_mapping
 261     if tag_suffix is None:
 262         data = constructor(self, node)
 263     else:
 264         data = constructor(self, tag_suffix, node)
 265     if isinstance(data, types.GeneratorType):
 266         generator = data
 267         data = generator.next()
 268         if self.deep_construct:
 269             self.recursive_objects[node] = data
 270             for dummy in generator:
 271                 pass
 272         else:
 273             self.state_generators.append(generator)
 274     self.constructed_objects[node] = data
 275     del self.recursive_objects[node]
 276     if deep:
 277         self.deep_construct = old_deep
 278     return data
 279 yaml.constructor.BaseConstructor.construct_object = construct_object