hooke/util/yaml.py

   1 # Copyright
   2
   3 """Add representers to YAML to support Hooke.
   4
   5 Without introspection, YAML cannot decide how to save some
   6 objects.  By refusing to save these objects, we obviously loose
   7 that information, so make sure the things you drop are either
   8 stored somewhere else or not important.
   9
  10 >>> import yaml
  11 >>> a = numpy.array([1,2,3])
  12 >>> print yaml.dump(a)
  13 null
  14 ...
  15 <BLANKLINE>
  16
  17 The default behavior is to crash.
  18
  19 >>> yaml.Dumper.yaml_representers.pop(numpy.ndarray)  # doctest: +ELLIPSIS
  20 <function none_representer at 0x...>
  21 >>> print yaml.dump(a)
  22 !!python/object/apply:numpy.core.multiarray._reconstruct
  23 args:
  24 - !!python/name:numpy.ndarray ''
  25 - !!python/tuple [0]
  26 - b
  27 state: !!python/tuple
  28 - 1
  29 - !!python/tuple [3]
  30 - null
  31 - false
  32 - "\\x01\\0\\0\\0\\x02\\0\\0\\0\\x03\\0\\0\\0"
  33 <BLANKLINE>
  34
  35 Hmm, at one point that crashed like this::
  36
  37     Traceback (most recent call last):
  38       ...
  39         if data in [None, ()]:
  40     TypeError: data type not understood
  41
  42 Must be because of the other representers I've loaded since.
  43
  44 Restore the representer for future tests.
  45
  46 >>> yaml.add_representer(numpy.ndarray, none_representer)
  47 """
  48
  49 from __future__ import absolute_import
  50 import copy_reg
  51 import sys
  52 import types
  53
  54 import numpy
  55 import yaml
  56 import yaml.constructor
  57 from yaml.constructor import ConstructorError
  58 import yaml.representer
  59
  60 from ..curve import Data, Curve
  61 from ..playlist import FilePlaylist
  62
  63
  64 DATA_INFO_TAG = u'!hooke.curve.DataInfo'
  65
  66
  67 if False: # YAML dump debugging code
  68     """To help isolate data types etc. that give YAML problems.
  69
  70     This is usually caused by external C modules (e.g. numpy) that
  71     define new types (e.g. numpy.ndarray) which YAML cannot inspect.
  72     """
  73     def ignore_aliases(data):
  74         print data, repr(data), type(data), repr(type(data))
  75         sys.stdout.flush()
  76         if data in [None, ()]:
  77             return True
  78         if isinstance(data, (str, unicode, bool, int, float)):
  79             return True
  80     yaml.representer.SafeRepresenter.ignore_aliases = staticmethod(
  81         ignore_aliases)
  82 else:
  83     # Avoid error with
  84     #   numpy.dtype(numpy.int32) in [None, ()]
  85     # See
  86     #   http://projects.scipy.org/numpy/ticket/1001
  87     def ignore_aliases(data):
  88         try:
  89             if data in [None, ()]:
  90                 return True
  91             if isinstance(data, (str, unicode, bool, int, float)):
  92                 return True
  93         except TypeError, e:
  94             pass
  95     yaml.representer.SafeRepresenter.ignore_aliases = staticmethod(
  96         ignore_aliases)
  97
  98
  99 def none_representer(dumper, data):
 100     return dumper.represent_none(None)
 101 yaml.add_representer(numpy.ndarray, none_representer)
 102 yaml.add_representer(numpy.dtype, none_representer)
 103
 104 def bool_representer(dumper, data):
 105     return dumper.represent_bool(data)
 106 yaml.add_representer(numpy.bool_, bool_representer)
 107
 108 def int_representer(dumper, data):
 109     return dumper.represent_int(data)
 110 yaml.add_representer(numpy.int32, int_representer)
 111 yaml.add_representer(numpy.dtype(numpy.int32), int_representer)
 112
 113 def long_representer(dumper, data):
 114     return dumper.represent_long(data)
 115 yaml.add_representer(numpy.int64, int_representer)
 116
 117 def float_representer(dumper, data):
 118     return dumper.represent_float(data)
 119 yaml.add_representer(numpy.float32, float_representer)
 120 yaml.add_representer(numpy.float64, float_representer)
 121
 122 def data_representer(dumper, data):
 123     info = dict(data.info)
 124     for key in info.keys():
 125         if key.startswith('raw '):
 126             del(info[key])
 127     return dumper.represent_mapping(DATA_INFO_TAG, info)
 128 yaml.add_representer(Data, data_representer)
 129
 130 def data_constructor(loader, node):
 131     info = loader.construct_mapping(node)
 132     return Data(shape=(0,0), dtype=numpy.float32, info=info)
 133 yaml.add_constructor(DATA_INFO_TAG, data_constructor)
 134
 135 def object_representer(dumper, data):
 136     cls = type(data)
 137     if cls in copy_reg.dispatch_table:
 138         reduce = copy_reg.dispatch_table[cls](data)
 139     elif hasattr(data, '__reduce_ex__'):
 140         reduce = data.__reduce_ex__(2)
 141     elif hasattr(data, '__reduce__'):
 142         reduce = data.__reduce__()
 143     else:
 144         raise RepresenterError("cannot represent object: %r" % data)
 145     reduce = (list(reduce)+[None]*5)[:5]
 146     function, args, state, listitems, dictitems = reduce
 147     args = list(args)
 148     if state is None:
 149         state = {}
 150     if isinstance(state, dict) and '_default_attrs' in state:
 151         for key in state['_default_attrs']:
 152             if key in state and state[key] == state['_default_attrs'][key]:
 153                 del(state[key])
 154         del(state['_default_attrs'])
 155     if listitems is not None:
 156         listitems = list(listitems)
 157     if dictitems is not None:
 158         dictitems = dict(dictitems)
 159     if function.__name__ == '__newobj__':
 160         function = args[0]
 161         args = args[1:]
 162         tag = u'tag:yaml.org,2002:python/object/new:'
 163         newobj = True
 164     else:
 165         tag = u'tag:yaml.org,2002:python/object/apply:'
 166         newobj = False
 167     function_name = u'%s.%s' % (function.__module__, function.__name__)
 168     if not args and not listitems and not dictitems \
 169             and isinstance(state, dict) and newobj:
 170         return dumper.represent_mapping(
 171                 u'tag:yaml.org,2002:python/object:'+function_name, state)
 172     if not listitems and not dictitems  \
 173             and isinstance(state, dict) and not state:
 174         return dumper.represent_sequence(tag+function_name, args)
 175     value = {}
 176     if args:
 177         value['args'] = args
 178     if state or not isinstance(state, dict):
 179         value['state'] = state
 180     if listitems:
 181         value['listitems'] = listitems
 182     if dictitems:
 183         value['dictitems'] = dictitems
 184     return dumper.represent_mapping(tag+function_name, value)
 185 yaml.add_representer(FilePlaylist, object_representer)
 186 yaml.add_representer(Curve, object_representer)
 187
 188
 189 # Monkey patch PyYAML bug 159.
 190 #   Yaml failed to restore loops in objects when __setstate__ is defined
 191 #   http://pyyaml.org/ticket/159
 192 # With viktor.x.voroshylo@jpmchase.com's patch
 193 def construct_object(self, node, deep=False):
 194     if deep:
 195         old_deep = self.deep_construct
 196         self.deep_construct = True
 197     if node in self.constructed_objects:
 198         return self.constructed_objects[node]
 199     if node in self.recursive_objects:
 200         obj = self.recursive_objects[node]
 201         if obj is None :
 202             raise ConstructorError(None, None,
 203                  "found unconstructable recursive node", node.start_mark)
 204         return obj
 205     self.recursive_objects[node] = None
 206     constructor = None
 207     tag_suffix = None
 208     if node.tag in self.yaml_constructors:
 209         constructor = self.yaml_constructors[node.tag]
 210     else:
 211         for tag_prefix in self.yaml_multi_constructors:
 212             if node.tag.startswith(tag_prefix):
 213                 tag_suffix = node.tag[len(tag_prefix):]
 214                 constructor = self.yaml_multi_constructors[tag_prefix]
 215                 break
 216         else:
 217             if None in self.yaml_multi_constructors:
 218                 tag_suffix = node.tag
 219                 constructor = self.yaml_multi_constructors[None]
 220             elif None in self.yaml_constructors:
 221                 constructor = self.yaml_constructors[None]
 222             elif isinstance(node, ScalarNode):
 223                 constructor = self.__class__.construct_scalar
 224             elif isinstance(node, SequenceNode):
 225                 constructor = self.__class__.construct_sequence
 226             elif isinstance(node, MappingNode):
 227                 constructor = self.__class__.construct_mapping
 228     if tag_suffix is None:
 229         data = constructor(self, node)
 230     else:
 231         data = constructor(self, tag_suffix, node)
 232     if isinstance(data, types.GeneratorType):
 233         generator = data
 234         data = generator.next()
 235         if self.deep_construct:
 236             self.recursive_objects[node] = data
 237             for dummy in generator:
 238                 pass
 239         else:
 240             self.state_generators.append(generator)
 241     self.constructed_objects[node] = data
 242     del self.recursive_objects[node]
 243     if deep:
 244         self.deep_construct = old_deep
 245     return data
 246 yaml.constructor.BaseConstructor.construct_object = construct_object