hooke/util/yaml.py

   1 # Copyright (C) 2010-2012 W. Trevor King <wking@tremily.us>
   2 #
   3 # This file is part of Hooke.
   4 #
   5 # Hooke is free software: you can redistribute it and/or modify it under the
   6 # terms of the GNU Lesser General Public License as published by the Free
   7 # Software Foundation, either version 3 of the License, or (at your option) any
   8 # later version.
   9 #
  10 # Hooke is distributed in the hope that it will be useful, but WITHOUT ANY
  11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  12 # A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  13 # details.
  14 #
  15 # You should have received a copy of the GNU Lesser General Public License
  16 # along with Hooke.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """Add representers to YAML to support Hooke.
  19
  20 Without introspection, YAML cannot decide how to save some
  21 objects.  By refusing to save these objects, we obviously loose
  22 that information, so make sure the things you drop are either
  23 stored somewhere else or not important.
  24
  25 >>> import yaml
  26 >>> a = numpy.array([1,2,3])
  27 >>> print yaml.dump(a)
  28 null
  29 ...
  30 <BLANKLINE>
  31
  32 The default behavior is to crash.
  33
  34 >>> yaml.Dumper.yaml_representers.pop(numpy.ndarray)  # doctest: +ELLIPSIS
  35 <function none_representer at 0x...>
  36 >>> print yaml.dump(a)  # doctest: +REPORT_UDIFF
  37 !!python/object/apply:numpy.core.multiarray._reconstruct
  38 args:
  39 - !!python/name:numpy.ndarray ''
  40 - !!python/tuple [0]
  41 - b
  42 state: !!python/tuple
  43 - 1
  44 - !!python/tuple [3]
  45 - !!python/object/apply:numpy.dtype
  46   args: [i4, 0, 1]
  47   state: !!python/tuple [3, <, null, null, null, -1, -1, 0]
  48 - false
  49 - "\\x01\\0\\0\\0\\x02\\0\\0\\0\\x03\\0\\0\\0"
  50 <BLANKLINE>
  51
  52 Hmm, at one point that crashed like this::
  53
  54     Traceback (most recent call last):
  55       ...
  56         if data in [None, ()]:
  57     TypeError: data type not understood
  58
  59 Must be because of the other representers I've loaded since.
  60
  61 Restore the representer for future tests.
  62
  63 >>> yaml.add_representer(numpy.ndarray, none_representer)
  64
  65 We also avoid !!python/unicode tags by sacrificing the string/unicode
  66 distinction.
  67
  68 >>> yaml.dump('ascii', allow_unicode=True)
  69 'ascii\\n...\\n'
  70 >>> yaml.dump(u'ascii', allow_unicode=True)
  71 'ascii\\n...\\n'
  72 >>> a = yaml.dump(u'Fran\\xe7ois', allow_unicode=True)
  73 >>> a
  74 'Fran\\xc3\\xa7ois\\n...\\n'
  75 >>> unicode(a, 'utf-8')
  76 u'Fran\\xe7ois\\n...\\n'
  77 """
  78
  79 from __future__ import absolute_import
  80 import copy_reg
  81 import sys
  82 import types
  83
  84 import numpy
  85 import yaml
  86 import yaml.constructor
  87 from yaml.constructor import ConstructorError
  88 import yaml.representer
  89
  90 from ..curve import Data, Curve
  91 from ..playlist import FilePlaylist
  92
  93
  94 DATA_INFO_TAG = u'!hooke.curve.DataInfo'
  95
  96
  97 if False: # YAML dump debugging code
  98     """To help isolate data types etc. that give YAML problems.
  99
 100     This is usually caused by external C modules (e.g. numpy) that
 101     define new types (e.g. numpy.ndarray) which YAML cannot inspect.
 102     """
 103     def ignore_aliases(data):
 104         print data, repr(data), type(data), repr(type(data))
 105         sys.stdout.flush()
 106         if data in [None, ()]:
 107             return True
 108         if isinstance(data, (str, unicode, bool, int, float)):
 109             return True
 110     yaml.representer.SafeRepresenter.ignore_aliases = staticmethod(
 111         ignore_aliases)
 112 else:
 113     # Avoid error with
 114     #   numpy.dtype(numpy.int32) in [None, ()]
 115     # See
 116     #   http://projects.scipy.org/numpy/ticket/1001
 117     def ignore_aliases(data):
 118         try:
 119             if data in [None, ()]:
 120                 return True
 121             if isinstance(data, (str, unicode, bool, int, float)):
 122                 return True
 123         except TypeError, e:
 124             pass
 125     yaml.representer.SafeRepresenter.ignore_aliases = staticmethod(
 126         ignore_aliases)
 127
 128 def unicode_representer(dumper, data):
 129     return dumper.represent_scalar(u'tag:yaml.org,2002:str', data)
 130 yaml.add_representer(unicode, unicode_representer)
 131
 132 def none_representer(dumper, data):
 133     return dumper.represent_none(None)
 134 yaml.add_representer(numpy.ndarray, none_representer)
 135
 136 def bool_representer(dumper, data):
 137     return dumper.represent_bool(data)
 138 yaml.add_representer(numpy.bool_, bool_representer)
 139
 140 def int_representer(dumper, data):
 141     return dumper.represent_int(data)
 142 yaml.add_representer(numpy.int32, int_representer)
 143 yaml.add_representer(numpy.dtype(numpy.int32), int_representer)
 144
 145 def long_representer(dumper, data):
 146     return dumper.represent_long(data)
 147 yaml.add_representer(numpy.int64, int_representer)
 148
 149 def float_representer(dumper, data):
 150     return dumper.represent_float(data)
 151 yaml.add_representer(numpy.float32, float_representer)
 152 yaml.add_representer(numpy.float64, float_representer)
 153
 154 def data_representer(dumper, data):
 155     info = dict(data.info)
 156     for key in info.keys():
 157         if key.startswith('raw '):
 158             del(info[key])
 159     return dumper.represent_mapping(DATA_INFO_TAG, info)
 160 yaml.add_representer(Data, data_representer)
 161
 162 def data_constructor(loader, node):
 163     info = loader.construct_mapping(node)
 164     return Data(shape=(0,0), dtype=numpy.float32, info=info)
 165 yaml.add_constructor(DATA_INFO_TAG, data_constructor)
 166
 167 def object_representer(dumper, data):
 168     cls = type(data)
 169     if cls in copy_reg.dispatch_table:
 170         reduce = copy_reg.dispatch_table[cls](data)
 171     elif hasattr(data, '__reduce_ex__'):
 172         reduce = data.__reduce_ex__(2)
 173     elif hasattr(data, '__reduce__'):
 174         reduce = data.__reduce__()
 175     else:
 176         raise RepresenterError("cannot represent object: %r" % data)
 177     reduce = (list(reduce)+[None]*5)[:5]
 178     function, args, state, listitems, dictitems = reduce
 179     args = list(args)
 180     if state is None:
 181         state = {}
 182     if isinstance(state, dict) and '_default_attrs' in state:
 183         for key in state['_default_attrs']:
 184             if key in state and state[key] == state['_default_attrs'][key]:
 185                 del(state[key])
 186         del(state['_default_attrs'])
 187     if listitems is not None:
 188         listitems = list(listitems)
 189     if dictitems is not None:
 190         dictitems = dict(dictitems)
 191     if function.__name__ == '__newobj__':
 192         function = args[0]
 193         args = args[1:]
 194         tag = u'tag:yaml.org,2002:python/object/new:'
 195         newobj = True
 196     else:
 197         tag = u'tag:yaml.org,2002:python/object/apply:'
 198         newobj = False
 199     function_name = u'%s.%s' % (function.__module__, function.__name__)
 200     if not args and not listitems and not dictitems \
 201             and isinstance(state, dict) and newobj:
 202         return dumper.represent_mapping(
 203                 u'tag:yaml.org,2002:python/object:'+function_name, state)
 204     if not listitems and not dictitems  \
 205             and isinstance(state, dict) and not state:
 206         return dumper.represent_sequence(tag+function_name, args)
 207     value = {}
 208     if args:
 209         value['args'] = args
 210     if state or not isinstance(state, dict):
 211         value['state'] = state
 212     if listitems:
 213         value['listitems'] = listitems
 214     if dictitems:
 215         value['dictitems'] = dictitems
 216     return dumper.represent_mapping(tag+function_name, value)
 217 yaml.add_representer(FilePlaylist, object_representer)
 218 yaml.add_representer(Curve, object_representer)
 219
 220
 221 # Monkey patch PyYAML bug 159.
 222 #   Yaml failed to restore loops in objects when __setstate__ is defined
 223 #   http://pyyaml.org/ticket/159
 224 # With viktor.x.voroshylo@jpmchase.com's patch
 225 def construct_object(self, node, deep=False):
 226     if deep:
 227         old_deep = self.deep_construct
 228         self.deep_construct = True
 229     if node in self.constructed_objects:
 230         return self.constructed_objects[node]
 231     if node in self.recursive_objects:
 232         obj = self.recursive_objects[node]
 233         if obj is None :
 234             raise ConstructorError(None, None,
 235                  "found unconstructable recursive node", node.start_mark)
 236         return obj
 237     self.recursive_objects[node] = None
 238     constructor = None
 239     tag_suffix = None
 240     if node.tag in self.yaml_constructors:
 241         constructor = self.yaml_constructors[node.tag]
 242     else:
 243         for tag_prefix in self.yaml_multi_constructors:
 244             if node.tag.startswith(tag_prefix):
 245                 tag_suffix = node.tag[len(tag_prefix):]
 246                 constructor = self.yaml_multi_constructors[tag_prefix]
 247                 break
 248         else:
 249             if None in self.yaml_multi_constructors:
 250                 tag_suffix = node.tag
 251                 constructor = self.yaml_multi_constructors[None]
 252             elif None in self.yaml_constructors:
 253                 constructor = self.yaml_constructors[None]
 254             elif isinstance(node, ScalarNode):
 255                 constructor = self.__class__.construct_scalar
 256             elif isinstance(node, SequenceNode):
 257                 constructor = self.__class__.construct_sequence
 258             elif isinstance(node, MappingNode):
 259                 constructor = self.__class__.construct_mapping
 260     if tag_suffix is None:
 261         data = constructor(self, node)
 262     else:
 263         data = constructor(self, tag_suffix, node)
 264     if isinstance(data, types.GeneratorType):
 265         generator = data
 266         data = generator.next()
 267         if self.deep_construct:
 268             self.recursive_objects[node] = data
 269             for dummy in generator:
 270                 pass
 271         else:
 272             self.state_generators.append(generator)
 273     self.constructed_objects[node] = data
 274     del self.recursive_objects[node]
 275     if deep:
 276         self.deep_construct = old_deep
 277     return data
 278 yaml.constructor.BaseConstructor.construct_object = construct_object