From: W. Trevor King Date: Sun, 22 Aug 2010 04:44:16 +0000 (-0400) Subject: Adjust Hooke internals to use unsafe YAML (not just builtin Python types). X-Git-Url: http://git.tremily.us/?p=hooke.git;a=commitdiff_plain;h=f35ab8638b8cf48a793f24006933ddd8de7686c4 Adjust Hooke internals to use unsafe YAML (not just builtin Python types). Pros: * More explicit YAML. * Simple, robust code. * Easily (de)serialize everything, regardless of nesting, etc. Cons: * Uglier YAML. After wrestling with a yaml.safe_dump()able representation, I've given up and shifted to yaml.dump(). Now we can dump anything easily, which is great :). For example, this will allow you to access polymer fit parameters. However, there will be things like !!python/object/new:hooke.playlist.FilePlaylist in the output YAML files. Such is life. While I was at it, I ran $ nosetests --with-doctest --doctest-tests hooke/ 2>&1 | log and cleaned up a number of the doctests. Now the only things that fail in there are the yet-to-be-upgraded submods left over from before my rewrite. I haven't checked test/ recently, that's next ;). --- diff --git a/hooke/command.py b/hooke/command.py index d88155c..f685327 100644 --- a/hooke/command.py +++ b/hooke/command.py @@ -94,6 +94,7 @@ class Command (object): Arguments: help BOOL (bool) Print a help message. + stack BOOL (bool) Add this command to appropriate command stacks. An example Command. ITEM: diff --git a/hooke/command_stack.py b/hooke/command_stack.py index 319d354..c6a4972 100644 --- a/hooke/command_stack.py +++ b/hooke/command_stack.py @@ -173,13 +173,7 @@ class CommandStack (list): listitems: - !!python/object:hooke.engine.CommandMessage arguments: - curve: !!python/object:hooke.curve.Curve - command_stack: !!python/object:hooke.command_stack.CommandStack {} - data: null - driver: null - info: {} - name: null - path: null + curve: !!python/object:hooke.curve.Curve {} command: curve info """ diff --git a/hooke/config.py b/hooke/config.py index 5948974..a8d006d 100644 --- a/hooke/config.py +++ b/hooke/config.py @@ -137,7 +137,8 @@ class HookeConfigParser (configparser.RawConfigParser): >>> c = HookeConfigParser(default_settings=DEFAULT_SETTINGS) >>> c.write(sys.stdout) # doctest: +ELLIPSIS # Default environmental conditions in case they are not specified in - # the force curve data. + # the force curve data. Configuration options in this section are + # available to every plugin. [conditions] # Temperature in Kelvin temperature = 301 @@ -161,7 +162,7 @@ class HookeConfigParser (configparser.RawConfigParser): ... Setting(section, option='my float', value=3.14159, type='float'), ... ]) >>> pprint.pprint(c.items(section)) # doctest: +ELLIPSIS - [('my string', 'Lorem ipsum'), + [('my string', u'Lorem ipsum'), ('my bool', True), ('my int', 13), ('my float', 3.1415...)] @@ -170,7 +171,7 @@ class HookeConfigParser (configparser.RawConfigParser): to use the standard `.get*()` methods. >>> c.get('test conversion', 'my bool') - 'True' + u'True' >>> c.getboolean('test conversion', 'my bool') True """ diff --git a/hooke/curve.py b/hooke/curve.py index 12ac5eb..42fa807 100644 --- a/hooke/curve.py +++ b/hooke/curve.py @@ -92,8 +92,8 @@ class Data (numpy.ndarray): >>> import yaml >>> print yaml.dump(d) - null - ... + !hooke.curve.DataInfo + columns: [distance (m), force (N)] """ def __new__(subtype, shape, dtype=numpy.float, buffer=None, offset=0, @@ -189,6 +189,8 @@ class Curve (object): >>> z = pickle.loads(s) >>> z + >>> z.command_stack + [}>] >>> z.command_stack[-1].arguments['curve'] == z True >>> print yaml.dump(c) # doctest: +REPORT_UDIFF @@ -199,9 +201,6 @@ class Curve (object): arguments: curve: *id001 command: curve info - data: null - driver: null - info: {} name: path path: some/path @@ -226,9 +225,6 @@ class Curve (object): arguments: curve: !!python/object:hooke.curve.Curve command_stack: *id001 - data: null - driver: null - info: {} name: path path: some/path command: curve info @@ -251,22 +247,35 @@ class Curve (object): if self.name == None and path != None: self.name = os.path.basename(path) + def _setup_default_attrs(self): + # .data contains: {name of data: list of data sets [{[x], [y]}] + # ._hooke contains a Hooke instance for Curve.load() + self._default_attrs = { + '_hooke': None, + 'command_stack': [], + 'data': None, + 'driver': None, + 'info': {}, + 'name': None, + 'path': None, + } + def __getstate__(self): - state = dict(self.__dict__) # make a copy of the attribute dict. + state = dict(self.__dict__) # make a copy of the attribute dict. del(state['_hooke']) return state def __setstate__(self, state): - # .data contains: {name of data: list of data sets [{[x], [y]}] - # ._hooke contains a Hooke instance for Curve.load() - self.name = self.driver = self.data = self._hooke = None - self.info = {} - self.command_stack = CommandStack() - for key,value in state.items(): - setattr(self, key, value) - if self.info == None: + self._setup_default_attrs() + self.__dict__.update(self._default_attrs) + if state == True: + return + self.__dict__.update(state) + self.set_path(getattr(self, 'path', None)) + if self.info in [None, {}]: self.info = {} - self.set_path(state.get('path', None)) + if type(self.command_stack) == list: + self.command_stack = CommandStack() def set_hooke(self, hooke=None): if hooke != None: diff --git a/hooke/playlist.py b/hooke/playlist.py index d6018d0..fd7ac4b 100644 --- a/hooke/playlist.py +++ b/hooke/playlist.py @@ -30,7 +30,8 @@ import types import yaml from yaml.representer import RepresenterError -from . import curve as curve +from .command_stack import CommandStack +from .curve import Curve from .util.itertools import reverse_enumerate @@ -43,10 +44,8 @@ class NoteIndexList (list): """ def __init__(self, name=None): super(NoteIndexList, self).__init__() - self.name = name - self.info = {} - self._index = 0 - self._set_ignored_attrs() + self._set_default_attrs() + self.__setstate__({'name': name}) def __str__(self): return str(self.__unicode__()) @@ -57,67 +56,33 @@ class NoteIndexList (list): def __repr__(self): return self.__str__() - def _set_ignored_attrs(self): - self._ignored_attrs = ['_ignored_attrs', '_default_attrs'] + def _set_default_attrs(self): self._default_attrs = { 'info': {}, + 'name': None, + '_index': 0, } def __getstate__(self): - state = dict(self.__dict__) - for key in self._ignored_attrs: - if key in state: - del(state[key]) - for key,value in self._default_attrs.items(): - if key in state and state[key] == value: - del(state[key]) - assert 'items' not in state - state['items'] = [] - self._assert_clean_state(self, state) - for item in self: # save curves and their attributes - item_state = self._item_getstate(item) - self._assert_clean_state(item, item_state) - state['items'].append(item_state) - return state + return self.__dict__.copy() def __setstate__(self, state): - self._set_ignored_attrs() - self.clear() - for key,value in self._default_attrs.items(): - setattr(self, key, value) - for key,value in state.items(): - if key == 'items': - continue - setattr(self, key, value) - for item_state in state['items']: - self.append(self._item_setstate(item_state)) - - def _item_getstate(self, item): - return item - - def _item_setstate(self, state): - return state - - def _assert_clean_state(self, owner, state): - for k,v in state.items(): - if k == 'drivers': # HACK. Need better driver serialization. - continue - try: - yaml.dump((k,v)) - except RepresenterError, e: - raise NotImplementedError( - 'cannot convert %s.%s = %s (%s) to YAML\n%s' - % (owner.__class__.__name__, k, v, type(v), e)) + self._set_default_attrs() + if state == True: + return + self.__dict__.update(self._default_attrs) + try: + self.__dict__.update(state) + except TypeError, e: + print state, type(state), e + if self.info in [None, {}]: + self.info = {} def _setup_item(self, item): """Perform any required initialization before returning an item. """ pass - def clear(self): - while len(self) > 0: - self.pop() - def index(self, value=None, *args, **kwargs): """Extend `list.index`, returning the current index if `value` is `None`. @@ -188,7 +153,7 @@ class NoteIndexList (list): class Playlist (NoteIndexList): - """A :class:`NoteIndexList` of :class:`hooke.curve.Curve`\s. + """A :class:`NoteIndexList` of :class:`hooke.Curve`\s. Keeps a list of :attr:`drivers` for loading curves. """ @@ -197,43 +162,26 @@ class Playlist (NoteIndexList): self.drivers = drivers self._max_loaded = 100 # curves to hold in memory simultaneously. - def _set_ignored_attrs(self): - super(Playlist, self)._set_ignored_attrs() - self._ignored_attrs.extend([ - '_item_ignored_attrs', '_item_default_attrs', - '_loaded']) - self._item_ignored_attrs = [] - self._item_default_attrs = { - 'command_stack': [], - 'data': None, - 'driver': None, - 'info': {}, - 'name': None, - } - self._loaded = [] # List of loaded curves, see :meth:`._setup_item`. - - def _item_getstate(self, item): - assert isinstance(item, curve.Curve), type(item) - state = item.__getstate__() - for key in self._item_ignored_attrs: - if key in state: - del(state[key]) - for key,value in self._item_default_attrs.items(): - if key in state and state[key] == value: - del(state[key]) - return state + def _set_default_attrs(self): + super(Playlist, self)._set_default_attrs() + self._default_attrs['drivers'] = [] + # List of loaded curves, see :meth:`._setup_item`. + self._default_attrs['_loaded'] = [] + self._default_attrs['_max_loaded'] = 100 - def _item_setstate(self, state): - for key,value in self._item_default_attrs.items(): - if key not in state: - state[key] = value - item = curve.Curve(path=None) - item.__setstate__(state) - return item + def __setstate__(self, state): + super(Playlist, self).__setstate__(state) + if self.drivers in [None, {}]: + self.drivers = [] + if self._loaded in [None, {}]: + self._loaded = [] + + def append_curve(self, curve): + self.append(curve) def append_curve_by_path(self, path, info=None, identify=True, hooke=None): path = os.path.normpath(path) - c = curve.Curve(path, info=info) + c = Curve(path, info=info) c.set_hooke(hooke) if identify == True: c.identify(self.drivers) @@ -256,6 +204,37 @@ class Playlist (NoteIndexList): class FilePlaylist (Playlist): """A file-backed :class:`Playlist`. + + Examples + -------- + + >>> p = FilePlaylist(drivers=['Driver A', 'Driver B']) + >>> p.append(Curve('dummy/path/A')) + >>> p.append(Curve('dummy/path/B')) + + The data-type is pickleable, to ensure we can move it between + processes with :class:`multiprocessing.Queue`\s. + + >>> import pickle + >>> s = pickle.dumps(p) + >>> z = pickle.loads(s) + >>> for curve in z: + ... print curve + + + >>> print z.drivers + ['Driver A', 'Driver B'] + + The data-type is also YAMLable (see :mod:`hooke.util.yaml`). + + >>> s = yaml.dump(p) + >>> z = yaml.load(s) + >>> for curve in z: + ... print curve + + + >>> print z.drivers + ['Driver A', 'Driver B'] """ version = '0.2' @@ -263,12 +242,14 @@ class FilePlaylist (Playlist): super(FilePlaylist, self).__init__(drivers, name) self.path = self._base_path = None self.set_path(path) - self._relative_curve_paths = True + self.relative_curve_paths = True + self._relative_curve_paths = False - def _set_ignored_attrs(self): - super(FilePlaylist, self)._set_ignored_attrs() - self._ignored_attrs.append('_digest') - self._digest = None + def _set_default_attrs(self): + super(FilePlaylist, self)._set_default_attrs() + self._default_attrs['relative_curve_paths'] = True + self._default_attrs['_relative_curve_paths'] = False + self._default_attrs['_digest'] = None def __getstate__(self): state = super(FilePlaylist, self).__getstate__() @@ -277,30 +258,16 @@ class FilePlaylist (Playlist): return state def __setstate__(self, state): - assert('version') in state, state - version = state.pop('version') - assert version == FilePlaylist.version, ( - 'invalid version %s (%s) != %s (%s)' - % (version, type(version), - FilePlaylist.version, type(FilePlaylist.version))) + if 'version' in state: + version = state.pop('version') + assert version == FilePlaylist.version, ( + 'invalid version %s (%s) != %s (%s)' + % (version, type(version), + FilePlaylist.version, type(FilePlaylist.version))) super(FilePlaylist, self).__setstate__(state) - def _item_getstate(self, item): - state = super(FilePlaylist, self)._item_getstate(item) - if state.get('path', None) != None: - path = os.path.abspath(os.path.expanduser(state['path'])) - if self._relative_curve_paths == True: - path = os.path.relpath(path, self._base_path) - state['path'] = path - return state - - def _item_setstate(self, state): - item = super(FilePlaylist, self)._item_setstate(state) - if 'path' in state: - item.set_path(os.path.join(self._base_path, state['path'])) - return item - def set_path(self, path): + orig_base_path = getattr(self, '_base_path', None) if path == None: if self._base_path == None: self._base_path = os.getcwd() @@ -312,10 +279,27 @@ class FilePlaylist (Playlist): os.path.expanduser(self.path))) if self.name == None: self.name = os.path.basename(path) + if self._base_path != orig_base_path: + self.update_curve_paths() + + def update_curve_paths(self): + for curve in self: + curve.set_path(self._curve_path(curve.path)) + + def _curve_path(self, path): + if self._base_path == None: + self._base_path = os.getcwd() + path = os.path.join(self._base_path, path) + if self._relative_curve_paths == True: + path = os.path.relpath(path, self._base_path) + return path + + def append_curve(self, curve): + curve.set_path(self._curve_path(curve.path)) + super(FilePlaylist, self).append_curve(curve) def append_curve_by_path(self, path, *args, **kwargs): - if self._base_path != None: - path = os.path.join(self._base_path, path) + path = self._curve_path(path) super(FilePlaylist, self).append_curve_by_path(path, *args, **kwargs) def is_saved(self): @@ -332,14 +316,14 @@ class FilePlaylist (Playlist): >>> p = FilePlaylist(drivers=[], ... path=os.path.join(root_path, 'to','playlist')) >>> p.info['note'] = 'An example playlist' - >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'one')) + >>> c = Curve(os.path.join(root_path, 'to', 'curve', 'one')) >>> c.info['note'] = 'The first curve' - >>> p.append(c) - >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'two')) + >>> p.append_curve(c) + >>> c = Curve(os.path.join(root_path, 'to', 'curve', 'two')) >>> c.info['note'] = 'The second curve' - >>> p.append(c) + >>> p.append_curve(c) >>> p.digest() - '\xa1\x1ax\xb1|\x84uA\xe4\x1d\xbf`\x004|\x82\xc2\xdd\xc1\x9e' + 'f\xe26i\xb98i\x1f\xb61J7:\xf2\x8e\x1d\xde\xc3}g' """ string = self.flatten() return hashlib.sha1(string).digest() @@ -349,10 +333,14 @@ class FilePlaylist (Playlist): A playlist is a YAML document with the following minimal syntax:: - version: '0.2' - items: - - path: picoforce.000 - - path: picoforce.001 + !!python/object/new:hooke.playlist.FilePlaylist + state: + version: '0.2' + listitems: + - !!python/object:hooke.curve.Curve + path: /path/to/curve/one + - !!python/object:hooke.curve.Curve + path: /path/to/curve/two Relative paths are interpreted relative to the location of the playlist file. @@ -366,151 +354,89 @@ class FilePlaylist (Playlist): >>> p = FilePlaylist(drivers=[], ... path=os.path.join(root_path, 'to','playlist')) >>> p.info['note'] = 'An example playlist' - >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'one')) + >>> c = Curve(os.path.join(root_path, 'to', 'curve', 'one')) >>> c.info['note'] = 'The first curve' - >>> p.append(c) - >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'two')) + >>> p.append_curve(c) + >>> c = Curve(os.path.join(root_path, 'to', 'curve', 'two')) >>> c.info['attr with spaces'] = 'The second curve\\nwith endlines' >>> c.command_stack.extend([ ... CommandMessage('command A', {'arg 0':0, 'arg 1':'X'}), - ... CommandMessage('command B', {'arg 0':1, 'arg 1':'Y'}), + ... CommandMessage('command B', {'arg 0':1, 'curve':c}), ... ]) - >>> p.append(c) + >>> p.append_curve(c) >>> print p.flatten() # doctest: +REPORT_UDIFF # Hooke playlist version 0.2 - _base_path: /path/to - _index: 0 - _max_loaded: 100 - _relative_curve_paths: true - drivers: [] - info: {note: An example playlist} - items: - - info: {note: The first curve} + !!python/object/new:hooke.playlist.FilePlaylist + listitems: + - !!python/object:hooke.curve.Curve + info: {note: The first curve} name: one path: curve/one - - command_stack: !!python/object/new:hooke.command_stack.CommandStack + - &id001 !!python/object:hooke.curve.Curve + command_stack: !!python/object/new:hooke.command_stack.CommandStack listitems: - !!python/object:hooke.engine.CommandMessage arguments: {arg 0: 0, arg 1: X} command: command A - !!python/object:hooke.engine.CommandMessage - arguments: {arg 0: 1, arg 1: Y} + arguments: + arg 0: 1 + curve: *id001 command: command B info: {attr with spaces: 'The second curve with endlines'} name: two path: curve/two - name: playlist.hkp - path: /path/to/playlist.hkp - version: '0.2' + state: + _base_path: /path/to + info: {note: An example playlist} + name: playlist.hkp + path: /path/to/playlist.hkp + version: '0.2' - >>> p._relative_curve_paths = False + >>> p.relative_curve_paths = False >>> print p.flatten() # doctest: +REPORT_UDIFF # Hooke playlist version 0.2 - _base_path: /path/to - _index: 0 - _max_loaded: 100 - _relative_curve_paths: false - drivers: [] - info: {note: An example playlist} - items: - - info: {note: The first curve} + !!python/object/new:hooke.playlist.FilePlaylist + listitems: + - !!python/object:hooke.curve.Curve + info: {note: The first curve} name: one path: /path/to/curve/one - - command_stack: !!python/object/new:hooke.command_stack.CommandStack + - &id001 !!python/object:hooke.curve.Curve + command_stack: !!python/object/new:hooke.command_stack.CommandStack listitems: - !!python/object:hooke.engine.CommandMessage arguments: {arg 0: 0, arg 1: X} command: command A - !!python/object:hooke.engine.CommandMessage - arguments: {arg 0: 1, arg 1: Y} + arguments: + arg 0: 1 + curve: *id001 command: command B info: {attr with spaces: 'The second curve with endlines'} name: two path: /path/to/curve/two - name: playlist.hkp - path: /path/to/playlist.hkp - version: '0.2' + state: + _base_path: /path/to + info: {note: An example playlist} + name: playlist.hkp + path: /path/to/playlist.hkp + relative_curve_paths: false + version: '0.2' """ - yaml_string = yaml.dump(self.__getstate__(), allow_unicode=True) - return ('# Hooke playlist version %s\n' % self.version) + yaml_string - - def from_string(self, string): - u"""Load a playlist from a string. - - Examples - -------- + rcp = self._relative_curve_paths + self._relative_curve_paths = self.relative_curve_paths + self.update_curve_paths() + self._relative_curve_paths = rcp - Minimal example. - - >>> string = '''# Hooke playlist version 0.2 - ... version: '0.2' - ... items: - ... - path: picoforce.000 - ... - path: picoforce.001 - ... ''' - >>> p = FilePlaylist(drivers=[], - ... path=os.path.join('/path', 'to', 'my', 'playlist')) - >>> p.from_string(string) - >>> for curve in p: - ... print curve.path - /path/to/my/picoforce.000 - /path/to/my/picoforce.001 - - More complicated example. - - >>> string = '''# Hooke playlist version 0.2 - ... _base_path: /path/to - ... _digest: null - ... _index: 1 - ... _max_loaded: 100 - ... _relative_curve_paths: true - ... info: {note: An example playlist} - ... items: - ... - info: {note: The first curve} - ... path: curve/one - ... - command_stack: !!python/object/new:hooke.command_stack.CommandStack - ... listitems: - ... - !!python/object:hooke.engine.CommandMessage - ... arguments: {arg 0: 0, arg 1: X} - ... command: command A - ... - !!python/object:hooke.engine.CommandMessage - ... arguments: {arg 0: 1, arg 1: Y} - ... command: command B - ... info: {attr with spaces: 'The second curve - ... - ... with endlines'} - ... name: two - ... path: curve/two - ... name: playlist.hkp - ... path: /path/to/playlist.hkp - ... version: '0.2' - ... ''' - >>> p = FilePlaylist(drivers=[], - ... path=os.path.join('path', 'to', 'my', 'playlist')) - >>> p.from_string(string) - >>> p._index - 1 - >>> p.info - {'note': 'An example playlist'} - >>> for curve in p: - ... print curve.name, curve.path - one /path/to/curve/one - two /path/to/curve/two - >>> p[-1].info['attr with spaces'] - 'The second curve\\nwith endlines' - >>> type(p[-1].command_stack) - - >>> p[-1].command_stack # doctest: +NORMALIZE_WHITESPACE - [, - ] - """ - state = yaml.load(string) - self.__setstate__(state) + yaml_string = yaml.dump(self, allow_unicode=True) + self.update_curve_paths() + return ('# Hooke playlist version %s\n' % self.version) + yaml_string def save(self, path=None, makedirs=True): """Saves the playlist to a YAML file. @@ -523,31 +449,109 @@ class FilePlaylist (Playlist): f.write(self.flatten()) self._digest = self.digest() - def load(self, path=None, identify=True, hooke=None): - """Load a playlist from a file. - """ - self.set_path(path) - with open(self.path, 'r') as f: - text = f.read() - self.from_string(text) - self._digest = self.digest() - for curve in self: - curve.set_hooke(hooke) - if identify == True: - curve.identify(self.drivers) +def from_string(string): + u"""Load a playlist from a string. + + Examples + -------- + + Minimal example. + + >>> string = '''# Hooke playlist version 0.2 + ... !!python/object/new:hooke.playlist.FilePlaylist + ... state: + ... version: '0.2' + ... listitems: + ... - !!python/object:hooke.curve.Curve + ... path: curve/one + ... - !!python/object:hooke.curve.Curve + ... path: curve/two + ... ''' + >>> p = from_string(string) + >>> p.set_path('/path/to/playlist') + >>> for curve in p: + ... print curve.name, curve.path + one /path/to/curve/one + two /path/to/curve/two + + More complicated example. + + >>> string = '''# Hooke playlist version 0.2 + ... !!python/object/new:hooke.playlist.FilePlaylist + ... listitems: + ... - !!python/object:hooke.curve.Curve + ... info: {note: The first curve} + ... name: one + ... path: /path/to/curve/one + ... - &id001 !!python/object:hooke.curve.Curve + ... command_stack: !!python/object/new:hooke.command_stack.CommandStack + ... listitems: + ... - !!python/object:hooke.engine.CommandMessage + ... arguments: {arg 0: 0, arg 1: X} + ... command: command A + ... - !!python/object:hooke.engine.CommandMessage + ... arguments: + ... arg 0: 1 + ... curve: *id001 + ... command: command B + ... info: {attr with spaces: 'The second curve + ... + ... with endlines'} + ... name: two + ... path: /path/to/curve/two + ... state: + ... _base_path: /path/to + ... _index: 1 + ... info: {note: An example playlist} + ... name: playlist.hkp + ... path: /path/to/playlist.hkp + ... version: '0.2' + ... ''' + >>> p = from_string(string) + >>> p.set_path('/path/to/playlist') + >>> p._index + 1 + >>> p.info + {'note': 'An example playlist'} + >>> for curve in p: + ... print curve.name, curve.path + one /path/to/curve/one + two /path/to/curve/two + >>> p[-1].info['attr with spaces'] + 'The second curve\\nwith endlines' + >>> type(p[-1].command_stack) + + >>> p[0].command_stack + [] + >>> type(p[0].command_stack) + + >>> p[-1].command_stack # doctest: +NORMALIZE_WHITESPACE + [, + }>] + >>> type(p[1].command_stack) + + >>> c2 = p[-1] + >>> c2.command_stack[-1].arguments['curve'] == c2 + True + """ + return yaml.load(string) -class Playlists (NoteIndexList): - """A :class:`NoteIndexList` of :class:`FilePlaylist`\s. +def load(path=None, identify=True, hooke=None): + """Load a playlist from a file. """ - def __init__(self, *arg, **kwargs): - super(Playlists, self).__init__(*arg, **kwargs) + with open(self.path, 'r') as f: + text = f.read() + playlist = from_string(text) + playlist._digest = playlist.digest() + for curve in playlist: + curve.set_hooke(hooke) + if identify == True: + curve.identify(playlist.drivers) + return playlist - def _item_getstate(self, item): - assert isinstance(item, FilePlaylist), type(item) - return item.__getstate__() - def _item_setstate(self, state): - item = FilePlaylist(drivers=[]) - item.__setstate__(state) - return item +class Playlists (NoteIndexList): + """A :class:`NoteIndexList` of :class:`FilePlaylist`\s. + """ + pass diff --git a/hooke/util/si.py b/hooke/util/si.py index 14d9010..f9614bf 100644 --- a/hooke/util/si.py +++ b/hooke/util/si.py @@ -182,7 +182,7 @@ def join_data_label(name, unit): >>> join_data_label('z piezo', 'm') 'z piezo (m)' >>> join_data_label('deflection', 'N') - 'deflection N' + 'deflection (N)' """ return '%s (%s)' % (name, unit) diff --git a/hooke/util/yaml.py b/hooke/util/yaml.py index f86e3b1..e7f9980 100644 --- a/hooke/util/yaml.py +++ b/hooke/util/yaml.py @@ -17,25 +17,47 @@ null The default behavior is to crash. >>> yaml.Dumper.yaml_representers.pop(numpy.ndarray) # doctest: +ELLIPSIS - + >>> print yaml.dump(a) -Traceback (most recent call last): - ... - if data in [None, ()]: -TypeError: data type not understood +!!python/object/apply:numpy.core.multiarray._reconstruct +args: +- !!python/name:numpy.ndarray '' +- !!python/tuple [0] +- b +state: !!python/tuple +- 1 +- !!python/tuple [3] +- null +- false +- "\\x01\\0\\0\\0\\x02\\0\\0\\0\\x03\\0\\0\\0" + + +Hmm, at one point that crashed like this:: + + Traceback (most recent call last): + ... + if data in [None, ()]: + TypeError: data type not understood + +Must be because of the other representers I've loaded since. Restore the representer for future tests. ->>> yaml.add_representer(numpy.ndarray, ndarray_representer) +>>> yaml.add_representer(numpy.ndarray, none_representer) """ from __future__ import absolute_import +import copy_reg import sys +import types import numpy -import yaml #from yaml.representer import Representer +import yaml +import yaml.constructor +import yaml.representer -from ..curve import Data +from ..curve import Data, Curve +from ..playlist import FilePlaylist if False: # YAML dump debugging code @@ -95,10 +117,121 @@ yaml.add_representer(numpy.float64, float_representer) def data_representer(dumper, data): info = dict(data.info) - print 'KEYS', info.keys() for key in info.keys(): - if key.startswith('raw '):# or 'peak' in key: #or key not in ['surface deflection offset (m)', 'z piezo sensitivity (m/V)', 'z piezo scan (V/bit)', 'z piezo gain', 'deflection range (V)', 'z piezo range (V)', 'spring constant (N/m)', 'z piezo scan size (V)', 'deflection sensitivity (V/bit)', 'z piezo ramp size (V/bit)', 'surface deflection offset', 'z piezo offset (V)', 'name']: + if key.startswith('raw '): del(info[key]) - print 'AAAS', info.keys() return dumper.represent_mapping(u'!hooke.curve.DataInfo', info) yaml.add_representer(Data, data_representer) + +def object_representer(dumper, data): + cls = type(data) + if cls in copy_reg.dispatch_table: + reduce = copy_reg.dispatch_table[cls](data) + elif hasattr(data, '__reduce_ex__'): + reduce = data.__reduce_ex__(2) + elif hasattr(data, '__reduce__'): + reduce = data.__reduce__() + else: + raise RepresenterError("cannot represent object: %r" % data) + reduce = (list(reduce)+[None]*5)[:5] + function, args, state, listitems, dictitems = reduce + args = list(args) + if state is None: + state = {} + if isinstance(state, dict) and '_default_attrs' in state: + for key in state['_default_attrs']: + if key in state and state[key] == state['_default_attrs'][key]: + del(state[key]) + del(state['_default_attrs']) + if listitems is not None: + listitems = list(listitems) + if dictitems is not None: + dictitems = dict(dictitems) + if function.__name__ == '__newobj__': + function = args[0] + args = args[1:] + tag = u'tag:yaml.org,2002:python/object/new:' + newobj = True + else: + tag = u'tag:yaml.org,2002:python/object/apply:' + newobj = False + function_name = u'%s.%s' % (function.__module__, function.__name__) + if not args and not listitems and not dictitems \ + and isinstance(state, dict) and newobj: + return dumper.represent_mapping( + u'tag:yaml.org,2002:python/object:'+function_name, state) + if not listitems and not dictitems \ + and isinstance(state, dict) and not state: + return dumper.represent_sequence(tag+function_name, args) + value = {} + if args: + value['args'] = args + if state or not isinstance(state, dict): + value['state'] = state + if listitems: + value['listitems'] = listitems + if dictitems: + value['dictitems'] = dictitems + return dumper.represent_mapping(tag+function_name, value) +yaml.add_representer(FilePlaylist, object_representer) +yaml.add_representer(Curve, object_representer) + + +# Monkey patch PyYAML bug 159. +# Yaml failed to restore loops in objects when __setstate__ is defined +# http://pyyaml.org/ticket/159 +# With viktor.x.voroshylo@jpmchase.com's patch +def construct_object(self, node, deep=False): + if deep: + old_deep = self.deep_construct + self.deep_construct = True + if node in self.constructed_objects: + return self.constructed_objects[node] + if node in self.recursive_objects: + obj = self.recursive_objects[node] + if obj is None : + raise ConstructorError(None, None, + "found unconstructable recursive node", node.start_mark) + return obj + self.recursive_objects[node] = None + constructor = None + tag_suffix = None + if node.tag in self.yaml_constructors: + constructor = self.yaml_constructors[node.tag] + else: + for tag_prefix in self.yaml_multi_constructors: + if node.tag.startswith(tag_prefix): + tag_suffix = node.tag[len(tag_prefix):] + constructor = self.yaml_multi_constructors[tag_prefix] + break + else: + if None in self.yaml_multi_constructors: + tag_suffix = node.tag + constructor = self.yaml_multi_constructors[None] + elif None in self.yaml_constructors: + constructor = self.yaml_constructors[None] + elif isinstance(node, ScalarNode): + constructor = self.__class__.construct_scalar + elif isinstance(node, SequenceNode): + constructor = self.__class__.construct_sequence + elif isinstance(node, MappingNode): + constructor = self.__class__.construct_mapping + if tag_suffix is None: + data = constructor(self, node) + else: + data = constructor(self, tag_suffix, node) + if isinstance(data, types.GeneratorType): + generator = data + data = generator.next() + if self.deep_construct: + self.recursive_objects[node] = data + for dummy in generator: + pass + else: + self.state_generators.append(generator) + self.constructed_objects[node] = data + del self.recursive_objects[node] + if deep: + self.deep_construct = old_deep + return data +yaml.constructor.BaseConstructor.construct_object = construct_object