From f1b96b0334501099017e9665d60ff238c37a01fa Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 21 Aug 2010 09:57:08 -0400 Subject: [PATCH] Transition from v0.1 XML playlists to v0.2 YAML playlists. All test/ tests pass except for tutorial.py and note.py, which take forever to run. I'm going to figure out what's going on there next. Anyone with old playlist files can either upgrade by hand (the new syntax is pretty simple, see the playlists under test/data/ for examples), or use the automatic script contrib/upgrade_playlist_0p1.py which converts the playlists automatically. The output from the upgrade script is, like most auto-generated files, less concise than a hand coded playlist, but it will work just fine. For an introduction to YAML, see http://www.yaml.orlg/ which contains links YAML libraries in a number of languages. The playlists are generated with PyYAML http://pyyaml.org/ which handles YAML 1.1 http://yaml.org/spec/1.1/ The Hooke playlist format is pretty simple, so it shouldn't be too strongly tied to any particular version of YAML, although I haven't actually tested it with a 1.0 or 1.2 parser. --- contrib/upgrade_playlist_0p1.py | 93 ++++++ hooke/command_stack.py | 45 ++- hooke/compat/minidom.py | 70 ---- hooke/curve.py | 31 +- hooke/engine.py | 1 + hooke/playlist.py | 416 ++++++++++++++++-------- hooke/plugin/note.py | 2 +- hooke/plugin/playlist.py | 6 +- test/curve_info.py | 4 +- test/data/fclamp_hemingway/playlist.hkp | 53 +-- test/data/test.hkp | 8 +- test/data/vclamp_jpk/playlist.hkp | 12 +- test/data/vclamp_mfp3d/playlist.hkp | 59 ++-- test/data/vclamp_picoforce/playlist.hkp | 211 ++++++------ test/data/vclamp_wtk/playlist.hkp | 67 ++-- test/flat_filter_playlist.py | 2 +- test/hemingway_driver.py | 6 +- test/jpk_driver.py | 27 +- test/mfp3d_driver.py | 20 +- test/multiple_curve_analysis.py | 2 +- test/picoforce_driver.py | 14 +- test/polymer_fit.py | 10 +- test/wtk_driver.py | 6 +- 23 files changed, 687 insertions(+), 478 deletions(-) create mode 100755 contrib/upgrade_playlist_0p1.py delete mode 100644 hooke/compat/minidom.py diff --git a/contrib/upgrade_playlist_0p1.py b/contrib/upgrade_playlist_0p1.py new file mode 100755 index 0000000..4a0cf80 --- /dev/null +++ b/contrib/upgrade_playlist_0p1.py @@ -0,0 +1,93 @@ +#!/usr/bin/python +# Copyright + +"""Upgrade version 0.1 playlists (XML) to the current Hooke playlist +file format. +""" + +import sys +import xml.dom.minidom + +import yaml + +from hooke.playlist import FilePlaylist + + +class Converter (FilePlaylist): + def _restore_key(self, key): + """Restore keys encoded with :meth:`_clean_key`. + """ + return key.replace(u'\u00B7', ' ') + + def _from_xml_doc(self, doc, identify=True): + """Load a playlist from an :class:`xml.dom.minidom.Document` + instance. + """ + root = doc.documentElement + for attribute,value in root.attributes.items(): + attribute = self._restore_key(attribute) + if attribute == 'version': + assert value == '0.1', \ + 'Cannot read v%s playlist with a v%s reader' \ + % (value, self.version) + elif attribute == 'index': + self._index = int(value) + else: + self.info[attribute] = value + for curve_element in doc.getElementsByTagName('curve'): + path = curve_element.getAttribute('path') + info = dict([(self._restore_key(key), value) + for key,value in curve_element.attributes.items()]) + info.pop('path') + self.append_curve_by_path(path, info, identify=identify) + self.jump(self._index) # ensure valid index + + def from_string(self, string, identify=True): + u"""Load a playlist from a string. + + Examples + -------- + + >>> string = ''' + ... + ... + ... + ... + ... ''' + >>> p = FilePlaylist(drivers=[], + ... path=os.path.join('path', 'to', 'my', 'playlist')) + >>> p.from_string(string, identify=False) + >>> p._index + 1 + >>> p.info + {u'note': u'An example playlist'} + >>> for curve in p: + ... print curve.path + path/to/curve/one + path/to/curve/two + >>> p[-1].info['attr with spaces'] + u'The second curve\\nwith endlines' + """ + doc = xml.dom.minidom.parseString(string) + self._from_xml_doc(doc, identify=identify) + + def load(self, path=None, identify=True, hooke=None): + """Load a playlist from a file. + """ + self.set_path(path) + doc = xml.dom.minidom.parse(self.path) + self._from_xml_doc(doc, identify=identify) + #self._digest = self.digest() + for curve in self: + curve.set_hooke(hooke) + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print >> sys.stderr, 'usage: upgrade_playlist_0p1.py X.hkp [Y.hkp ...]' + sys.exit(1) + + for path in sys.argv[1:]: + p = Converter(drivers=None, path=path) + p.load(identify=False) + p.save() diff --git a/hooke/command_stack.py b/hooke/command_stack.py index 17a633b..234b3f3 100644 --- a/hooke/command_stack.py +++ b/hooke/command_stack.py @@ -40,7 +40,7 @@ class CommandStack (list): Implement a dummy :meth:`execute_command` for testing. - >>> def execute_cmd(hooke, command_message): + >>> def execute_cmd(hooke, command_message, stack=None): ... cm = command_message ... print 'EXECUTE', cm.command, cm.arguments >>> c.execute_command = execute_cmd @@ -80,12 +80,38 @@ class CommandStack (list): '', ''] + The data-type is also pickleable, to ensure we can move it between + processes with :class:`multiprocessing.Queue`\s and easily save it + to disk. + + >>> import pickle + >>> s = pickle.dumps(c) + >>> z = pickle.loads(s) + >>> print [repr(cm) for cm in c] # doctest: +NORMALIZE_WHITESPACE + ['', + '', + '', + '', + ''] + There is also a convenience function for clearing the stack. >>> c.clear() >>> print [repr(cm) for cm in c] [] """ + def __getstate__(self): + state = [{'command':cm.command, 'arguments':cm.arguments} + for cm in self] + return state + + def __setstate__(self, state): + self.clear() + for cm_state in state: + self.append(CommandMessage( + command=cm_state['command'], + arguments=cm_state['arguments'])) + def execute(self, hooke, stack=False): """Execute a stack of commands. @@ -123,8 +149,23 @@ class FileCommandStack (CommandStack): def __init__(self, *args, **kwargs): super(FileCommandStack, self).__init__(*args, **kwargs) - self.name = None + self.name = self.path = None + + def __getstate__(self): + command_stack = super(FileCommandStack, self).__getstate__() + state = { + 'command stack': command_stack, + 'path': self.path, + 'name': self.name, + } + return state + + def __setstate__(self, state): + super(FileCommandStack, self).__setstate__( + state.get('command stack', [])) + self.name = state.get('name', None) self.path = None + self.set_path(state.get('path', None)) def set_path(self, path): """Set the path (and possibly the name) of the command stack. diff --git a/hooke/compat/minidom.py b/hooke/compat/minidom.py deleted file mode 100644 index 01d3be0..0000000 --- a/hooke/compat/minidom.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (C) 2010 W. Trevor King -# -# This file is part of Hooke. -# -# Hooke is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Hooke is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General -# Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with Hooke. If not, see -# . - -"""Dynamically patch :mod:`xml.dom.minidom`'s attribute value escaping. - -:meth:`xml.dom.minidom.Element.setAttribute` doesn't preform some -character escaping (see the `Python bug`_ and `XML specs`_). -Importing this module applies the suggested patch dynamically. - -.. _Python bug: http://bugs.python.org/issue5752 -.. _XML specs: - http://www.w3.org/TR/2000/WD-xml-c14n-20000119.html#charescaping -""" - -import logging -import xml.dom.minidom - - -def _write_data(writer, data, isAttrib=False): - "Writes datachars to writer." - if isAttrib: - data = data.replace("\r", " ").replace("\n", " ") - data = data.replace("\t", " ").replace('"', """) - writer.write(data) -xml.dom.minidom._write_data = _write_data - -def writexml(self, writer, indent="", addindent="", newl=""): - # indent = current indentation - # addindent = indentation to add to higher levels - # newl = newline string - writer.write(indent+"<" + self.tagName) - - attrs = self._get_attributes() - a_names = attrs.keys() - a_names.sort() - - for a_name in a_names: - writer.write(" %s=\"" % a_name) - _write_data(writer, attrs[a_name].value, isAttrib=True) - writer.write("\"") - if self.childNodes: - writer.write(">%s"%(newl)) - for node in self.childNodes: - node.writexml(writer,indent+addindent,addindent,newl) - writer.write("%s%s" % (indent,self.tagName,newl)) - else: - writer.write("/>%s"%(newl)) -# For an introduction to overriding instance methods, see -# http://irrepupavel.com/documents/python/instancemethod/ -instancemethod = type(xml.dom.minidom.Element.writexml) -xml.dom.minidom.Element.writexml = instancemethod( - writexml, None, xml.dom.minidom.Element) - -logging.warn( - 'monkey patched xml.dom.minidom.Element and ._write_data for issue5752') diff --git a/hooke/curve.py b/hooke/curve.py index a87a7fd..629e0a8 100644 --- a/hooke/curve.py +++ b/hooke/curve.py @@ -165,13 +165,13 @@ class Curve (object): """ def __init__(self, path, info=None): #the data dictionary contains: {name of data: list of data sets [{[x], [y]}] - self.path = path + self.name = None + self.set_path(path) self.driver = None self.data = None if info == None: info = {} self.info = info - self.name = os.path.basename(path) self.command_stack = CommandStack() self._hooke = None # Hooke instance for Curve.load() @@ -184,14 +184,29 @@ class Curve (object): def __repr__(self): return self.__str__() + def set_path(self, path): + self.path = path + if self.name == None and path != None: + self.name = os.path.basename(path) + def __getstate__(self): - data = dict(self.__dict__) - del(data['_hooke']) - return data + state = dict(self.__dict__) + del(state['_hooke']) + dc = state['command_stack'] + if hasattr(dc, '__getstate__'): + state['command_stack'] = dc.__getstate__() + return state - def __setstate__(self, data): - self._hooke = None - for key,value in data.items(): + def __setstate__(self, state): + self.name = self._hooke = None + for key,value in state.items(): + if key == 'path': + self.set_path(value) + continue + elif key == 'command_stack': + v = CommandStack() + v.__setstate__(value) + value = v setattr(self, key, value) def set_hooke(self, hooke=None): diff --git a/hooke/engine.py b/hooke/engine.py index e024c73..be4be5f 100644 --- a/hooke/engine.py +++ b/hooke/engine.py @@ -97,6 +97,7 @@ class CommandEngine (object): be ready to receive the next :class:`QueueMessage`. """ log = logging.getLogger('hooke') + log.debug('engine starting') while True: log.debug('engine waiting for command') msg = ui_to_command_queue.get() diff --git a/hooke/playlist.py b/hooke/playlist.py index f8199d4..fdebc40 100644 --- a/hooke/playlist.py +++ b/hooke/playlist.py @@ -26,10 +26,11 @@ import hashlib import os import os.path import types -import xml.dom.minidom + +import yaml +from yaml.representer import RepresenterError from . import curve as curve -from .compat import minidom as minidom # dynamically patch xml.sax.minidom from .util.itertools import reverse_enumerate @@ -45,6 +46,7 @@ class NoteIndexList (list): self.name = name self.info = {} self._index = 0 + self._set_ignored_attrs() def __str__(self): return str(self.__unicode__()) @@ -55,6 +57,56 @@ class NoteIndexList (list): def __repr__(self): return self.__str__() + def _set_ignored_attrs(self): + self._ignored_attrs = ['_ignored_attrs', '_default_attrs'] + self._default_attrs = { + 'info': {}, + } + + def __getstate__(self): + state = dict(self.__dict__) + for key in self._ignored_attrs: + if key in state: + del(state[key]) + for key,value in self._default_attrs.items(): + if key in state and state[key] == value: + del(state[key]) + assert 'items' not in state + state['items'] = [] + self._assert_clean_state(self, state) + for item in self: # save curves and their attributes + item_state = self._item_getstate(item) + self._assert_clean_state(item, item_state) + state['items'].append(item_state) + return state + + def __setstate__(self, state): + self._set_ignored_attrs() + for key,value in self._default_attrs.items(): + setattr(self, key, value) + for key,value in state.items(): + if key == 'items': + continue + setattr(self, key, value) + for item_state in state['items']: + self.append(self._item_setstate(item_state)) + + def _item_getstate(self, item): + return item + + def _item_setstate(self, state): + return state + + def _assert_clean_state(self, owner, state): + return + for k,v in state.items(): + try: + yaml.safe_dump((k,v)) + except RepresenterError, e: + raise NotImplementedError( + 'cannot convert %s.%s = %s (%s) to safe YAML' + % (owner.__class__.__name__, k, v, type(v))) + def _setup_item(self, item): """Perform any required initialization before returning an item. """ @@ -112,9 +164,14 @@ class NoteIndexList (list): yield item self._index = index - def filter(self, keeper_fn=lambda item:True, *args, **kwargs): + def filter(self, keeper_fn=lambda item:True, load_curves=True, + *args, **kwargs): c = copy.deepcopy(self) - for item in c.items(reverse=True): + if load_curves == True: + items = c.items(reverse=True) + else: + items = reversed(c) + for item in items: if keeper_fn(item, *args, **kwargs) != True: c.remove(item) try: # attempt to maintain the same current item @@ -132,9 +189,42 @@ class Playlist (NoteIndexList): def __init__(self, drivers, name=None): super(Playlist, self).__init__(name=name) self.drivers = drivers - self._loaded = [] # List of loaded curves, see :meth:`._setup_item`. self._max_loaded = 100 # curves to hold in memory simultaneously. + def _set_ignored_attrs(self): + super(Playlist, self)._set_ignored_attrs() + self._ignored_attrs.extend([ + '_item_ignored_attrs', '_item_default_attrs', + '_loaded']) + self._item_ignored_attrs = [] + self._item_default_attrs = { + 'command_stack': [], + 'data': None, + 'driver': None, + 'info': {}, + 'name': None, + } + self._loaded = [] # List of loaded curves, see :meth:`._setup_item`. + + def _item_getstate(self, item): + assert isinstance(item, curve.Curve), type(item) + state = item.__getstate__() + for key in self._item_ignored_attrs: + if key in state: + del(state[key]) + for key,value in self._item_default_attrs.items(): + if key in state and state[key] == value: + del(state[key]) + return state + + def _item_setstate(self, state): + for key,value in self._item_default_attrs.items(): + if key not in state: + state[key] = value + item = curve.Curve(path=None) + item.__setstate__(state) + return item + def append_curve_by_path(self, path, info=None, identify=True, hooke=None): path = os.path.normpath(path) c = curve.Curve(path, info=info) @@ -161,28 +251,65 @@ class Playlist (NoteIndexList): class FilePlaylist (Playlist): """A file-backed :class:`Playlist`. """ - version = '0.1' + version = '0.2' def __init__(self, drivers, name=None, path=None): super(FilePlaylist, self).__init__(drivers, name) - self.path = None + self.path = self._base_path = None self.set_path(path) + self._relative_curve_paths = True + + def _set_ignored_attrs(self): + super(FilePlaylist, self)._set_ignored_attrs() + self._ignored_attrs.append('_digest') self._digest = None - self._ignored_keys = [ - 'experiment', # class instance, not very exciting. - ] + + def __getstate__(self): + state = super(FilePlaylist, self).__getstate__() + assert 'version' not in state, state + state['version'] = self.version + return state + + def __setstate__(self, state): + assert('version') in state, state + version = state.pop('version') + assert version == FilePlaylist.version, ( + 'invalid version %s (%s) != %s (%s)' + % (version, type(version), + FilePlaylist.version, type(FilePlaylist.version))) + super(FilePlaylist, self).__setstate__(state) + + def _item_getstate(self, item): + state = super(FilePlaylist, self)._item_getstate(item) + if state.get('path', None) != None: + path = os.path.abspath(os.path.expanduser(state['path'])) + if self._relative_curve_paths == True: + path = os.path.relpath(path, self._base_path) + state['path'] = path + return state + + def _item_setstate(self, state): + item = super(FilePlaylist, self)._item_setstate(state) + if 'path' in state: + item.set_path(os.path.join(self._base_path, state['path'])) + return item def set_path(self, path): - if path != None: + if path == None: + if self._base_path == None: + self._base_path = os.getcwd() + else: if not path.endswith('.hkp'): path += '.hkp' self.path = path + self._base_path = os.path.dirname(os.path.abspath( + os.path.expanduser(self.path))) if self.name == None: self.name = os.path.basename(path) def append_curve_by_path(self, path, *args, **kwargs): - if self.path != None: - path = os.path.join(os.path.dirname(self.path), path) + if self._base_path != None: + path = os.path.join(self._base_path, path) super(FilePlaylist, self).append_curve_by_path(path, *args, **kwargs) def is_saved(self): @@ -206,41 +333,20 @@ class FilePlaylist (Playlist): >>> c.info['note'] = 'The second curve' >>> p.append(c) >>> p.digest() - '\\\x14\x87\x88*q\xf8\xaa\xa7\x84f\x82\xa1S>\xfd3+\xd0o' + '\xa1\x1ax\xb1|\x84uA\xe4\x1d\xbf`\x004|\x82\xc2\xdd\xc1\x9e' """ string = self.flatten() return hashlib.sha1(string).digest() - def _clean_key(self, key): - """Replace spaces in keys with \\u00B7 (middle dot). - - This character is deemed unlikely to occur in keys to our - playlist and curve info dictionaries, while many keys have - spaces in them. - - \\u00B7 is allowed in XML 1.0 as of the 5th edition. See - the `4th edition errata`_ for details. - - .. _4th edition errata: - http://www.w3.org/XML/xml-V10-4e-errata#E09 - """ - return key.replace(' ', u'\u00B7') - - def _restore_key(self, key): - """Restore keys encoded with :meth:`_clean_key`. - """ - return key.replace(u'\u00B7', ' ') - - def flatten(self, absolute_paths=False): + def flatten(self): """Create a string representation of the playlist. - A playlist is an XML document with the following syntax:: + A playlist is a YAML document with the following minimal syntax:: - - - - - + version: '0.2' + items: + - path: picoforce.000 + - path: picoforce.001 Relative paths are interpreted relative to the location of the playlist file. @@ -248,6 +354,8 @@ class FilePlaylist (Playlist): Examples -------- + >>> from .engine import CommandMessage + >>> root_path = os.path.sep + 'path' >>> p = FilePlaylist(drivers=[], ... path=os.path.join(root_path, 'to','playlist')) @@ -257,121 +365,140 @@ class FilePlaylist (Playlist): >>> p.append(c) >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'two')) >>> c.info['attr with spaces'] = 'The second curve\\nwith endlines' + >>> c.command_stack.extend([ + ... CommandMessage('command A', {'arg 0':0, 'arg 1':'X'}), + ... CommandMessage('command B', {'arg 0':1, 'arg 1':'Y'}), + ... ]) >>> p.append(c) - >>> def _print(string): - ... escaped_string = unicode(string, 'utf-8').encode('unicode escape') - ... print escaped_string.replace('\\\\n', '\\n').replace('\\\\t', '\\t'), - >>> _print(p.flatten()) # doctest: +NORMALIZE_WHITESPACE +REPORT_UDIFF - - - - - - >>> _print(p.flatten(absolute_paths=True)) # doctest: +NORMALIZE_WHITESPACE +REPORT_UDIFF - - - - - + >>> print p.flatten() # doctest: +REPORT_UDIFF + # Hooke playlist version 0.2 + _base_path: /path/to + _index: 0 + _max_loaded: 100 + _relative_curve_paths: true + drivers: [] + info: {note: An example playlist} + items: + - info: {note: The first curve} + name: one + path: curve/one + - command_stack: + - arguments: {arg 0: 0, arg 1: X} + command: command A + - arguments: {arg 0: 1, arg 1: Y} + command: command B + info: {attr with spaces: 'The second curve + + with endlines'} + name: two + path: curve/two + name: playlist.hkp + path: /path/to/playlist.hkp + version: '0.2' + + >>> p._relative_curve_paths = False + >>> print p.flatten() # doctest: +REPORT_UDIFF + # Hooke playlist version 0.2 + _base_path: /path/to + _index: 0 + _max_loaded: 100 + _relative_curve_paths: false + drivers: [] + info: {note: An example playlist} + items: + - info: {note: The first curve} + name: one + path: /path/to/curve/one + - command_stack: + - arguments: {arg 0: 0, arg 1: X} + command: command A + - arguments: {arg 0: 1, arg 1: Y} + command: command B + info: {attr with spaces: 'The second curve + + with endlines'} + name: two + path: /path/to/curve/two + name: playlist.hkp + path: /path/to/playlist.hkp + version: '0.2' + """ - implementation = xml.dom.minidom.getDOMImplementation() - # create the document DOM object and the root element - doc = implementation.createDocument(None, 'playlist', None) - root = doc.documentElement - root.setAttribute('version', self.version) # store playlist version - root.setAttribute('index', str(self._index)) - for key,value in self.info.items(): # save info variables - if (key in self._ignored_keys - or not isinstance(value, types.StringTypes)): - continue - root.setAttribute(self._clean_key(key), str(value)) - if self.path == None: - base_path = os.getcwd() - else: - base_path = os.path.abspath( - os.path.expanduser(self.path)) - for curve in self: # save curves and their attributes - curve_element = doc.createElement('curve') - root.appendChild(curve_element) - path = os.path.abspath(os.path.expanduser(curve.path)) - if absolute_paths == False: - path = os.path.relpath( - path, - os.path.dirname(base_path)) - curve_element.setAttribute('path', path) - for key,value in curve.info.items(): - if (key in self._ignored_keys - or not isinstance(value, types.StringTypes)): - continue - curve_element.setAttribute(self._clean_key(key), str(value)) - string = doc.toprettyxml(encoding='utf-8') - root.unlink() # break circular references for garbage collection - return string - - def _from_xml_doc(self, doc, identify=True): - """Load a playlist from an :class:`xml.dom.minidom.Document` - instance. - """ - root = doc.documentElement - for attribute,value in root.attributes.items(): - attribute = self._restore_key(attribute) - if attribute == 'version': - assert value == self.version, \ - 'Cannot read v%s playlist with a v%s reader' \ - % (value, self.version) - elif attribute == 'index': - self._index = int(value) - else: - self.info[attribute] = value - for curve_element in doc.getElementsByTagName('curve'): - path = curve_element.getAttribute('path') - info = dict([(self._restore_key(key), value) - for key,value in curve_element.attributes.items()]) - info.pop('path') - self.append_curve_by_path(path, info, identify=identify) - self.jump(self._index) # ensure valid index - - def from_string(self, string, identify=True): + yaml_string = yaml.dump(self.__getstate__(), allow_unicode=True) + return ('# Hooke playlist version %s\n' % self.version) + yaml_string + + def from_string(self, string): u"""Load a playlist from a string. Examples -------- - >>> string = ''' - ... - ... - ... - ... + Minimal example. + + >>> string = '''# Hooke playlist version 0.2 + ... version: '0.2' + ... items: + ... - path: picoforce.000 + ... - path: picoforce.001 + ... ''' + >>> p = FilePlaylist(drivers=[], + ... path=os.path.join('/path', 'to', 'my', 'playlist')) + >>> p.from_string(string) + >>> for curve in p: + ... print curve.path + /path/to/my/picoforce.000 + /path/to/my/picoforce.001 + + More complicated example. + + >>> string = '''# Hooke playlist version 0.2 + ... _base_path: /path/to + ... _digest: null + ... _index: 1 + ... _max_loaded: 100 + ... _relative_curve_paths: true + ... info: {note: An example playlist} + ... items: + ... - info: {note: The first curve} + ... path: curve/one + ... - command_stack: + ... - arguments: {arg 0: 0, arg 1: X} + ... command: command A + ... - arguments: {arg 0: 1, arg 1: Y} + ... command: command B + ... info: {attr with spaces: 'The second curve + ... + ... with endlines'} + ... name: two + ... path: curve/two + ... name: playlist.hkp + ... path: /path/to/playlist.hkp + ... version: '0.2' ... ''' >>> p = FilePlaylist(drivers=[], ... path=os.path.join('path', 'to', 'my', 'playlist')) - >>> p.from_string(string, identify=False) + >>> p.from_string(string) >>> p._index 1 >>> p.info - {u'note': u'An example playlist'} + {'note': 'An example playlist'} >>> for curve in p: - ... print curve.path - path/to/curve/one - path/to/curve/two + ... print curve.name, curve.path + one /path/to/curve/one + two /path/to/curve/two >>> p[-1].info['attr with spaces'] - u'The second curve\\nwith endlines' + 'The second curve\\nwith endlines' + >>> type(p[-1].command_stack) + + >>> p[-1].command_stack # doctest: +NORMALIZE_WHITESPACE + [, + ] """ - doc = xml.dom.minidom.parseString(string) - self._from_xml_doc(doc, identify=identify) - - def load(self, path=None, identify=True, hooke=None): - """Load a playlist from a file. - """ - self.set_path(path) - doc = xml.dom.minidom.parse(self.path) - self._from_xml_doc(doc, identify=identify) - self._digest = self.digest() - for curve in self: - curve.set_hooke(hooke) + state = yaml.load(string) + self.__setstate__(state) def save(self, path=None, makedirs=True): - """Saves the playlist in a XML file. + """Saves the playlist to a YAML file. """ self.set_path(path) dirname = os.path.dirname(self.path) or '.' @@ -380,3 +507,16 @@ class FilePlaylist (Playlist): with open(self.path, 'w') as f: f.write(self.flatten()) self._digest = self.digest() + + def load(self, path=None, identify=True, hooke=None): + """Load a playlist from a file. + """ + self.set_path(path) + with open(self.path, 'r') as f: + text = f.read() + self.from_string(text) + self._digest = self.digest() + for curve in self: + curve.set_hooke(hooke) + if identify == True: + curve.identify(self.drivers) diff --git a/hooke/plugin/note.py b/hooke/plugin/note.py index 0d0bada..b31dfd3 100644 --- a/hooke/plugin/note.py +++ b/hooke/plugin/note.py @@ -85,7 +85,7 @@ class NoteFilterCommand (FilterCommand): """ def __init__(self, plugin): super(NoteFilterCommand, self).__init__( - plugin, name='note filter playlist') + plugin, name='note filter playlist', load_curves=False) def filter(self, curve, hooke, inqueue, outqueue, params): return 'note' in curve.info and curve.info['note'] != None diff --git a/hooke/plugin/playlist.py b/hooke/plugin/playlist.py index fbd30fa..3b1d837 100644 --- a/hooke/plugin/playlist.py +++ b/hooke/plugin/playlist.py @@ -393,9 +393,10 @@ class FilterCommand (PlaylistAddingCommand, PlaylistCommand): method of their subclass. See, for example, :meth:`NoteFilterCommand.filter`. """ - def __init__(self, plugin, name='filter playlist'): + def __init__(self, plugin, name='filter playlist', load_curves=True): super(FilterCommand, self).__init__( name=name, help=self.__doc__, plugin=plugin) + self._load_curves = load_curves if not hasattr(self, 'filter'): self.arguments.append( Argument(name='filter', type='function', optional=False, @@ -409,7 +410,8 @@ Function returning `True` for "good" curves. filter_fn = params['filter'] else: filter_fn = self.filter - p = self._playlist(hooke, params).filter(filter_fn, + p = self._playlist(hooke, params).filter( + filter_fn, load_curves=self._load_curves, hooke=hooke, inqueue=inqueue, outqueue=outqueue, params=params) self._set_playlist(hooke, params, p) if hasattr(p, 'path') and p.path != None: diff --git a/test/curve_info.py b/test/curve_info.py index 9ab0ce6..28b14f3 100644 --- a/test/curve_info.py +++ b/test/curve_info.py @@ -26,11 +26,11 @@ Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: picoforce.000 -path: test/data/picoforce.000 +path: .../test/data/picoforce.000 experiment: driver: filetype: picoforce -note: +note: None blocks: 2 block sizes: [(2048, 2), (2048, 2)] Success diff --git a/test/data/fclamp_hemingway/playlist.hkp b/test/data/fclamp_hemingway/playlist.hkp index dbae686..05966ab 100644 --- a/test/data/fclamp_hemingway/playlist.hkp +++ b/test/data/fclamp_hemingway/playlist.hkp @@ -1,26 +1,27 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - +# Hooke playlist version 0.2 +version: '0.2' +name: Hemingway +items: +- path: 20080428_a53t-0-0-10.dat +- path: 20080428_a53t-0-0-31.dat +- path: 20080428_a53t-0-0-43.dat +- path: 20080428_a53t-0-2-2.dat +- path: 20080428_a53t-0-2-3.dat +- path: 20080428_a53t-0-4-28.dat +- path: 20080428_a53t-0-9-7.dat +- path: 20080428_a53t-1-0-14.dat +- path: 20080428_a53t-1-2-2.dat +- path: 20080428_a53t-1-4-16.dat +- path: 20080428_a53t-1-4-28.dat +- path: 20080428_a53t-1-6-20.dat +- path: 20080428_a53t-1-8-10.dat +- path: 20080428_a53t-1-9-10.dat +- path: 20080428_a53t-1-9-12.dat +- path: 20080428_a53t-1-9-13.dat +- path: 20080428_a53t-1-9-24.dat +- path: fclamp_i27-0-0-42.dat +- path: fclamp_i27-0-1-35.dat +- path: fclamp_i27-1-5-17.dat +- path: fclamp_i27-2-1-4.dat +- path: fclamp_i27-3-4-17.dat +- path: fclamp_i27-5-1-6.dat diff --git a/test/data/test.hkp b/test/data/test.hkp index a88c68b..fb51f08 100644 --- a/test/data/test.hkp +++ b/test/data/test.hkp @@ -1,4 +1,4 @@ - - - - +# Hooke playlist version 0.2 +version: '0.2' +items: +- path: picoforce.000 diff --git a/test/data/vclamp_jpk/playlist.hkp b/test/data/vclamp_jpk/playlist.hkp index 323b20a..7f3a90d 100644 --- a/test/data/vclamp_jpk/playlist.hkp +++ b/test/data/vclamp_jpk/playlist.hkp @@ -1,6 +1,6 @@ - - - - - +# Hooke playlist version 0.2 +version: '0.2' +name: JPK +items: +- path: 2009.04.23-15.15.47.jpk +- path: 2009.04.23-15.21.39.jpk diff --git a/test/data/vclamp_mfp3d/playlist.hkp b/test/data/vclamp_mfp3d/playlist.hkp index b3afa40..3665bc7 100644 --- a/test/data/vclamp_mfp3d/playlist.hkp +++ b/test/data/vclamp_mfp3d/playlist.hkp @@ -1,29 +1,30 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Hooke playlist version 0.2 +version: '0.2' +name: MFP3D +items: +- path: Line0004Point0000.ibw +- path: Line0004Point0001.ibw +- path: Line0004Point0002.ibw +- path: Line0004Point0003.ibw +- path: Line0004Point0004.ibw +- path: Line0004Point0005.ibw +- path: Line0004Point0006.ibw +- path: Line0004Point0007.ibw +- path: Line0004Point0008.ibw +- path: Line0004Point0009.ibw +- path: Line0004Point0010.ibw +- path: Line0004Point0011.ibw +- path: Line0004Point0012.ibw +- path: Line0004Point0013.ibw +- path: Line0004Point0014.ibw +- path: Line0004Point0015.ibw +- path: Line0004Point0016.ibw +- path: Line0004Point0017.ibw +- path: Line0004Point0018.ibw +- path: Line0004Point0019.ibw +- path: Image0238.ibw +- path: Image0255.ibw +- path: Image0360.ibw +- path: Image0365.ibw +- path: Image0386.ibw +- path: Image0396.ibw diff --git a/test/data/vclamp_picoforce/playlist.hkp b/test/data/vclamp_picoforce/playlist.hkp index fb10798..3fe3e11 100644 --- a/test/data/vclamp_picoforce/playlist.hkp +++ b/test/data/vclamp_picoforce/playlist.hkp @@ -1,105 +1,106 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Hooke playlist version 0.2 +version: '0.2' +name: PicoForce +items: +- path: 20071120a_i27_t33.100 +- path: 20071120a_i27_t33.101 +- path: 20071120a_i27_t33.102 +- path: 20071120a_i27_t33.103 +- path: 20071120a_i27_t33.104 +- path: 20071120a_i27_t33.105 +- path: 20071120a_i27_t33.106 +- path: 20071120a_i27_t33.107 +- path: 20071120a_i27_t33.108 +- path: 20071120a_i27_t33.109 +- path: 20071120a_i27_t33.110 +- path: 20071120a_i27_t33.111 +- path: 20071120a_i27_t33.112 +- path: 20071120a_i27_t33.113 +- path: 20071120a_i27_t33.114 +- path: 20071120a_i27_t33.115 +- path: 20071120a_i27_t33.116 +- path: 20071120a_i27_t33.117 +- path: 20071120a_i27_t33.118 +- path: 20071120a_i27_t33.119 +- path: 20071120a_i27_t33.120 +- path: 20071120a_i27_t33.121 +- path: 20071120a_i27_t33.122 +- path: 20071120a_i27_t33.123 +- path: 20071120a_i27_t33.124 +- path: 20071120a_i27_t33.125 +- path: 20071120a_i27_t33.126 +- path: 20071120a_i27_t33.127 +- path: 20071120a_i27_t33.128 +- path: 20071120a_i27_t33.129 +- path: 20071120a_i27_t33.130 +- path: 20071120a_i27_t33.131 +- path: 20071120a_i27_t33.132 +- path: 20071120a_i27_t33.133 +- path: 20071120a_i27_t33.134 +- path: 20071120a_i27_t33.135 +- path: 20071120a_i27_t33.136 +- path: 20071120a_i27_t33.137 +- path: 20071120a_i27_t33.138 +- path: 20071120a_i27_t33.139 +- path: 20071120a_i27_t33.140 +- path: 20071120a_i27_t33.141 +- path: 20071120a_i27_t33.142 +- path: 20071120a_i27_t33.143 +- path: 20071120a_i27_t33.144 +- path: 20071120a_i27_t33.145 +- path: 20071120a_i27_t33.146 +- path: 20071120a_i27_t33.147 +- path: 20071120a_i27_t33.148 +- path: 20071120a_i27_t33.149 +- path: 20071120a_i27_t33.150 +- path: 20071120a_i27_t33.151 +- path: 20071120a_i27_t33.152 +- path: 20071120a_i27_t33.153 +- path: 20071120a_i27_t33.154 +- path: 20071120a_i27_t33.155 +- path: 20071120a_i27_t33.156 +- path: 20071120a_i27_t33.157 +- path: 20071120a_i27_t33.158 +- path: 20071120a_i27_t33.159 +- path: 20071120a_i27_t33.160 +- path: 20071120a_i27_t33.161 +- path: 20071120a_i27_t33.162 +- path: 20071120a_i27_t33.163 +- path: 20071120a_i27_t33.164 +- path: 20071120a_i27_t33.165 +- path: 20071120a_i27_t33.166 +- path: 20071120a_i27_t33.167 +- path: 20071120a_i27_t33.168 +- path: 20071120a_i27_t33.169 +- path: 20071120a_i27_t33.170 +- path: 20071120a_i27_t33.171 +- path: 20071120a_i27_t33.172 +- path: 20071120a_i27_t33.173 +- path: 20071120a_i27_t33.174 +- path: 20071120a_i27_t33.175 +- path: 20071120a_i27_t33.176 +- path: 20071120a_i27_t33.177 +- path: 20071120a_i27_t33.178 +- path: 20071120a_i27_t33.179 +- path: 20071120a_i27_t33.180 +- path: 20071120a_i27_t33.181 +- path: 20071120a_i27_t33.182 +- path: 20071120a_i27_t33.183 +- path: 20071120a_i27_t33.184 +- path: 20071120a_i27_t33.185 +- path: 20071120a_i27_t33.186 +- path: 20071120a_i27_t33.187 +- path: 20071120a_i27_t33.188 +- path: 20071120a_i27_t33.189 +- path: 20071120a_i27_t33.190 +- path: 20071120a_i27_t33.191 +- path: 20071120a_i27_t33.192 +- path: 20071120a_i27_t33.193 +- path: 20071120a_i27_t33.194 +- path: 20071120a_i27_t33.195 +- path: 20071120a_i27_t33.196 +- path: 20071120a_i27_t33.197 +- path: 20071120a_i27_t33.198 +- path: 20071120a_i27_t33.199 +- path: '0x06130001' +- path: '0x07200000' diff --git a/test/data/vclamp_wtk/playlist.hkp b/test/data/vclamp_wtk/playlist.hkp index b579722..766a363 100644 --- a/test/data/vclamp_wtk/playlist.hkp +++ b/test/data/vclamp_wtk/playlist.hkp @@ -1,33 +1,34 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Hooke playlist version 0.2 +version: '0.2' +name: WTK +items: +- path: unfold/20100504/20100504144209_unfold +- path: unfold/20100504/20100504144221_unfold +- path: unfold/20100504/20100504144231_unfold +- path: unfold/20100504/20100504144239_unfold +- path: unfold/20100504/20100504144246_unfold +- path: unfold/20100504/20100504144252_unfold +- path: unfold/20100504/20100504144258_unfold +- path: unfold/20100504/20100504144304_unfold +- path: unfold/20100504/20100504144310_unfold +- path: unfold/20100504/20100504144335_unfold +- path: unfold/20100504/20100504144353_unfold +- path: unfold/20100504/20100504144405_unfold +- path: unfold/20100504/20100504144415_unfold +- path: unfold/20100504/20100504144423_unfold +- path: unfold/20100504/20100504144430_unfold +- path: unfold/20100504/20100504144436_unfold +- path: unfold/20100504/20100504144442_unfold +- path: unfold/20100504/20100504144448_unfold +- path: unfold/20100504/20100504144454_unfold +- path: unfold/20100504/20100504144519_unfold +- path: unfold/20100504/20100504144537_unfold +- path: unfold/20100504/20100504144549_unfold +- path: unfold/20100504/20100504144559_unfold +- path: unfold/20100504/20100504144607_unfold +- path: unfold/20100504/20100504144614_unfold +- path: unfold/20100504/20100504144620_unfold +- path: unfold/20100504/20100504144626_unfold +- path: unfold/20100504/20100504144632_unfold +- path: unfold/20100504/20100504144638_unfold +- path: unfold/20100504/20100504144703_unfold diff --git a/test/flat_filter_playlist.py b/test/flat_filter_playlist.py index dfd94a1..d84f19c 100644 --- a/test/flat_filter_playlist.py +++ b/test/flat_filter_playlist.py @@ -21,7 +21,7 @@ >>> h = Hooke() >>> r = HookeRunner() >>> h = r.run_lines(h, ['load_playlist test/data/vclamp_picoforce/playlist']) # doctest: +ELLIPSIS - + Success >>> h = r.run_lines(h, ['flat_filter_playlist --distance_column "z piezo (m)" --deflection_column "deflection (m)"']) # doctest: +ELLIPSIS diff --git a/test/hemingway_driver.py b/test/hemingway_driver.py index 374af16..3ac273b 100644 --- a/test/hemingway_driver.py +++ b/test/hemingway_driver.py @@ -23,16 +23,16 @@ >>> r = HookeRunner() >>> playlist = os.path.join('test', 'data', 'fclamp_hemingway', 'playlist') >>> h = r.run_lines(h, ['load_playlist ' + playlist]) # doctest: +ELLIPSIS - + Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: 20080428_a53t-0-0-10.dat -path: test/data/fclamp_hemingway/20080428_a53t-0-0-10.dat +path: .../test/data/fclamp_hemingway/20080428_a53t-0-0-10.dat experiment: driver: filetype: hemingway -note: +note: None blocks: 1 block sizes: [(14798, 5)] Success diff --git a/test/jpk_driver.py b/test/jpk_driver.py index 2515273..8261c7e 100644 --- a/test/jpk_driver.py +++ b/test/jpk_driver.py @@ -23,35 +23,18 @@ >>> r = HookeRunner() >>> playlist = os.path.join('test', 'data', 'vclamp_jpk', 'playlist') >>> h = r.run_lines(h, ['load_playlist ' + playlist]) # doctest: +ELLIPSIS - + Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: 2009.04.23-15.15.47.jpk -path: test/data/vclamp_jpk/2009.04.23-15.15.47.jpk -experiment: None +path: .../test/data/vclamp_jpk/2009.04.23-15.15.47.jpk +experiment: driver: -filetype: None -note: +filetype: jpk +note: None blocks: 2 block sizes: [(4096, 6), (4096, 4)] Success - -Load the second curve, to test calibration file overriding. - ->>> h = r.run_lines(h, ['next_curve']) -Success - ->>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF -name: 2009.04.23-15.21.39.jpk -path: test/data/vclamp_jpk/2009.04.23-15.21.39.jpk -experiment: None -driver: -filetype: None -note: -blocks: 3 -block sizes: [(4096, 6), (2048, 3), (4096, 4)] -Success - """ diff --git a/test/mfp3d_driver.py b/test/mfp3d_driver.py index ba9a3b8..cefb851 100644 --- a/test/mfp3d_driver.py +++ b/test/mfp3d_driver.py @@ -23,35 +23,35 @@ >>> r = HookeRunner() >>> playlist = os.path.join('test', 'data', 'vclamp_mfp3d', 'playlist') >>> h = r.run_lines(h, ['load_playlist ' + playlist]) - + Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: Line0004Point0000.ibw -path: test/data/vclamp_mfp3d/Line0004Point0000.ibw +path: .../test/data/vclamp_mfp3d/Line0004Point0000.ibw experiment: driver: filetype: mfp3d -note: -blocks: 2 -block sizes: [(1091, 2), (0, 2)] +note: None +blocks: 3 +block sizes: [(491, 2), (497, 2), (105, 2)] Success -Also checkout a newer Image* file +Also checkout a newer Image* file. >>> h = r.run_lines(h, ['previous_curve']) Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: Image0396.ibw -path: test/data/vclamp_mfp3d/Image0396.ibw +path: .../test/data/vclamp_mfp3d/Image0396.ibw experiment: driver: filetype: mfp3d -note: -blocks: 2 -block sizes: [(1006, 3), (0, 3)] +note: None +blocks: 3 +block sizes: [(506, 3), (6, 3), (496, 3)] Success """ diff --git a/test/multiple_curve_analysis.py b/test/multiple_curve_analysis.py index d7c82c6..dd5d126 100644 --- a/test/multiple_curve_analysis.py +++ b/test/multiple_curve_analysis.py @@ -24,7 +24,7 @@ Setup a playlist to act on. >>> h = r.run_lines(h, ['load_playlist test/data/vclamp_picoforce/playlist']) - + Success diff --git a/test/picoforce_driver.py b/test/picoforce_driver.py index ffe823c..d36aa18 100644 --- a/test/picoforce_driver.py +++ b/test/picoforce_driver.py @@ -23,16 +23,16 @@ >>> r = HookeRunner() >>> playlist = os.path.join('test', 'data', 'vclamp_picoforce', 'playlist') >>> h = r.run_lines(h, ['load_playlist ' + playlist]) - + Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: 20071120a_i27_t33.100 -path: test/data/vclamp_picoforce/20071120a_i27_t33.100 +path: .../test/data/vclamp_picoforce/20071120a_i27_t33.100 experiment: driver: filetype: picoforce -note: +note: None blocks: 2 block sizes: [(2048, 2), (2048, 2)] Success @@ -45,11 +45,11 @@ Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: 0x07200000 -path: test/data/vclamp_picoforce/0x07200000 +path: .../test/data/vclamp_picoforce/0x07200000 experiment: driver: filetype: picoforce -note: +note: None blocks: 2 block sizes: [(512, 2), (512, 2)] Success @@ -59,11 +59,11 @@ Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: 0x06130001 -path: test/data/vclamp_picoforce/0x06130001 +path: .../test/data/vclamp_picoforce/0x06130001 experiment: driver: filetype: picoforce -note: +note: None blocks: 2 block sizes: [(2048, 2), (2048, 2)] Success diff --git a/test/polymer_fit.py b/test/polymer_fit.py index fe5ccc7..c20ecd7 100644 --- a/test/polymer_fit.py +++ b/test/polymer_fit.py @@ -84,11 +84,11 @@ Data([[ NaN, NaN], [ NaN, NaN], [ NaN, NaN]]) >>> retract[1097:1103,-2:] # doctest: +ELLIPSIS -Data([[ NaN, 5.234...e-10], - [ NaN, 5.612...e-10], - [ NaN, 6.132...e-10], - [ NaN, 6.292...e-10], - [ NaN, 7.105...e-10], +Data([[ NaN, 5.2...e-10], + [ NaN, 5...e-10], + [ NaN, 6.1...e-10], + [ NaN, 6.2...e-10], + [ NaN, 7...e-10], [ NaN, NaN]]) >>> retract[-5:,-2:] Data([[ NaN, NaN], diff --git a/test/wtk_driver.py b/test/wtk_driver.py index 0953a3a..f34907c 100644 --- a/test/wtk_driver.py +++ b/test/wtk_driver.py @@ -38,16 +38,16 @@ Proceed with the test itself. >>> r = HookeRunner() >>> playlist = os.path.join('test', 'data', 'vclamp_wtk', 'playlist') >>> h = r.run_lines(h, ['load_playlist ' + playlist]) - + Success >>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF name: 20100504144209_unfold -path: test/data/vclamp_wtk/unfold/20100504/20100504144209_unfold +path: .../test/data/vclamp_wtk/unfold/20100504/20100504144209_unfold experiment: driver: filetype: wtk -note: +note: None blocks: 2 block sizes: [(810, 2), (8001, 2)] Success -- 2.26.2