Adjust Hooke internals to use unsafe YAML (not just builtin Python types).

Pros:
* More explicit YAML.
* Simple, robust code.
* Easily (de)serialize everything, regardless of nesting, etc.
Cons:
* Uglier YAML.

After wrestling with a yaml.safe_dump()able representation, I've given
up and shifted to yaml.dump().  Now we can dump anything easily, which
is great :).  For example, this will allow you to access polymer fit
parameters.  However, there will be things like
  !!python/object/new:hooke.playlist.FilePlaylist
in the output YAML files.  Such is life.

While I was at it, I ran
  $ nosetests --with-doctest --doctest-tests hooke/ 2>&1 | log
and cleaned up a number of the doctests.  Now the only things that
fail in there are the yet-to-be-upgraded submods left over from before
my rewrite.

I haven't checked test/ recently, that's next ;).

diff --git a/hooke/command.py b/hooke/command.py
index d88155c7a69cf1dc6b414da28ab683e32ead828b..f685327ec4be1fef6f71073df5215f49a85e7de9 100644 (file)
--- a/hooke/command.py
+++ b/hooke/command.py
@@ -94,6 +94,7 @@ class Command (object):
     Arguments:
     <BLANKLINE>
     help BOOL (bool) Print a help message.
     Arguments:
     <BLANKLINE>
     help BOOL (bool) Print a help message.

+    stack BOOL (bool) Add this command to appropriate command stacks.

     <BLANKLINE>
     An example Command.
     ITEM:
     <BLANKLINE>
     An example Command.
     ITEM:

diff --git a/hooke/command_stack.py b/hooke/command_stack.py
index 319d354fe6dc103dfa323a866e9acda50c9e2585..c6a4972175b382a350eb1d9e1a891f646a2462f7 100644 (file)
--- a/hooke/command_stack.py
+++ b/hooke/command_stack.py
@@ -173,13 +173,7 @@ class CommandStack (list):
     listitems:
     - !!python/object:hooke.engine.CommandMessage
       arguments:
     listitems:
     - !!python/object:hooke.engine.CommandMessage
       arguments:

-        curve: !!python/object:hooke.curve.Curve
-          command_stack: !!python/object:hooke.command_stack.CommandStack {}
-          data: null
-          driver: null
-          info: {}
-          name: null
-          path: null
+        curve: !!python/object:hooke.curve.Curve {}

       command: curve info
     <BLANKLINE>
     """
       command: curve info
     <BLANKLINE>
     """

diff --git a/hooke/config.py b/hooke/config.py
index 5948974f101c4ef647612f121fb3c6c69cd34af6..a8d006dc53ea69e288083d944d6a242947849f9f 100644 (file)
--- a/hooke/config.py
+++ b/hooke/config.py
@@ -137,7 +137,8 @@ class HookeConfigParser (configparser.RawConfigParser):
     >>> c = HookeConfigParser(default_settings=DEFAULT_SETTINGS)
     >>> c.write(sys.stdout) # doctest: +ELLIPSIS
     # Default environmental conditions in case they are not specified in
     >>> c = HookeConfigParser(default_settings=DEFAULT_SETTINGS)
     >>> c.write(sys.stdout) # doctest: +ELLIPSIS
     # Default environmental conditions in case they are not specified in

-    # the force curve data.
+    # the force curve data.  Configuration options in this section are
+    # available to every plugin.

     [conditions]
     # Temperature in Kelvin
     temperature = 301
     [conditions]
     # Temperature in Kelvin
     temperature = 301


@@ -161,7 +162,7 @@ class HookeConfigParser (configparser.RawConfigParser):
     ...         Setting(section, option='my float', value=3.14159, type='float'),
     ...         ])
     >>> pprint.pprint(c.items(section))  # doctest: +ELLIPSIS
     ...         Setting(section, option='my float', value=3.14159, type='float'),
     ...         ])
     >>> pprint.pprint(c.items(section))  # doctest: +ELLIPSIS

-    [('my string', 'Lorem ipsum'),
+    [('my string', u'Lorem ipsum'),

      ('my bool', True),
      ('my int', 13),
      ('my float', 3.1415...)]
      ('my bool', True),
      ('my int', 13),
      ('my float', 3.1415...)]


@@ -170,7 +171,7 @@ class HookeConfigParser (configparser.RawConfigParser):
     to use the standard `.get*()` methods.
 
     >>> c.get('test conversion', 'my bool')
     to use the standard `.get*()` methods.
 
     >>> c.get('test conversion', 'my bool')

-    'True'
+    u'True'

     >>> c.getboolean('test conversion', 'my bool')
     True
     """
     >>> c.getboolean('test conversion', 'my bool')
     True
     """

diff --git a/hooke/curve.py b/hooke/curve.py
index 12ac5eb98a2cb1cd2339d36d6edb69c16f9e5c82..42fa8079d44c90533b2964a98e3166bc86eb7baf 100644 (file)
--- a/hooke/curve.py
+++ b/hooke/curve.py
@@ -92,8 +92,8 @@ class Data (numpy.ndarray):
 
     >>> import yaml
     >>> print yaml.dump(d)
 
     >>> import yaml
     >>> print yaml.dump(d)

-    null
-    ...
+    !hooke.curve.DataInfo
+    columns: [distance (m), force (N)]

     <BLANKLINE>
     """
     def __new__(subtype, shape, dtype=numpy.float, buffer=None, offset=0,
     <BLANKLINE>
     """
     def __new__(subtype, shape, dtype=numpy.float, buffer=None, offset=0,


@@ -189,6 +189,8 @@ class Curve (object):
     >>> z = pickle.loads(s)
     >>> z
     <Curve path>
     >>> z = pickle.loads(s)
     >>> z
     <Curve path>

+    >>> z.command_stack
+    [<CommandMessage curve info {curve: <Curve path>}>]

     >>> z.command_stack[-1].arguments['curve'] == z
     True
     >>> print yaml.dump(c)  # doctest: +REPORT_UDIFF
     >>> z.command_stack[-1].arguments['curve'] == z
     True
     >>> print yaml.dump(c)  # doctest: +REPORT_UDIFF


@@ -199,9 +201,6 @@ class Curve (object):
         arguments:
           curve: *id001
         command: curve info
         arguments:
           curve: *id001
         command: curve info

-    data: null
-    driver: null
-    info: {}

     name: path
     path: some/path
     <BLANKLINE>
     name: path
     path: some/path
     <BLANKLINE>


@@ -226,9 +225,6 @@ class Curve (object):
       arguments:
         curve: !!python/object:hooke.curve.Curve
           command_stack: *id001
       arguments:
         curve: !!python/object:hooke.curve.Curve
           command_stack: *id001

-          data: null
-          driver: null
-          info: {}

           name: path
           path: some/path
       command: curve info
           name: path
           path: some/path
       command: curve info


@@ -251,22 +247,35 @@ class Curve (object):
         if self.name == None and path != None:
             self.name = os.path.basename(path)
 
         if self.name == None and path != None:
             self.name = os.path.basename(path)
 

+    def _setup_default_attrs(self):
+        # .data contains: {name of data: list of data sets [{[x], [y]}]
+        # ._hooke contains a Hooke instance for Curve.load()
+        self._default_attrs = {
+            '_hooke': None,
+            'command_stack': [],
+            'data': None,
+            'driver': None,
+            'info': {},
+            'name': None,
+            'path': None,
+            }
+

     def __getstate__(self):
     def __getstate__(self):

-        state = dict(self.__dict__)      # make a copy of the attribute dict.
+        state = dict(self.__dict__)  # make a copy of the attribute dict.

         del(state['_hooke'])
         return state
 
     def __setstate__(self, state):
         del(state['_hooke'])
         return state
 
     def __setstate__(self, state):

-        # .data contains: {name of data: list of data sets [{[x], [y]}]
-        # ._hooke contains a Hooke instance for Curve.load()
-        self.name = self.driver = self.data = self._hooke = None
-        self.info = {}
-        self.command_stack = CommandStack()
-        for key,value in state.items():
-            setattr(self, key, value)
-        if self.info == None:
+        self._setup_default_attrs()
+        self.__dict__.update(self._default_attrs)
+        if state == True:
+            return
+        self.__dict__.update(state)
+        self.set_path(getattr(self, 'path', None))
+        if self.info in [None, {}]:

             self.info = {}
             self.info = {}

-        self.set_path(state.get('path', None))
+        if type(self.command_stack) == list:
+            self.command_stack = CommandStack()

 
     def set_hooke(self, hooke=None):
         if hooke != None:
 
     def set_hooke(self, hooke=None):
         if hooke != None:

diff --git a/hooke/playlist.py b/hooke/playlist.py
index d6018d08c39e54b0dda1b1cadb025bfa28102860..fd7ac4bf81107795bb3811f2ee36877b6b7b17c4 100644 (file)
--- a/hooke/playlist.py
+++ b/hooke/playlist.py
@@ -30,7 +30,8 @@ import types
 import yaml
 from yaml.representer import RepresenterError
 
 import yaml
 from yaml.representer import RepresenterError
 

-from . import curve as curve
+from .command_stack import CommandStack
+from .curve import Curve

 from .util.itertools import reverse_enumerate
 
 
 from .util.itertools import reverse_enumerate
 
 


@@ -43,10 +44,8 @@ class NoteIndexList (list):
     """
     def __init__(self, name=None):
         super(NoteIndexList, self).__init__()
     """
     def __init__(self, name=None):
         super(NoteIndexList, self).__init__()

-        self.name = name
-        self.info = {}
-        self._index = 0
-        self._set_ignored_attrs()
+        self._set_default_attrs()
+        self.__setstate__({'name': name})

 
     def __str__(self):
         return str(self.__unicode__())
 
     def __str__(self):
         return str(self.__unicode__())


@@ -57,67 +56,33 @@ class NoteIndexList (list):
     def __repr__(self):
         return self.__str__()
 
     def __repr__(self):
         return self.__str__()
 

-    def _set_ignored_attrs(self):
-        self._ignored_attrs = ['_ignored_attrs', '_default_attrs']
+    def _set_default_attrs(self):

         self._default_attrs = {
             'info': {},
         self._default_attrs = {
             'info': {},

+            'name': None,
+            '_index': 0,

             }
 
     def __getstate__(self):
             }
 
     def __getstate__(self):

-        state = dict(self.__dict__)
-        for key in self._ignored_attrs:
-            if key in state:
-                del(state[key])
-        for key,value in self._default_attrs.items():
-            if key in state and state[key] == value:
-                del(state[key])
-        assert 'items' not in state
-        state['items'] = []
-        self._assert_clean_state(self, state)
-        for item in self:  # save curves and their attributes
-            item_state = self._item_getstate(item)
-            self._assert_clean_state(item, item_state)
-            state['items'].append(item_state)
-        return state
+        return self.__dict__.copy()

 
     def __setstate__(self, state):
 
     def __setstate__(self, state):

-        self._set_ignored_attrs()
-        self.clear()
-        for key,value in self._default_attrs.items():
-            setattr(self, key, value)
-        for key,value in state.items():
-            if key == 'items':
-                continue
-            setattr(self, key, value)
-        for item_state in state['items']:
-            self.append(self._item_setstate(item_state))
-
-    def _item_getstate(self, item):
-        return item
-
-    def _item_setstate(self, state):
-        return state
-
-    def _assert_clean_state(self, owner, state):
-        for k,v in state.items():
-            if k == 'drivers':  # HACK.  Need better driver serialization.
-                continue
-            try:
-                yaml.dump((k,v))
-            except RepresenterError, e:
-                raise NotImplementedError(
-                    'cannot convert %s.%s = %s (%s) to YAML\n%s'
-                    % (owner.__class__.__name__, k, v, type(v), e))
+        self._set_default_attrs()
+        if state == True:
+            return
+        self.__dict__.update(self._default_attrs)
+        try:
+            self.__dict__.update(state)
+        except TypeError, e:
+            print state, type(state), e
+        if self.info in [None, {}]:
+            self.info = {}

 
     def _setup_item(self, item):
         """Perform any required initialization before returning an item.
         """
         pass
 
 
     def _setup_item(self, item):
         """Perform any required initialization before returning an item.
         """
         pass
 

-    def clear(self):
-        while len(self) > 0:
-            self.pop()
-

     def index(self, value=None, *args, **kwargs):
         """Extend `list.index`, returning the current index if `value`
         is `None`.
     def index(self, value=None, *args, **kwargs):
         """Extend `list.index`, returning the current index if `value`
         is `None`.


@@ -188,7 +153,7 @@ class NoteIndexList (list):
 
 
 class Playlist (NoteIndexList):
 
 
 class Playlist (NoteIndexList):

-    """A :class:`NoteIndexList` of :class:`hooke.curve.Curve`\s.
+    """A :class:`NoteIndexList` of :class:`hooke.Curve`\s.

 
     Keeps a list of :attr:`drivers` for loading curves.
     """
 
     Keeps a list of :attr:`drivers` for loading curves.
     """


@@ -197,43 +162,26 @@ class Playlist (NoteIndexList):
         self.drivers = drivers
         self._max_loaded = 100 # curves to hold in memory simultaneously.
 
         self.drivers = drivers
         self._max_loaded = 100 # curves to hold in memory simultaneously.
 

-    def _set_ignored_attrs(self):
-        super(Playlist, self)._set_ignored_attrs()
-        self._ignored_attrs.extend([
-                '_item_ignored_attrs', '_item_default_attrs',
-                '_loaded'])
-        self._item_ignored_attrs = []
-        self._item_default_attrs = {
-            'command_stack': [],
-            'data': None,
-            'driver': None,
-            'info': {},
-            'name': None,
-            }
-        self._loaded = [] # List of loaded curves, see :meth:`._setup_item`.
-
-    def _item_getstate(self, item):
-        assert isinstance(item, curve.Curve), type(item)
-        state = item.__getstate__()
-        for key in self._item_ignored_attrs:
-            if key in state:
-                del(state[key])
-        for key,value in self._item_default_attrs.items():
-            if key in state and state[key] == value:
-                del(state[key])
-        return state
+    def _set_default_attrs(self):
+        super(Playlist, self)._set_default_attrs()
+        self._default_attrs['drivers'] = []
+        # List of loaded curves, see :meth:`._setup_item`.
+        self._default_attrs['_loaded'] = []
+        self._default_attrs['_max_loaded'] = 100

 
 

-    def _item_setstate(self, state):
-        for key,value in self._item_default_attrs.items():
-            if key not in state:
-                state[key] = value
-        item = curve.Curve(path=None)
-        item.__setstate__(state)
-        return item
+    def __setstate__(self, state):
+        super(Playlist, self).__setstate__(state)
+        if self.drivers in [None, {}]:
+            self.drivers = []
+        if self._loaded in [None, {}]:
+            self._loaded = []
+
+    def append_curve(self, curve):
+        self.append(curve)

 
     def append_curve_by_path(self, path, info=None, identify=True, hooke=None):
         path = os.path.normpath(path)
 
     def append_curve_by_path(self, path, info=None, identify=True, hooke=None):
         path = os.path.normpath(path)

-        c = curve.Curve(path, info=info)
+        c = Curve(path, info=info)

         c.set_hooke(hooke)
         if identify == True:
             c.identify(self.drivers)
         c.set_hooke(hooke)
         if identify == True:
             c.identify(self.drivers)


@@ -256,6 +204,37 @@ class Playlist (NoteIndexList):
 
 class FilePlaylist (Playlist):
     """A file-backed :class:`Playlist`.
 
 class FilePlaylist (Playlist):
     """A file-backed :class:`Playlist`.

+
+    Examples
+    --------
+
+    >>> p = FilePlaylist(drivers=['Driver A', 'Driver B'])
+    >>> p.append(Curve('dummy/path/A'))
+    >>> p.append(Curve('dummy/path/B'))
+
+    The data-type is pickleable, to ensure we can move it between
+    processes with :class:`multiprocessing.Queue`\s.
+
+    >>> import pickle
+    >>> s = pickle.dumps(p)
+    >>> z = pickle.loads(s)
+    >>> for curve in z:
+    ...     print curve
+    <Curve A>
+    <Curve B>
+    >>> print z.drivers
+    ['Driver A', 'Driver B']
+
+    The data-type is also YAMLable (see :mod:`hooke.util.yaml`).
+
+    >>> s = yaml.dump(p)
+    >>> z = yaml.load(s)
+    >>> for curve in z:
+    ...     print curve
+    <Curve A>
+    <Curve B>
+    >>> print z.drivers
+    ['Driver A', 'Driver B']

     """
     version = '0.2'
 
     """
     version = '0.2'
 


@@ -263,12 +242,14 @@ class FilePlaylist (Playlist):
         super(FilePlaylist, self).__init__(drivers, name)
         self.path = self._base_path = None
         self.set_path(path)
         super(FilePlaylist, self).__init__(drivers, name)
         self.path = self._base_path = None
         self.set_path(path)

-        self._relative_curve_paths = True
+        self.relative_curve_paths = True
+        self._relative_curve_paths = False

 
 

-    def _set_ignored_attrs(self):
-        super(FilePlaylist, self)._set_ignored_attrs()
-        self._ignored_attrs.append('_digest')
-        self._digest = None
+    def _set_default_attrs(self):
+        super(FilePlaylist, self)._set_default_attrs()
+        self._default_attrs['relative_curve_paths'] = True
+        self._default_attrs['_relative_curve_paths'] = False
+        self._default_attrs['_digest'] = None

 
     def __getstate__(self):
         state = super(FilePlaylist, self).__getstate__()
 
     def __getstate__(self):
         state = super(FilePlaylist, self).__getstate__()


@@ -277,30 +258,16 @@ class FilePlaylist (Playlist):
         return state
 
     def __setstate__(self, state):
         return state
 
     def __setstate__(self, state):

-        assert('version') in state, state
-        version = state.pop('version')
-        assert version == FilePlaylist.version, (
-            'invalid version %s (%s) != %s (%s)'
-            % (version, type(version),
-               FilePlaylist.version, type(FilePlaylist.version)))
+        if 'version' in state:
+            version = state.pop('version')
+            assert version == FilePlaylist.version, (
+                'invalid version %s (%s) != %s (%s)'
+                % (version, type(version),
+                   FilePlaylist.version, type(FilePlaylist.version)))

         super(FilePlaylist, self).__setstate__(state)
 
         super(FilePlaylist, self).__setstate__(state)
 

-    def _item_getstate(self, item):
-        state = super(FilePlaylist, self)._item_getstate(item)
-        if state.get('path', None) != None:
-            path = os.path.abspath(os.path.expanduser(state['path']))
-            if self._relative_curve_paths == True:
-                path = os.path.relpath(path, self._base_path)
-            state['path'] = path
-        return state
-
-    def _item_setstate(self, state):
-        item = super(FilePlaylist, self)._item_setstate(state)
-        if 'path' in state:
-            item.set_path(os.path.join(self._base_path, state['path']))
-        return item
-

     def set_path(self, path):
     def set_path(self, path):

+        orig_base_path = getattr(self, '_base_path', None)

         if path == None:
             if self._base_path == None:
                 self._base_path = os.getcwd()
         if path == None:
             if self._base_path == None:
                 self._base_path = os.getcwd()


@@ -312,10 +279,27 @@ class FilePlaylist (Playlist):
                 os.path.expanduser(self.path)))
             if self.name == None:
                 self.name = os.path.basename(path)
                 os.path.expanduser(self.path)))
             if self.name == None:
                 self.name = os.path.basename(path)

+        if self._base_path != orig_base_path:
+            self.update_curve_paths()
+
+    def update_curve_paths(self):
+        for curve in self:
+            curve.set_path(self._curve_path(curve.path))
+
+    def _curve_path(self, path):
+        if self._base_path == None:
+            self._base_path = os.getcwd()
+        path = os.path.join(self._base_path, path)
+        if self._relative_curve_paths == True:
+            path = os.path.relpath(path, self._base_path)
+        return path
+
+    def append_curve(self, curve):
+        curve.set_path(self._curve_path(curve.path))
+        super(FilePlaylist, self).append_curve(curve)

 
     def append_curve_by_path(self, path, *args, **kwargs):
 
     def append_curve_by_path(self, path, *args, **kwargs):

-        if self._base_path != None:
-            path = os.path.join(self._base_path, path)
+        path = self._curve_path(path)

         super(FilePlaylist, self).append_curve_by_path(path, *args, **kwargs)
 
     def is_saved(self):
         super(FilePlaylist, self).append_curve_by_path(path, *args, **kwargs)
 
     def is_saved(self):


@@ -332,14 +316,14 @@ class FilePlaylist (Playlist):
         >>> p = FilePlaylist(drivers=[],
         ...                  path=os.path.join(root_path, 'to','playlist'))
         >>> p.info['note'] = 'An example playlist'
         >>> p = FilePlaylist(drivers=[],
         ...                  path=os.path.join(root_path, 'to','playlist'))
         >>> p.info['note'] = 'An example playlist'

-        >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'one'))
+        >>> c = Curve(os.path.join(root_path, 'to', 'curve', 'one'))

         >>> c.info['note'] = 'The first curve'
         >>> c.info['note'] = 'The first curve'

-        >>> p.append(c)
-        >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'two'))
+        >>> p.append_curve(c)
+        >>> c = Curve(os.path.join(root_path, 'to', 'curve', 'two'))

         >>> c.info['note'] = 'The second curve'
         >>> c.info['note'] = 'The second curve'

-        >>> p.append(c)
+        >>> p.append_curve(c)

         >>> p.digest()
         >>> p.digest()

-        '\xa1\x1ax\xb1|\x84uA\xe4\x1d\xbf`\x004|\x82\xc2\xdd\xc1\x9e'
+        'f\xe26i\xb98i\x1f\xb61J7:\xf2\x8e\x1d\xde\xc3}g'

         """
         string = self.flatten()
         return hashlib.sha1(string).digest()
         """
         string = self.flatten()
         return hashlib.sha1(string).digest()


@@ -349,10 +333,14 @@ class FilePlaylist (Playlist):
 
         A playlist is a YAML document with the following minimal syntax::
 
 
         A playlist is a YAML document with the following minimal syntax::
 

-            version: '0.2'
-            items:
-            - path: picoforce.000
-            - path: picoforce.001
+            !!python/object/new:hooke.playlist.FilePlaylist
+            state:
+              version: '0.2'
+            listitems:
+            - !!python/object:hooke.curve.Curve
+              path: /path/to/curve/one
+            - !!python/object:hooke.curve.Curve
+              path: /path/to/curve/two

 
         Relative paths are interpreted relative to the location of the
         playlist file.
 
         Relative paths are interpreted relative to the location of the
         playlist file.


@@ -366,151 +354,89 @@ class FilePlaylist (Playlist):
         >>> p = FilePlaylist(drivers=[],
         ...                  path=os.path.join(root_path, 'to','playlist'))
         >>> p.info['note'] = 'An example playlist'
         >>> p = FilePlaylist(drivers=[],
         ...                  path=os.path.join(root_path, 'to','playlist'))
         >>> p.info['note'] = 'An example playlist'

-        >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'one'))
+        >>> c = Curve(os.path.join(root_path, 'to', 'curve', 'one'))

         >>> c.info['note'] = 'The first curve'
         >>> c.info['note'] = 'The first curve'

-        >>> p.append(c)
-        >>> c = curve.Curve(os.path.join(root_path, 'to', 'curve', 'two'))
+        >>> p.append_curve(c)
+        >>> c = Curve(os.path.join(root_path, 'to', 'curve', 'two'))

         >>> c.info['attr with spaces'] = 'The second curve\\nwith endlines'
         >>> c.command_stack.extend([
         ...         CommandMessage('command A', {'arg 0':0, 'arg 1':'X'}),
         >>> c.info['attr with spaces'] = 'The second curve\\nwith endlines'
         >>> c.command_stack.extend([
         ...         CommandMessage('command A', {'arg 0':0, 'arg 1':'X'}),

-        ...         CommandMessage('command B', {'arg 0':1, 'arg 1':'Y'}),
+        ...         CommandMessage('command B', {'arg 0':1, 'curve':c}),

         ...         ])
         ...         ])

-        >>> p.append(c)
+        >>> p.append_curve(c)

         >>> print p.flatten()  # doctest: +REPORT_UDIFF
         # Hooke playlist version 0.2
         >>> print p.flatten()  # doctest: +REPORT_UDIFF
         # Hooke playlist version 0.2

-        _base_path: /path/to
-        _index: 0
-        _max_loaded: 100
-        _relative_curve_paths: true
-        drivers: []
-        info: {note: An example playlist}
-        items:
-        - info: {note: The first curve}
+        !!python/object/new:hooke.playlist.FilePlaylist
+        listitems:
+        - !!python/object:hooke.curve.Curve
+          info: {note: The first curve}

           name: one
           path: curve/one
           name: one
           path: curve/one

-        - command_stack: !!python/object/new:hooke.command_stack.CommandStack
+        - &id001 !!python/object:hooke.curve.Curve
+          command_stack: !!python/object/new:hooke.command_stack.CommandStack

             listitems:
             - !!python/object:hooke.engine.CommandMessage
               arguments: {arg 0: 0, arg 1: X}
               command: command A
             - !!python/object:hooke.engine.CommandMessage
             listitems:
             - !!python/object:hooke.engine.CommandMessage
               arguments: {arg 0: 0, arg 1: X}
               command: command A
             - !!python/object:hooke.engine.CommandMessage

-              arguments: {arg 0: 1, arg 1: Y}
+              arguments:
+                arg 0: 1
+                curve: *id001

               command: command B
           info: {attr with spaces: 'The second curve
         <BLANKLINE>
               with endlines'}
           name: two
           path: curve/two
               command: command B
           info: {attr with spaces: 'The second curve
         <BLANKLINE>
               with endlines'}
           name: two
           path: curve/two

-        name: playlist.hkp
-        path: /path/to/playlist.hkp
-        version: '0.2'
+        state:
+          _base_path: /path/to
+          info: {note: An example playlist}
+          name: playlist.hkp
+          path: /path/to/playlist.hkp
+          version: '0.2'

         <BLANKLINE>
         <BLANKLINE>

-        >>> p._relative_curve_paths = False
+        >>> p.relative_curve_paths = False

         >>> print p.flatten()  # doctest: +REPORT_UDIFF
         # Hooke playlist version 0.2
         >>> print p.flatten()  # doctest: +REPORT_UDIFF
         # Hooke playlist version 0.2

-        _base_path: /path/to
-        _index: 0
-        _max_loaded: 100
-        _relative_curve_paths: false
-        drivers: []
-        info: {note: An example playlist}
-        items:
-        - info: {note: The first curve}
+        !!python/object/new:hooke.playlist.FilePlaylist
+        listitems:
+        - !!python/object:hooke.curve.Curve
+          info: {note: The first curve}

           name: one
           path: /path/to/curve/one
           name: one
           path: /path/to/curve/one

-        - command_stack: !!python/object/new:hooke.command_stack.CommandStack
+        - &id001 !!python/object:hooke.curve.Curve
+          command_stack: !!python/object/new:hooke.command_stack.CommandStack

             listitems:
             - !!python/object:hooke.engine.CommandMessage
               arguments: {arg 0: 0, arg 1: X}
               command: command A
             - !!python/object:hooke.engine.CommandMessage
             listitems:
             - !!python/object:hooke.engine.CommandMessage
               arguments: {arg 0: 0, arg 1: X}
               command: command A
             - !!python/object:hooke.engine.CommandMessage

-              arguments: {arg 0: 1, arg 1: Y}
+              arguments:
+                arg 0: 1
+                curve: *id001

               command: command B
           info: {attr with spaces: 'The second curve
         <BLANKLINE>
               with endlines'}
           name: two
           path: /path/to/curve/two
               command: command B
           info: {attr with spaces: 'The second curve
         <BLANKLINE>
               with endlines'}
           name: two
           path: /path/to/curve/two

-        name: playlist.hkp
-        path: /path/to/playlist.hkp
-        version: '0.2'
+        state:
+          _base_path: /path/to
+          info: {note: An example playlist}
+          name: playlist.hkp
+          path: /path/to/playlist.hkp
+          relative_curve_paths: false
+          version: '0.2'

         <BLANKLINE>
         """
         <BLANKLINE>
         """

-        yaml_string = yaml.dump(self.__getstate__(), allow_unicode=True)
-        return ('# Hooke playlist version %s\n' % self.version) + yaml_string
-
-    def from_string(self, string):
-        u"""Load a playlist from a string.
-
-        Examples
-        --------
+        rcp = self._relative_curve_paths
+        self._relative_curve_paths = self.relative_curve_paths
+        self.update_curve_paths()
+        self._relative_curve_paths = rcp

 
 

-        Minimal example.
-
-        >>> string = '''# Hooke playlist version 0.2
-        ... version: '0.2'
-        ... items:
-        ... - path: picoforce.000
-        ... - path: picoforce.001
-        ... '''
-        >>> p = FilePlaylist(drivers=[],
-        ...                 path=os.path.join('/path', 'to', 'my', 'playlist'))
-        >>> p.from_string(string)
-        >>> for curve in p:
-        ...     print curve.path
-        /path/to/my/picoforce.000
-        /path/to/my/picoforce.001
-
-        More complicated example.
-
-        >>> string = '''# Hooke playlist version 0.2
-        ... _base_path: /path/to
-        ... _digest: null
-        ... _index: 1
-        ... _max_loaded: 100
-        ... _relative_curve_paths: true
-        ... info: {note: An example playlist}
-        ... items:
-        ... - info: {note: The first curve}
-        ...   path: curve/one
-        ... - command_stack: !!python/object/new:hooke.command_stack.CommandStack
-        ...      listitems:
-        ...      - !!python/object:hooke.engine.CommandMessage
-        ...        arguments: {arg 0: 0, arg 1: X}
-        ...        command: command A
-        ...      - !!python/object:hooke.engine.CommandMessage
-        ...        arguments: {arg 0: 1, arg 1: Y}
-        ...        command: command B
-        ...   info: {attr with spaces: 'The second curve
-        ... 
-        ...       with endlines'}
-        ...   name: two
-        ...   path: curve/two
-        ... name: playlist.hkp
-        ... path: /path/to/playlist.hkp
-        ... version: '0.2'
-        ... '''
-        >>> p = FilePlaylist(drivers=[],
-        ...                  path=os.path.join('path', 'to', 'my', 'playlist'))
-        >>> p.from_string(string)
-        >>> p._index
-        1
-        >>> p.info
-        {'note': 'An example playlist'}
-        >>> for curve in p:
-        ...     print curve.name, curve.path
-        one /path/to/curve/one
-        two /path/to/curve/two
-        >>> p[-1].info['attr with spaces']
-        'The second curve\\nwith endlines'
-        >>> type(p[-1].command_stack)
-        <class 'hooke.command_stack.CommandStack'>
-        >>> p[-1].command_stack  # doctest: +NORMALIZE_WHITESPACE
-        [<CommandMessage command A {arg 0: 0, arg 1: X}>,
-         <CommandMessage command B {arg 0: 1, arg 1: Y}>]
-        """
-        state = yaml.load(string)
-        self.__setstate__(state)
+        yaml_string = yaml.dump(self, allow_unicode=True)
+        self.update_curve_paths()
+        return ('# Hooke playlist version %s\n' % self.version) + yaml_string

 
     def save(self, path=None, makedirs=True):
         """Saves the playlist to a YAML file.
 
     def save(self, path=None, makedirs=True):
         """Saves the playlist to a YAML file.


@@ -523,31 +449,109 @@ class FilePlaylist (Playlist):
             f.write(self.flatten())
             self._digest = self.digest()
 
             f.write(self.flatten())
             self._digest = self.digest()
 

-    def load(self, path=None, identify=True, hooke=None):
-        """Load a playlist from a file.
-        """
-        self.set_path(path)
-        with open(self.path, 'r') as f:
-            text = f.read()
-        self.from_string(text)
-        self._digest = self.digest()
-        for curve in self:
-            curve.set_hooke(hooke)
-            if identify == True:
-                curve.identify(self.drivers)

 
 

+def from_string(string):
+    u"""Load a playlist from a string.
+
+    Examples
+    --------
+
+    Minimal example.
+
+    >>> string = '''# Hooke playlist version 0.2
+    ... !!python/object/new:hooke.playlist.FilePlaylist
+    ... state:
+    ...   version: '0.2'
+    ... listitems:
+    ... - !!python/object:hooke.curve.Curve
+    ...   path: curve/one
+    ... - !!python/object:hooke.curve.Curve
+    ...   path: curve/two
+    ... '''
+    >>> p = from_string(string)
+    >>> p.set_path('/path/to/playlist')
+    >>> for curve in p:
+    ...     print curve.name, curve.path
+    one /path/to/curve/one
+    two /path/to/curve/two
+
+    More complicated example.
+
+    >>> string = '''# Hooke playlist version 0.2
+    ... !!python/object/new:hooke.playlist.FilePlaylist
+    ... listitems:
+    ... - !!python/object:hooke.curve.Curve
+    ...   info: {note: The first curve}
+    ...   name: one
+    ...   path: /path/to/curve/one
+    ... - &id001 !!python/object:hooke.curve.Curve
+    ...   command_stack: !!python/object/new:hooke.command_stack.CommandStack
+    ...     listitems:
+    ...     - !!python/object:hooke.engine.CommandMessage
+    ...       arguments: {arg 0: 0, arg 1: X}
+    ...       command: command A
+    ...     - !!python/object:hooke.engine.CommandMessage
+    ...       arguments:
+    ...         arg 0: 1
+    ...         curve: *id001
+    ...       command: command B
+    ...   info: {attr with spaces: 'The second curve
+    ... 
+    ...       with endlines'}
+    ...   name: two
+    ...   path: /path/to/curve/two
+    ... state:
+    ...   _base_path: /path/to
+    ...   _index: 1
+    ...   info: {note: An example playlist}
+    ...   name: playlist.hkp
+    ...   path: /path/to/playlist.hkp
+    ...   version: '0.2'
+    ... '''
+    >>> p = from_string(string)
+    >>> p.set_path('/path/to/playlist')
+    >>> p._index
+    1
+    >>> p.info
+    {'note': 'An example playlist'}
+    >>> for curve in p:
+    ...     print curve.name, curve.path
+    one /path/to/curve/one
+    two /path/to/curve/two
+    >>> p[-1].info['attr with spaces']
+    'The second curve\\nwith endlines'
+    >>> type(p[-1].command_stack)
+    <class 'hooke.command_stack.CommandStack'>
+    >>> p[0].command_stack
+    []
+    >>> type(p[0].command_stack)
+    <class 'hooke.command_stack.CommandStack'>
+    >>> p[-1].command_stack  # doctest: +NORMALIZE_WHITESPACE
+    [<CommandMessage command A {arg 0: 0, arg 1: X}>,
+     <CommandMessage command B {arg 0: 1, curve: <Curve two>}>]
+    >>> type(p[1].command_stack)
+    <class 'hooke.command_stack.CommandStack'>
+    >>> c2 = p[-1]
+    >>> c2.command_stack[-1].arguments['curve'] == c2
+    True
+    """
+    return yaml.load(string)

 
 

-class Playlists (NoteIndexList):
-    """A :class:`NoteIndexList` of :class:`FilePlaylist`\s.
+def load(path=None, identify=True, hooke=None):
+    """Load a playlist from a file.

     """
     """

-    def __init__(self, *arg, **kwargs):
-        super(Playlists, self).__init__(*arg, **kwargs)
+    with open(self.path, 'r') as f:
+        text = f.read()
+    playlist = from_string(text)
+    playlist._digest = playlist.digest()
+    for curve in playlist:
+        curve.set_hooke(hooke)
+        if identify == True:
+            curve.identify(playlist.drivers)
+    return playlist

 
 

-    def _item_getstate(self, item):
-        assert isinstance(item, FilePlaylist), type(item)
-        return item.__getstate__()

 
 

-    def _item_setstate(self, state):
-        item = FilePlaylist(drivers=[])
-        item.__setstate__(state)
-        return item
+class Playlists (NoteIndexList):
+    """A :class:`NoteIndexList` of :class:`FilePlaylist`\s.
+    """
+    pass

diff --git a/hooke/util/si.py b/hooke/util/si.py
index 14d90104494831ab567d0a893b46d5e783700b6b..f9614bf3795c07d0f493962563dc14aedd4d0d67 100644 (file)
--- a/hooke/util/si.py
+++ b/hooke/util/si.py
@@ -182,7 +182,7 @@ def join_data_label(name, unit):
     >>> join_data_label('z piezo', 'm')
     'z piezo (m)'
     >>> join_data_label('deflection', 'N')
     >>> join_data_label('z piezo', 'm')
     'z piezo (m)'
     >>> join_data_label('deflection', 'N')

-    'deflection N'
+    'deflection (N)'

     """
     return '%s (%s)' % (name, unit)
 
     """
     return '%s (%s)' % (name, unit)
 

diff --git a/hooke/util/yaml.py b/hooke/util/yaml.py
index f86e3b181d8dead69fbd8c355225820100d156b1..e7f99806a49c6382716f49764d8ac72e3d1092f1 100644 (file)
--- a/hooke/util/yaml.py
+++ b/hooke/util/yaml.py
@@ -17,25 +17,47 @@ null
 The default behavior is to crash.
 
 >>> yaml.Dumper.yaml_representers.pop(numpy.ndarray)  # doctest: +ELLIPSIS
 The default behavior is to crash.
 
 >>> yaml.Dumper.yaml_representers.pop(numpy.ndarray)  # doctest: +ELLIPSIS

-<function ndarray_representer at 0x...>
+<function none_representer at 0x...>

 >>> print yaml.dump(a)
 >>> print yaml.dump(a)

-Traceback (most recent call last):
-  ...
-    if data in [None, ()]:
-TypeError: data type not understood
+!!python/object/apply:numpy.core.multiarray._reconstruct
+args:
+- !!python/name:numpy.ndarray ''
+- !!python/tuple [0]
+- b
+state: !!python/tuple
+- 1
+- !!python/tuple [3]
+- null
+- false
+- "\\x01\\0\\0\\0\\x02\\0\\0\\0\\x03\\0\\0\\0"
+<BLANKLINE>
+
+Hmm, at one point that crashed like this::
+
+    Traceback (most recent call last):
+      ...
+        if data in [None, ()]:
+    TypeError: data type not understood
+
+Must be because of the other representers I've loaded since.

 
 Restore the representer for future tests.
 
 
 Restore the representer for future tests.
 

->>> yaml.add_representer(numpy.ndarray, ndarray_representer)
+>>> yaml.add_representer(numpy.ndarray, none_representer)

 """
 
 from __future__ import absolute_import
 """
 
 from __future__ import absolute_import

+import copy_reg

 import sys
 import sys

+import types

 
 import numpy
 
 import numpy

-import yaml #from yaml.representer import Representer
+import yaml
+import yaml.constructor
+import yaml.representer

 
 

-from ..curve import Data
+from ..curve import Data, Curve
+from ..playlist import FilePlaylist

 
 
 if False: # YAML dump debugging code
 
 
 if False: # YAML dump debugging code


@@ -95,10 +117,121 @@ yaml.add_representer(numpy.float64, float_representer)
 
 def data_representer(dumper, data):
     info = dict(data.info)
 
 def data_representer(dumper, data):
     info = dict(data.info)

-    print 'KEYS', info.keys()

     for key in info.keys():
     for key in info.keys():

-        if key.startswith('raw '):# or 'peak' in key: #or key not in ['surface deflection offset (m)', 'z piezo sensitivity (m/V)', 'z piezo scan (V/bit)', 'z piezo gain', 'deflection range (V)', 'z piezo range (V)', 'spring constant (N/m)', 'z piezo scan size (V)', 'deflection sensitivity (V/bit)', 'z piezo ramp size (V/bit)', 'surface deflection offset', 'z piezo offset (V)', 'name']:
+        if key.startswith('raw '):

             del(info[key])
             del(info[key])

-    print 'AAAS', info.keys()

     return dumper.represent_mapping(u'!hooke.curve.DataInfo', info)
 yaml.add_representer(Data, data_representer)
     return dumper.represent_mapping(u'!hooke.curve.DataInfo', info)
 yaml.add_representer(Data, data_representer)

+
+def object_representer(dumper, data):
+    cls = type(data)
+    if cls in copy_reg.dispatch_table:
+        reduce = copy_reg.dispatch_table[cls](data)
+    elif hasattr(data, '__reduce_ex__'):
+        reduce = data.__reduce_ex__(2)
+    elif hasattr(data, '__reduce__'):
+        reduce = data.__reduce__()
+    else:
+        raise RepresenterError("cannot represent object: %r" % data)
+    reduce = (list(reduce)+[None]*5)[:5]
+    function, args, state, listitems, dictitems = reduce
+    args = list(args)
+    if state is None:
+        state = {}
+    if isinstance(state, dict) and '_default_attrs' in state:
+        for key in state['_default_attrs']:
+            if key in state and state[key] == state['_default_attrs'][key]:
+                del(state[key])
+        del(state['_default_attrs'])
+    if listitems is not None:
+        listitems = list(listitems)
+    if dictitems is not None:
+        dictitems = dict(dictitems)
+    if function.__name__ == '__newobj__':
+        function = args[0]
+        args = args[1:]
+        tag = u'tag:yaml.org,2002:python/object/new:'
+        newobj = True
+    else:
+        tag = u'tag:yaml.org,2002:python/object/apply:'
+        newobj = False
+    function_name = u'%s.%s' % (function.__module__, function.__name__)
+    if not args and not listitems and not dictitems \
+            and isinstance(state, dict) and newobj:
+        return dumper.represent_mapping(
+                u'tag:yaml.org,2002:python/object:'+function_name, state)
+    if not listitems and not dictitems  \
+            and isinstance(state, dict) and not state:
+        return dumper.represent_sequence(tag+function_name, args)
+    value = {}
+    if args:
+        value['args'] = args
+    if state or not isinstance(state, dict):
+        value['state'] = state
+    if listitems:
+        value['listitems'] = listitems
+    if dictitems:
+        value['dictitems'] = dictitems
+    return dumper.represent_mapping(tag+function_name, value)
+yaml.add_representer(FilePlaylist, object_representer)
+yaml.add_representer(Curve, object_representer)
+
+
+# Monkey patch PyYAML bug 159.
+#   Yaml failed to restore loops in objects when __setstate__ is defined
+#   http://pyyaml.org/ticket/159
+# With viktor.x.voroshylo@jpmchase.com's patch
+def construct_object(self, node, deep=False):
+    if deep:
+        old_deep = self.deep_construct
+        self.deep_construct = True
+    if node in self.constructed_objects:
+        return self.constructed_objects[node]
+    if node in self.recursive_objects:
+        obj = self.recursive_objects[node]
+        if obj is None :
+            raise ConstructorError(None, None,
+                 "found unconstructable recursive node", node.start_mark)
+        return obj
+    self.recursive_objects[node] = None
+    constructor = None
+    tag_suffix = None
+    if node.tag in self.yaml_constructors:
+        constructor = self.yaml_constructors[node.tag]
+    else:
+        for tag_prefix in self.yaml_multi_constructors:
+            if node.tag.startswith(tag_prefix):
+                tag_suffix = node.tag[len(tag_prefix):]
+                constructor = self.yaml_multi_constructors[tag_prefix]
+                break
+        else:
+            if None in self.yaml_multi_constructors:
+                tag_suffix = node.tag
+                constructor = self.yaml_multi_constructors[None]
+            elif None in self.yaml_constructors:
+                constructor = self.yaml_constructors[None]
+            elif isinstance(node, ScalarNode):
+                constructor = self.__class__.construct_scalar
+            elif isinstance(node, SequenceNode):
+                constructor = self.__class__.construct_sequence
+            elif isinstance(node, MappingNode):
+                constructor = self.__class__.construct_mapping
+    if tag_suffix is None:
+        data = constructor(self, node)
+    else:
+        data = constructor(self, tag_suffix, node)
+    if isinstance(data, types.GeneratorType):
+        generator = data
+        data = generator.next()
+        if self.deep_construct:
+            self.recursive_objects[node] = data
+            for dummy in generator:
+                pass
+        else:
+            self.state_generators.append(generator)
+    self.constructed_objects[node] = data
+    del self.recursive_objects[node]
+    if deep:
+        self.deep_construct = old_deep
+    return data
+yaml.constructor.BaseConstructor.construct_object = construct_object