Added calibration file support to the JPK driver.

I'm not sure about my interpretation of the calibration file format.
I've emailed JPK for clarification, so we'll see what they have to say.

Also:
* Check for 'filetype' attributes when loading playlists.  If it
  exists, try to load using the matching driver before going through
  the whole list of available drivers.  This should speed up checks
  for curves that were identified by some previous playlist activity
  (or by hand if the user is generating playlists from scratch).
* Pass curve info into Driver.read().  This allows playlist attributes
  to affect curve loading.  The particular example that motivated this
  was
    <CHANNEL-NAME>_<BASE-NAME>_to_<TARGET-NAME>_calibration_file
  for the JPK driver, but it is a good idea in general.  Experiment
  temperature should probably be set this way vs. its current global
  config setting.
* Added Driver.logger(), to reduce code duplication.

diff --git a/hooke/curve.py b/hooke/curve.py
index 146acfa8abf18928f823672ff4e774addd4eaed0..db313182c7f95b0f138fc351cd232b8f39829bc2 100644 (file)
--- a/hooke/curve.py
+++ b/hooke/curve.py
@@ -168,6 +168,13 @@ class Curve (object):
         """Identify the appropriate :class:`hooke.driver.Driver` for
         the curve file (`.path`).
         """
+        if 'filetype' in self.info:
+            driver = [d for d in drivers if d.name == self.info['filetype']]
+            if len(driver) == 1:
+                driver = driver[0]
+                if driver.is_me(self.path):
+                    self.driver = driver
+                    return
         for driver in drivers:
             if driver.is_me(self.path):
                 self.driver = driver # remember the working driver
@@ -177,7 +184,7 @@ class Curve (object):
     def load(self):
         """Use the driver to read the curve into memory.
         """
-        data,info = self.driver.read(self.path)
+        data,info = self.driver.read(self.path, self.info)
         self.data = data
         for key,value in info.items():
             self.info[key] = value

diff --git a/hooke/driver/__init__.py b/hooke/driver/__init__.py
index 15c05c1d7e038dafbba6e829194055858bb9db23..824a5e28b6efaec1ccb36cb761a88ab7d7d0d514 100644 (file)
--- a/hooke/driver/__init__.py
+++ b/hooke/driver/__init__.py
@@ -24,6 +24,8 @@ commercial force spectroscopy microscopes are provided, and it's easy
 to write your own to handle your lab's specific format.
 """
 
+import logging
+
 from ..config import Setting
 from ..util.pluggable import IsSubclass, construct_graph
 
@@ -79,16 +81,22 @@ class Driver(object):
         """
         return False
 
-    def read(self, path):
+    def read(self, path, info=None):
         """Read data from `path` and return a
         ([:class:`hooke.curve.Data`, ...], `info`) tuple.
 
+        The input `info` :class:`dict` may contain attributes read
+        from the :class:`~hooke.playlist.FilePlaylist`.
+
         The `info` :class:`dict` must contain values for the keys:
         'filetype' and 'experiment'.  See :class:`hooke.curve.Curve`
         for details.
         """
         raise NotImplementedError
 
+    def logger(self):
+        return logging.getLogger('hooke')
+
 # Construct driver dependency graph and load default drivers.
 
 DRIVER_GRAPH = construct_graph(

diff --git a/hooke/driver/jpk.py b/hooke/driver/jpk.py
index f560c09eca55bfa2443e28cef1334978ecbce1ce..8879c85d3d3f5fab326aaac0bf4a9be73ab209d8 100644 (file)
--- a/hooke/driver/jpk.py
+++ b/hooke/driver/jpk.py
@@ -20,7 +20,6 @@
 """Driver for JPK ForceRobot's velocity clamp data format.
 """
 
-import logging
 import os.path
 import pprint
 import zipfile
@@ -76,29 +75,37 @@ class JPKDriver (Driver):
                     return True
         return False
 
-    def read(self, path):
+    def read(self, path, info=None):
+        if info == None:
+            info = {}
         if zipfile.is_zipfile(path):  # JPK file versions since at least 0.5
-            return self._read_zip(path)
+            return self._read_zip(path, info)
         else:
-            return self._read_old(path)
+            return self._read_old(path, info)
 
-    def _read_zip(self, path):
+    def _read_zip(self, path, info):
         with Closing(zipfile.ZipFile(path, 'r')) as f:
             f.path = path
-            info = self._zip_info(f)
-            approach = self._zip_segment(f, info, 0)
-            retract = self._zip_segment(f, info, 1)
-            assert approach.info['name'] == 'approach', approach.info['name']
-            assert retract.info['name'] == 'retract', retract.info['name']
-            return ([approach, retract],
-                    self._zip_translate_params(info, retract.info['raw info']))
+            zip_info = self._zip_info(f)
+            segments = []
+            for i in range(len([p for p in f.namelist()
+                                if p.endswith('segment-header.properties')])):
+                segments.append(self._zip_segment(f, path, info, zip_info, i))
+            for name in ['approach', 'retract']:
+                if len([s for s in segments if s.info['name'] == name]) == 0:
+                    raise ValueError(
+                        'No segment for %s in %s, only %s'
+                        % (name, path, [s.info['name'] for s in segments]))
+            return (segments,
+                    self._zip_translate_params(zip_info,
+                                               segments[0].info['raw info']))
 
     def _zip_info(self, zipfile):
         with Closing(zipfile.open('header.properties')) as f:
             info = self._parse_params(f.readlines())
             return info
 
-    def _zip_segment(self, zipfile, info, index):
+    def _zip_segment(self, zipfile, path, info, zip_info, index):
         prop_file = zipfile.open(os.path.join(
                 'segments', str(index), 'segment-header.properties'))
         prop = self._parse_params(prop_file.readlines())
@@ -119,7 +126,7 @@ class JPKDriver (Driver):
             info=self._zip_translate_segment_params(prop))
         for i,chan in enumerate(channels):
             d[:,i] = chan
-        return self._zip_scale_segment(d)
+        return self._zip_scale_segment(d, path, info)
 
     def _zip_channel(self, zipfile, segment_index, channel_name, chan_info):
         f = zipfile.open(os.path.join(
@@ -164,16 +171,16 @@ class JPKDriver (Driver):
             'columns':list(params['channels']['list']),
             'name':params['force-segment-header']['name']['name'],
             }
-        if info['name'] == 'extend-spm':
-            info['name'] = 'approach'
-        elif info['name'] == 'retract-spm':
-            info['name'] = 'retract'
+        if info['name'] in ['extend-spm', 'retract-spm', 'pause-at-end-spm']:
+            info['name'] = info['name'][:-len('-spm')]
+            if info['name'] == 'extend':
+                info['name'] = 'approach'
         else:
             raise NotImplementedError(
                 'Unrecognized segment type %s' % info['name'])
         return info
 
-    def _zip_scale_segment(self, segment):
+    def _zip_scale_segment(self, segment, path, info):
         data = curve.Data(
             shape=segment.shape,
             dtype=segment.dtype,
@@ -186,8 +193,10 @@ class JPKDriver (Driver):
         z_col = channels.index('height')
         d_col = channels.index('vDeflection')
         
-        segment = self._zip_scale_channel(segment, z_col, 'calibrated')
-        segment = self._zip_scale_channel(segment, d_col, 'distance')
+        segment = self._zip_scale_channel(
+            segment, z_col, 'calibrated', path, info)
+        segment = self._zip_scale_channel(
+            segment, d_col, 'distance', path, info)
 
         assert segment.info['columns'][z_col] == 'height (m)', \
             segment.info['columns'][z_col]
@@ -200,7 +209,7 @@ class JPKDriver (Driver):
         segment.info['columns'][d_col] = 'deflection (m)'
         return segment
 
-    def _zip_scale_channel(self, segment, channel, conversion):
+    def _zip_scale_channel(self, segment, channel, conversion, path, info):
         channel_name = segment.info['raw info']['channels']['list'][channel]
         conversion_set = segment.info['raw info']['channel'][channel_name]['conversion-set']
         conversion_info = conversion_set['conversion'][conversion]
@@ -209,16 +218,41 @@ class JPKDriver (Driver):
             # Our conversion is stacked on a previous conversion.  Do
             # the previous conversion first.
             segment = self._zip_scale_channel(
-                segment, channel, conversion_info['base-calibration-slot'])
+                segment, channel, conversion_info['base-calibration-slot'],
+                info, path)
         if conversion_info['type'] == 'file':
-            if os.path.exists(conversion_info['file']):
-                raise NotImplementedError('No calibration files were available for testing')
+            key = ('%s_%s_to_%s_calibration_file'
+                   % (channel_name,
+                      conversion_info['base-calibration-slot'],
+                      conversion))
+            calib_path = conversion_info['file']
+            if key in info:
+                calib_path = os.path.join(os.path.dirname(path), info[key])
+                self.logger().debug(
+                    'Overriding %s -> %s calibration for %s channel: %s'
+                    % (conversion_info['base-calibration-slot'],
+                       conversion, channel_name, calib_path))
+            if os.path.exists(calib_path):
+                with file(calib_path, 'r') as f:
+                    lines = [x.strip() for x in f.readlines()]
+                    f.close()
+                calib = {  # I've emailed JPK to confirm this file format.
+                    'title':lines[0],
+                    'multiplier':float(lines[1]),
+                    'offset':float(lines[2]),
+                    'unit':lines[3],
+                    'note':'\n'.join(lines[4:]),
+                    }
+                segment[:,channel] = (segment[:,channel] * calib['multiplier']
+                                      + calib['offset'])
+                segment.info['columns'][channel] = (
+                    '%s (%s)' % (channel_name, calib['unit']))
+                return segment
             else:
-                log = logging.getLogger('hooke')                
-                log.warn(
+                self.logger().warn(
                     'Skipping %s -> %s calibration for %s channel.  Calibration file %s not found'
                     % (conversion_info['base-calibration-slot'],
-                       conversion, channel_name, conversion_info['file']))
+                       conversion, channel_name, calib_path))
         else:
             assert conversion_info['type'] == 'simple', conversion_info['type']
         assert conversion_info['scaling']['type'] == 'linear', \
@@ -254,5 +288,7 @@ class JPKDriver (Driver):
                     sub_info[setting[-1]] = fields[1]
         return info
 
-    def _read_old(self, path):
+    def _read_old(self, path, info):
         raise NotImplementedError('No old-style JPK files were available for testing, please send us yours: %s' % path)
+
+#  LocalWords:  JPK ForceRobot's

diff --git a/hooke/driver/mfp3d.py b/hooke/driver/mfp3d.py
index 7f5b2bb36ac8a9d810ae9be1ea59cf00f5484753..fce318c0a8e87d1c346b4124e8a9d1a4fc0e3a43 100644 (file)
--- a/hooke/driver/mfp3d.py
+++ b/hooke/driver/mfp3d.py
@@ -67,7 +67,7 @@ class MFP3DDriver (Driver):
             return True
         return False
     
-    def read(self, path):
+    def read(self, path, info=None):
         data,bin_info,wave_info = loadibw(path)
         approach,retract = self._translate_ibw(data, bin_info, wave_info)
 

diff --git a/hooke/driver/picoforce.py b/hooke/driver/picoforce.py
index 4ca9c5f35dc4dea50c4ba9a0c504fd6365132721..295480404e545c41f464c6d01180b06e8d94df4c 100644 (file)
--- a/hooke/driver/picoforce.py
+++ b/hooke/driver/picoforce.py
@@ -48,7 +48,7 @@ class PicoForceDriver (Driver):
 
         return header[2:17] == 'Force file list'
 
-    def read(self, path):
+    def read(self, path, info=None):
         info = self._read_header_path(path)
         self._check_version(info)
         data = self._read_data_path(path, info)

diff --git a/hooke/driver/tutorial.py b/hooke/driver/tutorial.py
index adcaf2cec1529d7b87d6f3d1cda3338325c19e47..42204d3d5670665fae4b99832e54ad1350825e72 100644 (file)
--- a/hooke/driver/tutorial.py
+++ b/hooke/driver/tutorial.py
@@ -120,7 +120,7 @@ class TutorialDriver (Driver):
             return True
         return False
 
-    def read(self, path):
+    def read(self, path, info=None):
         f = open(path,'r') # open the file for reading
         """In this case, we have a data format that is just a list of
         ASCII values, so we can just divide that in rows, and generate

diff --git a/hooke/driver/wtk.py b/hooke/driver/wtk.py
index 45f0946dc906c5ddffdaafa6dc2c4730f8a335d0..8cef93947c75f1a217887d0c87a746793d6805b0 100644 (file)
--- a/hooke/driver/wtk.py
+++ b/hooke/driver/wtk.py
@@ -81,7 +81,7 @@ class WTKDriver (Driver):
                 return False
         return True
 
-    def read(self, path):
+    def read(self, path, info=None):
         approach_path,retract_path,param_path = self._paths(path)
 
         unlabeled_approach_data = numpy.loadtxt(

diff --git a/test/data/vclamp_jpk/README b/test/data/vclamp_jpk/README
index 31c9b3bea8a930ce64b0b95467f8adec8f869f61..f2f624db05a2838c7a3ef9cdf4d901e88401ab45 100644 (file)
--- a/test/data/vclamp_jpk/README
+++ b/test/data/vclamp_jpk/README
@@ -3,3 +3,15 @@
 Posted to
   http://code.google.com/p/hooke/issues/detail?id=22
 by albedomanu on Oct 16, 2009.
+
+default.cal is albedomanu's piezo calibration file
+  /etc/opt/jpkspm-config-3.3.1/instruments/JPK00343/calibrations/default.cal
+The path is stored in the force curve zip file in
+  segments/*/segment-header.properties
+as the value for
+  channel.<CHANNEL-NAME>.conversion-set.conversion.<CONVERSION-NAME>.file
+  channel.height.conversion-set.conversion.calibrated.file
+but can be overridden by an attrubute in the XML playlist,
+  <CHANNEL-NAME>_<SOURCE-CONVERSION-NAME>_to_<CONVERSION-NAME>_calibration_file
+  height_nominal_to_calibrated_calibration_file="default.cal"
+Relative paths are rooted in the same directory as the curve file.

diff --git a/test/data/vclamp_jpk/default.cal b/test/data/vclamp_jpk/default.cal
new file mode 100644 (file)
index 0000000..cdcc15f
--- /dev/null
+++ b/test/data/vclamp_jpk/default.cal
@@ -0,0 +1,5 @@
+JPK Default
+0.579372
+0.0
+m
+Original JPK calibration for device "JPK00343".

diff --git a/test/data/vclamp_jpk/playlist.hkp b/test/data/vclamp_jpk/playlist.hkp
index e8678bf9d2bf10d8ab455da1045b75f4e36ec88b..323b20a19e0837874f419d5805d27177c5ba6ab0 100644 (file)
--- a/test/data/vclamp_jpk/playlist.hkp
+++ b/test/data/vclamp_jpk/playlist.hkp
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <playlist index="0" version="0.1">
   <curve note="" path="2009.04.23-15.15.47.jpk"/>
-  <curve note="" path="2009.04.23-15.21.39.jpk"/>
+  <curve note="" height_nominal_to_calibrated_calibration_file="default.cal"
+        path="2009.04.23-15.21.39.jpk"/>
 </playlist>

diff --git a/test/jpk_driver.py b/test/jpk_driver.py
index d3b80b63dcdfd10730069793d6a9cb3bcdd28ed0..d35934afd08a8341ee096540c012a2e3726f86be 100644 (file)
--- a/test/jpk_driver.py
+++ b/test/jpk_driver.py
@@ -37,4 +37,21 @@ blocks: 2
 block sizes: [(4096, 6), (4096, 4)]
 Success
 <BLANKLINE>
+
+Load the second curve, to test calibration file overriding.
+
+>>> h = r.run_lines(h, ['next_curve'])
+Success
+<BLANKLINE>
+>>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF
+name: 2009.04.23-15.21.39.jpk
+path: test/data/vclamp_jpk/2009.04.23-15.21.39.jpk
+experiment: None
+driver: <hooke.driver.jpk.JPKDriver object at 0x...>
+filetype: None
+note: 
+blocks: 3
+block sizes: [(4096, 6), (2048, 3), (4096, 4)]
+Success
+<BLANKLINE>
 """