Added calibration file support to the JPK driver.
authorW. Trevor King <wking@drexel.edu>
Thu, 17 Jun 2010 17:17:57 +0000 (13:17 -0400)
committerW. Trevor King <wking@drexel.edu>
Thu, 17 Jun 2010 17:17:57 +0000 (13:17 -0400)
I'm not sure about my interpretation of the calibration file format.
I've emailed JPK for clarification, so we'll see what they have to say.

Also:
* Check for 'filetype' attributes when loading playlists.  If it
  exists, try to load using the matching driver before going through
  the whole list of available drivers.  This should speed up checks
  for curves that were identified by some previous playlist activity
  (or by hand if the user is generating playlists from scratch).
* Pass curve info into Driver.read().  This allows playlist attributes
  to affect curve loading.  The particular example that motivated this
  was
    <CHANNEL-NAME>_<BASE-NAME>_to_<TARGET-NAME>_calibration_file
  for the JPK driver, but it is a good idea in general.  Experiment
  temperature should probably be set this way vs. its current global
  config setting.
* Added Driver.logger(), to reduce code duplication.

hooke/curve.py
hooke/driver/__init__.py
hooke/driver/jpk.py
hooke/driver/mfp3d.py
hooke/driver/picoforce.py
hooke/driver/tutorial.py
hooke/driver/wtk.py
test/data/vclamp_jpk/README
test/data/vclamp_jpk/default.cal [new file with mode: 0644]
test/data/vclamp_jpk/playlist.hkp
test/jpk_driver.py

index 146acfa..db31318 100644 (file)
@@ -168,6 +168,13 @@ class Curve (object):
         """Identify the appropriate :class:`hooke.driver.Driver` for
         the curve file (`.path`).
         """
+        if 'filetype' in self.info:
+            driver = [d for d in drivers if d.name == self.info['filetype']]
+            if len(driver) == 1:
+                driver = driver[0]
+                if driver.is_me(self.path):
+                    self.driver = driver
+                    return
         for driver in drivers:
             if driver.is_me(self.path):
                 self.driver = driver # remember the working driver
@@ -177,7 +184,7 @@ class Curve (object):
     def load(self):
         """Use the driver to read the curve into memory.
         """
-        data,info = self.driver.read(self.path)
+        data,info = self.driver.read(self.path, self.info)
         self.data = data
         for key,value in info.items():
             self.info[key] = value
index 15c05c1..824a5e2 100644 (file)
@@ -24,6 +24,8 @@ commercial force spectroscopy microscopes are provided, and it's easy
 to write your own to handle your lab's specific format.
 """
 
+import logging
+
 from ..config import Setting
 from ..util.pluggable import IsSubclass, construct_graph
 
@@ -79,16 +81,22 @@ class Driver(object):
         """
         return False
 
-    def read(self, path):
+    def read(self, path, info=None):
         """Read data from `path` and return a
         ([:class:`hooke.curve.Data`, ...], `info`) tuple.
 
+        The input `info` :class:`dict` may contain attributes read
+        from the :class:`~hooke.playlist.FilePlaylist`.
+
         The `info` :class:`dict` must contain values for the keys:
         'filetype' and 'experiment'.  See :class:`hooke.curve.Curve`
         for details.
         """
         raise NotImplementedError
 
+    def logger(self):
+        return logging.getLogger('hooke')
+
 # Construct driver dependency graph and load default drivers.
 
 DRIVER_GRAPH = construct_graph(
index f560c09..8879c85 100644 (file)
@@ -20,7 +20,6 @@
 """Driver for JPK ForceRobot's velocity clamp data format.
 """
 
-import logging
 import os.path
 import pprint
 import zipfile
@@ -76,29 +75,37 @@ class JPKDriver (Driver):
                     return True
         return False
 
-    def read(self, path):
+    def read(self, path, info=None):
+        if info == None:
+            info = {}
         if zipfile.is_zipfile(path):  # JPK file versions since at least 0.5
-            return self._read_zip(path)
+            return self._read_zip(path, info)
         else:
-            return self._read_old(path)
+            return self._read_old(path, info)
 
-    def _read_zip(self, path):
+    def _read_zip(self, path, info):
         with Closing(zipfile.ZipFile(path, 'r')) as f:
             f.path = path
-            info = self._zip_info(f)
-            approach = self._zip_segment(f, info, 0)
-            retract = self._zip_segment(f, info, 1)
-            assert approach.info['name'] == 'approach', approach.info['name']
-            assert retract.info['name'] == 'retract', retract.info['name']
-            return ([approach, retract],
-                    self._zip_translate_params(info, retract.info['raw info']))
+            zip_info = self._zip_info(f)
+            segments = []
+            for i in range(len([p for p in f.namelist()
+                                if p.endswith('segment-header.properties')])):
+                segments.append(self._zip_segment(f, path, info, zip_info, i))
+            for name in ['approach', 'retract']:
+                if len([s for s in segments if s.info['name'] == name]) == 0:
+                    raise ValueError(
+                        'No segment for %s in %s, only %s'
+                        % (name, path, [s.info['name'] for s in segments]))
+            return (segments,
+                    self._zip_translate_params(zip_info,
+                                               segments[0].info['raw info']))
 
     def _zip_info(self, zipfile):
         with Closing(zipfile.open('header.properties')) as f:
             info = self._parse_params(f.readlines())
             return info
 
-    def _zip_segment(self, zipfile, info, index):
+    def _zip_segment(self, zipfile, path, info, zip_info, index):
         prop_file = zipfile.open(os.path.join(
                 'segments', str(index), 'segment-header.properties'))
         prop = self._parse_params(prop_file.readlines())
@@ -119,7 +126,7 @@ class JPKDriver (Driver):
             info=self._zip_translate_segment_params(prop))
         for i,chan in enumerate(channels):
             d[:,i] = chan
-        return self._zip_scale_segment(d)
+        return self._zip_scale_segment(d, path, info)
 
     def _zip_channel(self, zipfile, segment_index, channel_name, chan_info):
         f = zipfile.open(os.path.join(
@@ -164,16 +171,16 @@ class JPKDriver (Driver):
             'columns':list(params['channels']['list']),
             'name':params['force-segment-header']['name']['name'],
             }
-        if info['name'] == 'extend-spm':
-            info['name'] = 'approach'
-        elif info['name'] == 'retract-spm':
-            info['name'] = 'retract'
+        if info['name'] in ['extend-spm', 'retract-spm', 'pause-at-end-spm']:
+            info['name'] = info['name'][:-len('-spm')]
+            if info['name'] == 'extend':
+                info['name'] = 'approach'
         else:
             raise NotImplementedError(
                 'Unrecognized segment type %s' % info['name'])
         return info
 
-    def _zip_scale_segment(self, segment):
+    def _zip_scale_segment(self, segment, path, info):
         data = curve.Data(
             shape=segment.shape,
             dtype=segment.dtype,
@@ -186,8 +193,10 @@ class JPKDriver (Driver):
         z_col = channels.index('height')
         d_col = channels.index('vDeflection')
         
-        segment = self._zip_scale_channel(segment, z_col, 'calibrated')
-        segment = self._zip_scale_channel(segment, d_col, 'distance')
+        segment = self._zip_scale_channel(
+            segment, z_col, 'calibrated', path, info)
+        segment = self._zip_scale_channel(
+            segment, d_col, 'distance', path, info)
 
         assert segment.info['columns'][z_col] == 'height (m)', \
             segment.info['columns'][z_col]
@@ -200,7 +209,7 @@ class JPKDriver (Driver):
         segment.info['columns'][d_col] = 'deflection (m)'
         return segment
 
-    def _zip_scale_channel(self, segment, channel, conversion):
+    def _zip_scale_channel(self, segment, channel, conversion, path, info):
         channel_name = segment.info['raw info']['channels']['list'][channel]
         conversion_set = segment.info['raw info']['channel'][channel_name]['conversion-set']
         conversion_info = conversion_set['conversion'][conversion]
@@ -209,16 +218,41 @@ class JPKDriver (Driver):
             # Our conversion is stacked on a previous conversion.  Do
             # the previous conversion first.
             segment = self._zip_scale_channel(
-                segment, channel, conversion_info['base-calibration-slot'])
+                segment, channel, conversion_info['base-calibration-slot'],
+                info, path)
         if conversion_info['type'] == 'file':
-            if os.path.exists(conversion_info['file']):
-                raise NotImplementedError('No calibration files were available for testing')
+            key = ('%s_%s_to_%s_calibration_file'
+                   % (channel_name,
+                      conversion_info['base-calibration-slot'],
+                      conversion))
+            calib_path = conversion_info['file']
+            if key in info:
+                calib_path = os.path.join(os.path.dirname(path), info[key])
+                self.logger().debug(
+                    'Overriding %s -> %s calibration for %s channel: %s'
+                    % (conversion_info['base-calibration-slot'],
+                       conversion, channel_name, calib_path))
+            if os.path.exists(calib_path):
+                with file(calib_path, 'r') as f:
+                    lines = [x.strip() for x in f.readlines()]
+                    f.close()
+                calib = {  # I've emailed JPK to confirm this file format.
+                    'title':lines[0],
+                    'multiplier':float(lines[1]),
+                    'offset':float(lines[2]),
+                    'unit':lines[3],
+                    'note':'\n'.join(lines[4:]),
+                    }
+                segment[:,channel] = (segment[:,channel] * calib['multiplier']
+                                      + calib['offset'])
+                segment.info['columns'][channel] = (
+                    '%s (%s)' % (channel_name, calib['unit']))
+                return segment
             else:
-                log = logging.getLogger('hooke')                
-                log.warn(
+                self.logger().warn(
                     'Skipping %s -> %s calibration for %s channel.  Calibration file %s not found'
                     % (conversion_info['base-calibration-slot'],
-                       conversion, channel_name, conversion_info['file']))
+                       conversion, channel_name, calib_path))
         else:
             assert conversion_info['type'] == 'simple', conversion_info['type']
         assert conversion_info['scaling']['type'] == 'linear', \
@@ -254,5 +288,7 @@ class JPKDriver (Driver):
                     sub_info[setting[-1]] = fields[1]
         return info
 
-    def _read_old(self, path):
+    def _read_old(self, path, info):
         raise NotImplementedError('No old-style JPK files were available for testing, please send us yours: %s' % path)
+
+#  LocalWords:  JPK ForceRobot's
index 7f5b2bb..fce318c 100644 (file)
@@ -67,7 +67,7 @@ class MFP3DDriver (Driver):
             return True
         return False
     
-    def read(self, path):
+    def read(self, path, info=None):
         data,bin_info,wave_info = loadibw(path)
         approach,retract = self._translate_ibw(data, bin_info, wave_info)
 
index 4ca9c5f..2954804 100644 (file)
@@ -48,7 +48,7 @@ class PicoForceDriver (Driver):
 
         return header[2:17] == 'Force file list'
 
-    def read(self, path):
+    def read(self, path, info=None):
         info = self._read_header_path(path)
         self._check_version(info)
         data = self._read_data_path(path, info)
index adcaf2c..42204d3 100644 (file)
@@ -120,7 +120,7 @@ class TutorialDriver (Driver):
             return True
         return False
 
-    def read(self, path):
+    def read(self, path, info=None):
         f = open(path,'r') # open the file for reading
         """In this case, we have a data format that is just a list of
         ASCII values, so we can just divide that in rows, and generate
index 45f0946..8cef939 100644 (file)
@@ -81,7 +81,7 @@ class WTKDriver (Driver):
                 return False
         return True
 
-    def read(self, path):
+    def read(self, path, info=None):
         approach_path,retract_path,param_path = self._paths(path)
 
         unlabeled_approach_data = numpy.loadtxt(
index 31c9b3b..f2f624d 100644 (file)
@@ -3,3 +3,15 @@
 Posted to
   http://code.google.com/p/hooke/issues/detail?id=22
 by albedomanu on Oct 16, 2009.
+
+default.cal is albedomanu's piezo calibration file
+  /etc/opt/jpkspm-config-3.3.1/instruments/JPK00343/calibrations/default.cal
+The path is stored in the force curve zip file in
+  segments/*/segment-header.properties
+as the value for
+  channel.<CHANNEL-NAME>.conversion-set.conversion.<CONVERSION-NAME>.file
+  channel.height.conversion-set.conversion.calibrated.file
+but can be overridden by an attrubute in the XML playlist,
+  <CHANNEL-NAME>_<SOURCE-CONVERSION-NAME>_to_<CONVERSION-NAME>_calibration_file
+  height_nominal_to_calibrated_calibration_file="default.cal"
+Relative paths are rooted in the same directory as the curve file.
diff --git a/test/data/vclamp_jpk/default.cal b/test/data/vclamp_jpk/default.cal
new file mode 100644 (file)
index 0000000..cdcc15f
--- /dev/null
@@ -0,0 +1,5 @@
+JPK Default
+0.579372
+0.0
+m
+Original JPK calibration for device "JPK00343".
index e8678bf..323b20a 100644 (file)
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <playlist index="0" version="0.1">
   <curve note="" path="2009.04.23-15.15.47.jpk"/>
-  <curve note="" path="2009.04.23-15.21.39.jpk"/>
+  <curve note="" height_nominal_to_calibrated_calibration_file="default.cal"
+        path="2009.04.23-15.21.39.jpk"/>
 </playlist>
index d3b80b6..d35934a 100644 (file)
@@ -37,4 +37,21 @@ blocks: 2
 block sizes: [(4096, 6), (4096, 4)]
 Success
 <BLANKLINE>
+
+Load the second curve, to test calibration file overriding.
+
+>>> h = r.run_lines(h, ['next_curve'])
+Success
+<BLANKLINE>
+>>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF
+name: 2009.04.23-15.21.39.jpk
+path: test/data/vclamp_jpk/2009.04.23-15.21.39.jpk
+experiment: None
+driver: <hooke.driver.jpk.JPKDriver object at 0x...>
+filetype: None
+note: 
+blocks: 3
+block sizes: [(4096, 6), (2048, 3), (4096, 4)]
+Success
+<BLANKLINE>
 """