From 2ea2718975ae124c808aa54d4f468bd55ef3234c Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 17 Jun 2010 13:17:57 -0400 Subject: [PATCH] Added calibration file support to the JPK driver. I'm not sure about my interpretation of the calibration file format. I've emailed JPK for clarification, so we'll see what they have to say. Also: * Check for 'filetype' attributes when loading playlists. If it exists, try to load using the matching driver before going through the whole list of available drivers. This should speed up checks for curves that were identified by some previous playlist activity (or by hand if the user is generating playlists from scratch). * Pass curve info into Driver.read(). This allows playlist attributes to affect curve loading. The particular example that motivated this was __to__calibration_file for the JPK driver, but it is a good idea in general. Experiment temperature should probably be set this way vs. its current global config setting. * Added Driver.logger(), to reduce code duplication. --- hooke/curve.py | 9 ++- hooke/driver/__init__.py | 10 +++- hooke/driver/jpk.py | 94 +++++++++++++++++++++---------- hooke/driver/mfp3d.py | 2 +- hooke/driver/picoforce.py | 2 +- hooke/driver/tutorial.py | 2 +- hooke/driver/wtk.py | 2 +- test/data/vclamp_jpk/README | 12 ++++ test/data/vclamp_jpk/default.cal | 5 ++ test/data/vclamp_jpk/playlist.hkp | 3 +- test/jpk_driver.py | 17 ++++++ 11 files changed, 122 insertions(+), 36 deletions(-) create mode 100644 test/data/vclamp_jpk/default.cal diff --git a/hooke/curve.py b/hooke/curve.py index 146acfa..db31318 100644 --- a/hooke/curve.py +++ b/hooke/curve.py @@ -168,6 +168,13 @@ class Curve (object): """Identify the appropriate :class:`hooke.driver.Driver` for the curve file (`.path`). """ + if 'filetype' in self.info: + driver = [d for d in drivers if d.name == self.info['filetype']] + if len(driver) == 1: + driver = driver[0] + if driver.is_me(self.path): + self.driver = driver + return for driver in drivers: if driver.is_me(self.path): self.driver = driver # remember the working driver @@ -177,7 +184,7 @@ class Curve (object): def load(self): """Use the driver to read the curve into memory. """ - data,info = self.driver.read(self.path) + data,info = self.driver.read(self.path, self.info) self.data = data for key,value in info.items(): self.info[key] = value diff --git a/hooke/driver/__init__.py b/hooke/driver/__init__.py index 15c05c1..824a5e2 100644 --- a/hooke/driver/__init__.py +++ b/hooke/driver/__init__.py @@ -24,6 +24,8 @@ commercial force spectroscopy microscopes are provided, and it's easy to write your own to handle your lab's specific format. """ +import logging + from ..config import Setting from ..util.pluggable import IsSubclass, construct_graph @@ -79,16 +81,22 @@ class Driver(object): """ return False - def read(self, path): + def read(self, path, info=None): """Read data from `path` and return a ([:class:`hooke.curve.Data`, ...], `info`) tuple. + The input `info` :class:`dict` may contain attributes read + from the :class:`~hooke.playlist.FilePlaylist`. + The `info` :class:`dict` must contain values for the keys: 'filetype' and 'experiment'. See :class:`hooke.curve.Curve` for details. """ raise NotImplementedError + def logger(self): + return logging.getLogger('hooke') + # Construct driver dependency graph and load default drivers. DRIVER_GRAPH = construct_graph( diff --git a/hooke/driver/jpk.py b/hooke/driver/jpk.py index f560c09..8879c85 100644 --- a/hooke/driver/jpk.py +++ b/hooke/driver/jpk.py @@ -20,7 +20,6 @@ """Driver for JPK ForceRobot's velocity clamp data format. """ -import logging import os.path import pprint import zipfile @@ -76,29 +75,37 @@ class JPKDriver (Driver): return True return False - def read(self, path): + def read(self, path, info=None): + if info == None: + info = {} if zipfile.is_zipfile(path): # JPK file versions since at least 0.5 - return self._read_zip(path) + return self._read_zip(path, info) else: - return self._read_old(path) + return self._read_old(path, info) - def _read_zip(self, path): + def _read_zip(self, path, info): with Closing(zipfile.ZipFile(path, 'r')) as f: f.path = path - info = self._zip_info(f) - approach = self._zip_segment(f, info, 0) - retract = self._zip_segment(f, info, 1) - assert approach.info['name'] == 'approach', approach.info['name'] - assert retract.info['name'] == 'retract', retract.info['name'] - return ([approach, retract], - self._zip_translate_params(info, retract.info['raw info'])) + zip_info = self._zip_info(f) + segments = [] + for i in range(len([p for p in f.namelist() + if p.endswith('segment-header.properties')])): + segments.append(self._zip_segment(f, path, info, zip_info, i)) + for name in ['approach', 'retract']: + if len([s for s in segments if s.info['name'] == name]) == 0: + raise ValueError( + 'No segment for %s in %s, only %s' + % (name, path, [s.info['name'] for s in segments])) + return (segments, + self._zip_translate_params(zip_info, + segments[0].info['raw info'])) def _zip_info(self, zipfile): with Closing(zipfile.open('header.properties')) as f: info = self._parse_params(f.readlines()) return info - def _zip_segment(self, zipfile, info, index): + def _zip_segment(self, zipfile, path, info, zip_info, index): prop_file = zipfile.open(os.path.join( 'segments', str(index), 'segment-header.properties')) prop = self._parse_params(prop_file.readlines()) @@ -119,7 +126,7 @@ class JPKDriver (Driver): info=self._zip_translate_segment_params(prop)) for i,chan in enumerate(channels): d[:,i] = chan - return self._zip_scale_segment(d) + return self._zip_scale_segment(d, path, info) def _zip_channel(self, zipfile, segment_index, channel_name, chan_info): f = zipfile.open(os.path.join( @@ -164,16 +171,16 @@ class JPKDriver (Driver): 'columns':list(params['channels']['list']), 'name':params['force-segment-header']['name']['name'], } - if info['name'] == 'extend-spm': - info['name'] = 'approach' - elif info['name'] == 'retract-spm': - info['name'] = 'retract' + if info['name'] in ['extend-spm', 'retract-spm', 'pause-at-end-spm']: + info['name'] = info['name'][:-len('-spm')] + if info['name'] == 'extend': + info['name'] = 'approach' else: raise NotImplementedError( 'Unrecognized segment type %s' % info['name']) return info - def _zip_scale_segment(self, segment): + def _zip_scale_segment(self, segment, path, info): data = curve.Data( shape=segment.shape, dtype=segment.dtype, @@ -186,8 +193,10 @@ class JPKDriver (Driver): z_col = channels.index('height') d_col = channels.index('vDeflection') - segment = self._zip_scale_channel(segment, z_col, 'calibrated') - segment = self._zip_scale_channel(segment, d_col, 'distance') + segment = self._zip_scale_channel( + segment, z_col, 'calibrated', path, info) + segment = self._zip_scale_channel( + segment, d_col, 'distance', path, info) assert segment.info['columns'][z_col] == 'height (m)', \ segment.info['columns'][z_col] @@ -200,7 +209,7 @@ class JPKDriver (Driver): segment.info['columns'][d_col] = 'deflection (m)' return segment - def _zip_scale_channel(self, segment, channel, conversion): + def _zip_scale_channel(self, segment, channel, conversion, path, info): channel_name = segment.info['raw info']['channels']['list'][channel] conversion_set = segment.info['raw info']['channel'][channel_name]['conversion-set'] conversion_info = conversion_set['conversion'][conversion] @@ -209,16 +218,41 @@ class JPKDriver (Driver): # Our conversion is stacked on a previous conversion. Do # the previous conversion first. segment = self._zip_scale_channel( - segment, channel, conversion_info['base-calibration-slot']) + segment, channel, conversion_info['base-calibration-slot'], + info, path) if conversion_info['type'] == 'file': - if os.path.exists(conversion_info['file']): - raise NotImplementedError('No calibration files were available for testing') + key = ('%s_%s_to_%s_calibration_file' + % (channel_name, + conversion_info['base-calibration-slot'], + conversion)) + calib_path = conversion_info['file'] + if key in info: + calib_path = os.path.join(os.path.dirname(path), info[key]) + self.logger().debug( + 'Overriding %s -> %s calibration for %s channel: %s' + % (conversion_info['base-calibration-slot'], + conversion, channel_name, calib_path)) + if os.path.exists(calib_path): + with file(calib_path, 'r') as f: + lines = [x.strip() for x in f.readlines()] + f.close() + calib = { # I've emailed JPK to confirm this file format. + 'title':lines[0], + 'multiplier':float(lines[1]), + 'offset':float(lines[2]), + 'unit':lines[3], + 'note':'\n'.join(lines[4:]), + } + segment[:,channel] = (segment[:,channel] * calib['multiplier'] + + calib['offset']) + segment.info['columns'][channel] = ( + '%s (%s)' % (channel_name, calib['unit'])) + return segment else: - log = logging.getLogger('hooke') - log.warn( + self.logger().warn( 'Skipping %s -> %s calibration for %s channel. Calibration file %s not found' % (conversion_info['base-calibration-slot'], - conversion, channel_name, conversion_info['file'])) + conversion, channel_name, calib_path)) else: assert conversion_info['type'] == 'simple', conversion_info['type'] assert conversion_info['scaling']['type'] == 'linear', \ @@ -254,5 +288,7 @@ class JPKDriver (Driver): sub_info[setting[-1]] = fields[1] return info - def _read_old(self, path): + def _read_old(self, path, info): raise NotImplementedError('No old-style JPK files were available for testing, please send us yours: %s' % path) + +# LocalWords: JPK ForceRobot's diff --git a/hooke/driver/mfp3d.py b/hooke/driver/mfp3d.py index 7f5b2bb..fce318c 100644 --- a/hooke/driver/mfp3d.py +++ b/hooke/driver/mfp3d.py @@ -67,7 +67,7 @@ class MFP3DDriver (Driver): return True return False - def read(self, path): + def read(self, path, info=None): data,bin_info,wave_info = loadibw(path) approach,retract = self._translate_ibw(data, bin_info, wave_info) diff --git a/hooke/driver/picoforce.py b/hooke/driver/picoforce.py index 4ca9c5f..2954804 100644 --- a/hooke/driver/picoforce.py +++ b/hooke/driver/picoforce.py @@ -48,7 +48,7 @@ class PicoForceDriver (Driver): return header[2:17] == 'Force file list' - def read(self, path): + def read(self, path, info=None): info = self._read_header_path(path) self._check_version(info) data = self._read_data_path(path, info) diff --git a/hooke/driver/tutorial.py b/hooke/driver/tutorial.py index adcaf2c..42204d3 100644 --- a/hooke/driver/tutorial.py +++ b/hooke/driver/tutorial.py @@ -120,7 +120,7 @@ class TutorialDriver (Driver): return True return False - def read(self, path): + def read(self, path, info=None): f = open(path,'r') # open the file for reading """In this case, we have a data format that is just a list of ASCII values, so we can just divide that in rows, and generate diff --git a/hooke/driver/wtk.py b/hooke/driver/wtk.py index 45f0946..8cef939 100644 --- a/hooke/driver/wtk.py +++ b/hooke/driver/wtk.py @@ -81,7 +81,7 @@ class WTKDriver (Driver): return False return True - def read(self, path): + def read(self, path, info=None): approach_path,retract_path,param_path = self._paths(path) unlabeled_approach_data = numpy.loadtxt( diff --git a/test/data/vclamp_jpk/README b/test/data/vclamp_jpk/README index 31c9b3b..f2f624d 100644 --- a/test/data/vclamp_jpk/README +++ b/test/data/vclamp_jpk/README @@ -3,3 +3,15 @@ Posted to http://code.google.com/p/hooke/issues/detail?id=22 by albedomanu on Oct 16, 2009. + +default.cal is albedomanu's piezo calibration file + /etc/opt/jpkspm-config-3.3.1/instruments/JPK00343/calibrations/default.cal +The path is stored in the force curve zip file in + segments/*/segment-header.properties +as the value for + channel..conversion-set.conversion..file + channel.height.conversion-set.conversion.calibrated.file +but can be overridden by an attrubute in the XML playlist, + __to__calibration_file + height_nominal_to_calibrated_calibration_file="default.cal" +Relative paths are rooted in the same directory as the curve file. diff --git a/test/data/vclamp_jpk/default.cal b/test/data/vclamp_jpk/default.cal new file mode 100644 index 0000000..cdcc15f --- /dev/null +++ b/test/data/vclamp_jpk/default.cal @@ -0,0 +1,5 @@ +JPK Default +0.579372 +0.0 +m +Original JPK calibration for device "JPK00343". diff --git a/test/data/vclamp_jpk/playlist.hkp b/test/data/vclamp_jpk/playlist.hkp index e8678bf..323b20a 100644 --- a/test/data/vclamp_jpk/playlist.hkp +++ b/test/data/vclamp_jpk/playlist.hkp @@ -1,5 +1,6 @@ - + diff --git a/test/jpk_driver.py b/test/jpk_driver.py index d3b80b6..d35934a 100644 --- a/test/jpk_driver.py +++ b/test/jpk_driver.py @@ -37,4 +37,21 @@ blocks: 2 block sizes: [(4096, 6), (4096, 4)] Success + +Load the second curve, to test calibration file overriding. + +>>> h = r.run_lines(h, ['next_curve']) +Success + +>>> h = r.run_lines(h, ['curve_info']) # doctest: +ELLIPSIS, +REPORT_UDIFF +name: 2009.04.23-15.21.39.jpk +path: test/data/vclamp_jpk/2009.04.23-15.21.39.jpk +experiment: None +driver: +filetype: None +note: +blocks: 3 +block sizes: [(4096, 6), (2048, 3), (4096, 4)] +Success + """ -- 2.26.2