1 # Copyright (C) 2010 W. Trevor King <wking@drexel.edu>
3 # This file is part of Hooke.
5 # Hooke is free software: you can redistribute it and/or modify it
6 # under the terms of the GNU Lesser General Public License as
7 # published by the Free Software Foundation, either version 3 of the
8 # License, or (at your option) any later version.
10 # Hooke is distributed in the hope that it will be useful, but WITHOUT
11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
13 # Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with Hooke. If not, see
17 # <http://www.gnu.org/licenses/>.
19 "Read IGOR Binary Wave files into Numpy arrays."
21 # Based on WaveMetric's Technical Note 003, "Igor Binary Format"
22 # ftp://ftp.wavemetrics.net/IgorPro/Technical_Notes/TN003.zip
23 # From ftp://ftp.wavemetrics.net/IgorPro/Technical_Notes/TN000.txt
24 # We place no restrictions on copying Technical Notes, with the
25 # exception that you cannot resell them. So read, enjoy, and
26 # share. We hope IGOR Technical Notes will provide you with lots of
27 # valuable information while you are developing IGOR applications.
29 import array as _array
31 import types as _types
33 import numpy as _numpy
35 from .struct import Structure as _Structure
36 from .struct import Field as _Field
37 from .util import assert_null as _assert_null
38 from .util import byte_order as _byte_order
39 from .util import need_to_reorder_bytes as _need_to_reorder_bytes
40 from .util import checksum as _checksum
43 # Numpy doesn't support complex integers by default, see
44 # http://mail.python.org/pipermail/python-dev/2002-April/022408.html
45 # http://mail.scipy.org/pipermail/numpy-discussion/2007-October/029447.html
46 # So we roll our own types. See
47 # http://docs.scipy.org/doc/numpy/user/basics.rec.html
48 # http://docs.scipy.org/doc/numpy/reference/generated/numpy.dtype.html
49 complexInt8 = _numpy.dtype([('real', _numpy.int8), ('imag', _numpy.int8)])
50 complexInt16 = _numpy.dtype([('real', _numpy.int16), ('imag', _numpy.int16)])
51 complexInt32 = _numpy.dtype([('real', _numpy.int32), ('imag', _numpy.int32)])
52 complexUInt8 = _numpy.dtype([('real', _numpy.uint8), ('imag', _numpy.uint8)])
53 complexUInt16 = _numpy.dtype(
54 [('real', _numpy.uint16), ('imag', _numpy.uint16)])
55 complexUInt32 = _numpy.dtype(
56 [('real', _numpy.uint32), ('imag', _numpy.uint32)])
58 # Begin IGOR constants and typedefs from IgorBin.h
61 TYPE_TABLE = { # (key: integer flag, value: numpy dtype)
62 0:None, # Text wave, not handled in ReadWave.c
63 1:_numpy.complex, # NT_CMPLX, makes number complex.
64 2:_numpy.float32, # NT_FP32, 32 bit fp numbers.
66 4:_numpy.float64, # NT_FP64, 64 bit fp numbers.
68 8:_numpy.int8, # NT_I8, 8 bit signed integer. Requires Igor Pro
71 0x10:_numpy.int16,# NT_I16, 16 bit integer numbers. Requires Igor
74 0x20:_numpy.int32,# NT_I32, 32 bit integer numbers. Requires Igor
77 # 0x40:None, # NT_UNSIGNED, Makes above signed integers
78 # # unsigned. Requires Igor Pro 3.0 or later.
91 BinHeaderCommon = _Structure( # WTK: this one is mine.
92 name='BinHeaderCommon',
94 _Field('h', 'version', help='Version number for backwards compatibility.'),
97 BinHeader1 = _Structure(
100 _Field('h', 'version', help='Version number for backwards compatibility.'),
101 _Field('l', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'),
102 _Field('h', 'checksum', help='Checksum over this header and the wave header.'),
105 BinHeader2 = _Structure(
108 _Field('h', 'version', help='Version number for backwards compatibility.'),
109 _Field('l', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'),
110 _Field('l', 'noteSize', help='The size of the note text.'),
111 _Field('l', 'pictSize', default=0, help='Reserved. Write zero. Ignore on read.'),
112 _Field('h', 'checksum', help='Checksum over this header and the wave header.'),
115 BinHeader3 = _Structure(
118 _Field('h', 'version', help='Version number for backwards compatibility.'),
119 _Field('h', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'),
120 _Field('l', 'noteSize', help='The size of the note text.'),
121 _Field('l', 'formulaSize', help='The size of the dependency formula, if any.'),
122 _Field('l', 'pictSize', default=0, help='Reserved. Write zero. Ignore on read.'),
123 _Field('h', 'checksum', help='Checksum over this header and the wave header.'),
126 BinHeader5 = _Structure(
129 _Field('h', 'version', help='Version number for backwards compatibility.'),
130 _Field('h', 'checksum', help='Checksum over this header and the wave header.'),
131 _Field('l', 'wfmSize', help='The size of the WaveHeader5 data structure plus the wave data.'),
132 _Field('l', 'formulaSize', help='The size of the dependency formula, if any.'),
133 _Field('l', 'noteSize', help='The size of the note text.'),
134 _Field('l', 'dataEUnitsSize', help='The size of optional extended data units.'),
135 _Field('l', 'dimEUnitsSize', help='The size of optional extended dimension units.', count=MAXDIMS),
136 _Field('l', 'dimLabelsSize', help='The size of optional dimension labels.', count=MAXDIMS),
137 _Field('l', 'sIndicesSize', help='The size of string indicies if this is a text wave.'),
138 _Field('l', 'optionsSize1', default=0, help='Reserved. Write zero. Ignore on read.'),
139 _Field('l', 'optionsSize2', default=0, help='Reserved. Write zero. Ignore on read.'),
144 MAX_WAVE_NAME2 = 18 # Maximum length of wave name in version 1 and 2
145 # files. Does not include the trailing null.
146 MAX_WAVE_NAME5 = 31 # Maximum length of wave name in version 5
147 # files. Does not include the trailing null.
150 # Header to an array of waveform data.
152 WaveHeader2 = _Structure(
155 _Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'),
156 _Field('P', 'next', default=0, help='Used in memory only. Write zero. Ignore on read.'),
157 _Field('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME2+2),
158 _Field('h', 'whVersion', default=0, help='Write 0. Ignore on read.'),
159 _Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'),
160 _Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'),
161 _Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1),
162 _Field('c', 'xUnits', default=0, help='Natural x-axis units go here - null if none.', count=MAX_UNIT_CHARS+1),
163 _Field('l', 'npnts', help='Number of data points in wave.'),
164 _Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
165 _Field('d', 'hsA', help='X value for point p = hsA*p + hsB'),
166 _Field('d', 'hsB', help='X value for point p = hsA*p + hsB'),
167 _Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
168 _Field('h', 'swModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
169 _Field('h', 'fsValid', help='True if full scale values have meaning.'),
170 _Field('d', 'topFullScale', help='The min full scale value for wave.'), # sic, 'min' should probably be 'max'
171 _Field('d', 'botFullScale', help='The min full scale value for wave.'),
172 _Field('c', 'useBits', default=0, help='Used in memory only. Write zero. Ignore on read.'),
173 _Field('c', 'kindBits', default=0, help='Reserved. Write zero. Ignore on read.'),
174 _Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'),
175 _Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'),
176 _Field('L', 'creationDate', help='DateTime of creation. Not used in version 1 files.'),
177 _Field('c', 'wUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=2),
178 _Field('L', 'modDate', help='DateTime of last modification.'),
179 _Field('P', 'waveNoteH', help='Used in memory only. Write zero. Ignore on read.'),
180 _Field('f', 'wData', help='The start of the array of waveform data.', count=4),
183 WaveHeader5 = _Structure(
186 _Field('P', 'next', help='link to next wave in linked list.'),
187 _Field('L', 'creationDate', help='DateTime of creation.'),
188 _Field('L', 'modDate', help='DateTime of last modification.'),
189 _Field('l', 'npnts', help='Total number of points (multiply dimensions up to first zero).'),
190 _Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'),
191 _Field('h', 'dLock', default=0, help='Reserved. Write zero. Ignore on read.'),
192 _Field('c', 'whpad1', default=0, help='Reserved. Write zero. Ignore on read.', count=6),
193 _Field('h', 'whVersion', default=1, help='Write 1. Ignore on read.'),
194 _Field('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME5+1),
195 _Field('l', 'whpad2', default=0, help='Reserved. Write zero. Ignore on read.'),
196 _Field('P', 'dFolder', default=0, help='Used in memory only. Write zero. Ignore on read.'),
197 # Dimensioning info. [0] == rows, [1] == cols etc
198 _Field('l', 'nDim', help='Number of of items in a dimension -- 0 means no data.', count=MAXDIMS),
199 _Field('d', 'sfA', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS),
200 _Field('d', 'sfB', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS),
202 _Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1),
203 _Field('c', 'dimUnits', default=0, help='Natural dimension units go here - null if none.', count=(MAXDIMS, MAX_UNIT_CHARS+1)),
204 _Field('h', 'fsValid', help='TRUE if full scale values have meaning.'),
205 _Field('h', 'whpad3', default=0, help='Reserved. Write zero. Ignore on read.'),
206 _Field('d', 'topFullScale', help='The max and max full scale value for wave'), # sic, probably "max and min"
207 _Field('d', 'botFullScale', help='The max and max full scale value for wave.'), # sic, probably "max and min"
208 _Field('P', 'dataEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.'),
209 _Field('P', 'dimEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS),
210 _Field('P', 'dimLabels', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS),
211 _Field('P', 'waveNoteH', default=0, help='Used in memory only. Write zero. Ignore on read.'),
212 _Field('l', 'whUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=16),
213 # The following stuff is considered private to Igor.
214 _Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
215 _Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
216 _Field('h', 'swModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
217 _Field('c', 'useBits', default=0, help='Used in memory only. Write zero. Ignore on read.'),
218 _Field('c', 'kindBits', default=0, help='Reserved. Write zero. Ignore on read.'),
219 _Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'),
220 _Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'),
221 _Field('h', 'whpad4', default=0, help='Reserved. Write zero. Ignore on read.'),
222 _Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'),
223 _Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'),
224 _Field('P', 'sIndices', default=0, help='Used in memory only. Write zero. Ignore on read.'),
225 _Field('f', 'wData', help='The start of the array of data. Must be 64 bit aligned.', count=1),
228 # End IGOR constants and typedefs from IgorBin.h
230 def _version_structs(version, byte_order):
245 ('This does not appear to be a valid Igor binary wave file. '
246 'The version field = {}.\n').format(version))
247 checkSumSize = bin.size + wave.size
249 checkSumSize -= 4 # Version 5 checksum does not include the wData field.
250 bin.set_byte_order(byte_order)
251 wave.set_byte_order(byte_order)
252 return (bin, wave, checkSumSize)
254 # Translated from ReadWave()
255 def load(filename, strict=True):
256 if hasattr(filename, 'read'):
257 f = filename # filename is actually a stream object
259 f = open(filename, 'rb')
261 BinHeaderCommon.set_byte_order('=')
262 b = buffer(f.read(BinHeaderCommon.size))
263 version = BinHeaderCommon.unpack_from(b)['version']
264 needToReorderBytes = _need_to_reorder_bytes(version)
265 byteOrder = _byte_order(needToReorderBytes)
267 if needToReorderBytes:
268 BinHeaderCommon.set_byte_order(byteOrder)
269 version = BinHeaderCommon.unpack_from(b)['version']
270 bin_struct,wave_struct,checkSumSize = _version_structs(
273 b = buffer(b + f.read(bin_struct.size + wave_struct.size - BinHeaderCommon.size))
274 c = _checksum(b, byteOrder, 0, checkSumSize)
277 ('This does not appear to be a valid Igor binary wave file. '
278 'Error in checksum: should be 0, is {}.').format(c))
279 bin_info = bin_struct.unpack_from(b)
280 wave_info = wave_struct.unpack_from(b, offset=bin_struct.size)
281 if version in [1,2,3]:
282 tail = 16 # 16 = size of wData field in WaveHeader2 structure
283 waveDataSize = bin_info['wfmSize'] - wave_struct.size
284 # = bin_info['wfmSize']-16 - (wave_struct.size - tail)
286 assert version == 5, version
287 tail = 4 # 4 = size of wData field in WaveHeader5 structure
288 waveDataSize = bin_info['wfmSize'] - (wave_struct.size - tail)
289 # dtype() wrapping to avoid numpy.generic and
290 # getset_descriptor issues with the builtin numpy types
291 # (e.g. int32). It has no effect on our local complex
294 shape = [n for n in wave_info['nDim'] if n > 0] or (0,)
296 shape = (wave_info['npnts'],)
297 t = _numpy.dtype(_numpy.int8) # setup a safe default
298 if wave_info['type'] == 0: # text wave
299 shape = (waveDataSize,)
300 elif wave_info['type'] in TYPE_TABLE or wave_info['npnts']:
301 t = _numpy.dtype(TYPE_TABLE[wave_info['type']])
302 assert waveDataSize == wave_info['npnts'] * t.itemsize, (
303 '{}, {}, {}, {}'.format(
304 waveDataSize, wave_info['npnts'], t.itemsize, t))
307 if wave_info['npnts'] == 0:
310 tail_data = _array.array('f', b[-tail:])
311 data_b = buffer(buffer(tail_data) + f.read(waveDataSize-tail))
312 data = _numpy.ndarray(
314 dtype=t.newbyteorder(byteOrder),
320 pass # No post-data information
323 # * 16 bytes of padding
324 # * Optional wave note data
325 pad_b = buffer(f.read(16)) # skip the padding
326 _assert_null(pad_b, strict=strict)
327 bin_info['note'] = str(f.read(bin_info['noteSize'])).strip()
330 # * 16 bytes of padding
331 # * Optional wave note data
332 # * Optional wave dependency formula
333 """Excerpted from TN003:
335 A wave has a dependency formula if it has been bound by a
336 statement such as "wave0 := sin(x)". In this example, the
337 dependency formula is "sin(x)". The formula is stored with
338 no trailing null byte.
340 pad_b = buffer(f.read(16)) # skip the padding
341 _assert_null(pad_b, strict=strict)
342 bin_info['note'] = str(f.read(bin_info['noteSize'])).strip()
343 bin_info['formula'] = str(f.read(bin_info['formulaSize'])).strip()
346 # * Optional wave dependency formula
347 # * Optional wave note data
348 # * Optional extended data units data
349 # * Optional extended dimension units data
350 # * Optional dimension label data
351 # * String indices used for text waves only
352 """Excerpted from TN003:
354 dataUnits - Present in versions 1, 2, 3, 5. The dataUnits
355 field stores the units for the data represented by the
356 wave. It is a C string terminated with a null
357 character. This field supports units of 0 to 3 bytes. In
358 version 1, 2 and 3 files, longer units can not be
359 represented. In version 5 files, longer units can be
360 stored using the optional extended data units section of
363 xUnits - Present in versions 1, 2, 3. The xUnits field
364 stores the X units for a wave. It is a C string
365 terminated with a null character. This field supports
366 units of 0 to 3 bytes. In version 1, 2 and 3 files,
367 longer units can not be represented.
369 dimUnits - Present in version 5 only. This field is an
370 array of 4 strings, one for each possible wave
371 dimension. Each string supports units of 0 to 3
372 bytes. Longer units can be stored using the optional
373 extended dimension units section of the file.
375 bin_info['formula'] = str(f.read(bin_info['formulaSize'])).strip()
376 bin_info['note'] = str(f.read(bin_info['noteSize'])).strip()
377 bin_info['dataEUnits'] = str(f.read(bin_info['dataEUnitsSize'])).strip()
378 bin_info['dimEUnits'] = [
379 str(f.read(size)).strip() for size in bin_info['dimEUnitsSize']]
380 bin_info['dimLabels'] = []
381 for size in bin_info['dimLabelsSize']:
382 labels = str(f.read(size)).split(chr(0)) # split null-delimited strings
383 bin_info['dimLabels'].append([L for L in labels if len(L) > 0])
384 if wave_info['type'] == 0: # text wave
385 sIndices_data = f.read(bin_info['sIndicesSize'])
386 sIndices_count = bin_info['sIndicesSize']/4
387 bin_info['sIndices'] = _numpy.ndarray(
388 shape=(sIndices_count,),
390 _numpy.uint32()).newbyteorder(byteOrder),
391 buffer=sIndices_data,
395 if wave_info['type'] == 0: # text wave
396 # use sIndices to split data into strings
399 for i,offset in enumerate(bin_info['sIndices']):
401 string = data[start:offset]
402 strings.append(''.join(chr(x) for x in string))
404 elif offset == start:
407 raise ValueError((offset, bin_info['sIndices']))
408 data = _numpy.array(strings)
409 shape = [n for n in wave_info['nDim'] if n > 0] or (0,)
412 if not hasattr(filename, 'read'):
415 return data, bin_info, wave_info
419 raise NotImplementedError