3 # Copyright (C) 2010-2012 W. Trevor King <wking@drexel.edu>
5 # This file is part of Hooke.
7 # Hooke is free software: you can redistribute it and/or modify it under the
8 # terms of the GNU Lesser General Public License as published by the Free
9 # Software Foundation, either version 3 of the License, or (at your option) any
12 # Hooke is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
17 # You should have received a copy of the GNU Lesser General Public License
18 # along with Hooke. If not, see <http://www.gnu.org/licenses/>.
20 """igorbinarywave provides pure Python interface between IGOR Binary
21 Wave files and Numpy arrays.
23 This is basically a stand-alone package that we bundle into Hooke for
24 convenience. It is used by the mfp*d drivers, whose data is saved in
28 # Based on WaveMetric's Technical Note 003, "Igor Binary Format"
29 # ftp://ftp.wavemetrics.net/IgorPro/Technical_Notes/TN003.zip
30 # From ftp://ftp.wavemetrics.net/IgorPro/Technical_Notes/TN000.txt
31 # We place no restrictions on copying Technical Notes, with the
32 # exception that you cannot resell them. So read, enjoy, and
33 # share. We hope IGOR Technical Notes will provide you with lots of
34 # valuable information while you are developing IGOR applications.
48 """Represent a Structure field.
54 def __init__(self, format, name, default=None, help=None, count=1):
55 self.format = format # See the struct documentation
60 self.total_count = numpy.prod(count)
62 class Structure (struct.Struct):
63 """Represent a C structure.
65 A convenient wrapper around struct.Struct that uses Fields and
66 adds dict-handling methods for transparent name assignment.
75 Represent the C structure::
84 >>> from pprint import pprint
85 >>> thing = Structure(name='thing',
86 ... fields=[Field('h', 'version'), Field('l', 'size', count=3)])
87 >>> thing.set_byte_order('>')
88 >>> b = array.array('b', range(2+4*3))
89 >>> d = thing.unpack_dict_from(buffer=b)
91 {'size': array([ 33752069, 101124105, 168496141]), 'version': 1}
92 >>> [hex(x) for x in d['size']]
93 ['0x2030405L', '0x6070809L', '0xa0b0c0dL']
95 You can even get fancy with multi-dimensional arrays.
97 >>> thing = Structure(name='thing',
98 ... fields=[Field('h', 'version'), Field('l', 'size', count=(3,2))])
99 >>> thing.set_byte_order('>')
100 >>> b = array.array('b', range(2+4*3*2))
101 >>> d = thing.unpack_dict_from(buffer=b)
105 {'size': array([[ 33752069, 101124105],
106 [168496141, 235868177],
107 [303240213, 370612249]]),
110 def __init__(self, name, fields, byte_order='='):
111 # '=' for native byte order, standard size and alignment
112 # See http://docs.python.org/library/struct for details
115 self.set_byte_order(byte_order)
120 def set_byte_order(self, byte_order):
121 """Allow changing the format byte_order on the fly.
123 if (hasattr(self, 'format') and self.format != None
124 and self.format.startswith(byte_order)):
125 return # no need to change anything
127 for field in self.fields:
128 format.extend([field.format]*field.total_count)
129 struct.Struct.__init__(self, format=byte_order+''.join(format).replace('P', 'L'))
131 def _flatten_args(self, args):
132 # handle Field.count > 0
134 for a,f in zip(args, self.fields):
135 if f.total_count > 1:
141 def _unflatten_args(self, args):
142 # handle Field.count > 0
145 for f in self.fields:
146 if f.total_count > 1:
147 data = numpy.array(args[i:i+f.total_count])
148 data = data.reshape(f.count)
149 unflat_args.append(data)
151 unflat_args.append(args[i])
155 def pack(self, *args):
156 return struct.Struct.pack(self, *self._flatten_args(args))
158 def pack_into(self, buffer, offset, *args):
159 return struct.Struct.pack_into(self, buffer, offset,
160 *self._flatten_args(args))
162 def _clean_dict(self, dict):
163 for f in self.fields:
164 if f.name not in dict:
165 if f.default != None:
166 dict[f.name] = f.default
168 raise ValueError('%s field not set for %s'
169 % f.name, self.__class__.__name__)
172 def pack_dict(self, dict):
173 dict = self._clean_dict(dict)
174 return self.pack(*[dict[f.name] for f in self.fields])
176 def pack_dict_into(self, buffer, offset, dict={}):
177 dict = self._clean_dict(dict)
178 return self.pack_into(buffer, offset,
179 *[dict[f.name] for f in self.fields])
181 def unpack(self, string):
182 return self._unflatten_args(struct.Struct.unpack(self, string))
184 def unpack_from(self, buffer, offset=0):
185 return self._unflatten_args(
186 struct.Struct.unpack_from(self, buffer, offset))
188 def unpack_dict(self, string):
189 return dict(zip([f.name for f in self.fields],
190 self.unpack(string)))
192 def unpack_dict_from(self, buffer, offset=0):
193 return dict(zip([f.name for f in self.fields],
194 self.unpack_from(buffer, offset)))
197 # Numpy doesn't support complex integers by default, see
198 # http://mail.python.org/pipermail/python-dev/2002-April/022408.html
199 # http://mail.scipy.org/pipermail/numpy-discussion/2007-October/029447.html
200 # So we roll our own types. See
201 # http://docs.scipy.org/doc/numpy/user/basics.rec.html
202 # http://docs.scipy.org/doc/numpy/reference/generated/numpy.dtype.html
203 complexInt8 = numpy.dtype([('real', numpy.int8), ('imag', numpy.int8)])
204 complexInt16 = numpy.dtype([('real', numpy.int16), ('imag', numpy.int16)])
205 complexInt32 = numpy.dtype([('real', numpy.int32), ('imag', numpy.int32)])
206 complexUInt8 = numpy.dtype([('real', numpy.uint8), ('imag', numpy.uint8)])
207 complexUInt16 = numpy.dtype([('real', numpy.uint16), ('imag', numpy.uint16)])
208 complexUInt32 = numpy.dtype([('real', numpy.uint32), ('imag', numpy.uint32)])
211 # Begin IGOR constants and typedefs from IgorBin.h
214 TYPE_TABLE = { # (key: integer flag, value: numpy dtype)
215 0:None, # Text wave, not handled in ReadWave.c
216 1:numpy.complex, # NT_CMPLX, makes number complex.
217 2:numpy.float32, # NT_FP32, 32 bit fp numbers.
219 4:numpy.float64, # NT_FP64, 64 bit fp numbers.
221 8:numpy.int8, # NT_I8, 8 bit signed integer. Requires Igor Pro
224 0x10:numpy.int16,# NT_I16, 16 bit integer numbers. Requires Igor
227 0x20:numpy.int32,# NT_I32, 32 bit integer numbers. Requires Igor
230 # 0x40:None, # NT_UNSIGNED, Makes above signed integers
231 # # unsigned. Requires Igor Pro 3.0 or later.
244 BinHeaderCommon = Structure( # WTK: this one is mine.
245 name='BinHeaderCommon',
247 Field('h', 'version', help='Version number for backwards compatibility.'),
250 BinHeader1 = Structure(
253 Field('h', 'version', help='Version number for backwards compatibility.'),
254 Field('l', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'),
255 Field('h', 'checksum', help='Checksum over this header and the wave header.'),
258 BinHeader2 = Structure(
261 Field('h', 'version', help='Version number for backwards compatibility.'),
262 Field('l', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'),
263 Field('l', 'noteSize', help='The size of the note text.'),
264 Field('l', 'pictSize', default=0, help='Reserved. Write zero. Ignore on read.'),
265 Field('h', 'checksum', help='Checksum over this header and the wave header.'),
268 BinHeader3 = Structure(
271 Field('h', 'version', help='Version number for backwards compatibility.'),
272 Field('h', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'),
273 Field('l', 'noteSize', help='The size of the note text.'),
274 Field('l', 'formulaSize', help='The size of the dependency formula, if any.'),
275 Field('l', 'pictSize', default=0, help='Reserved. Write zero. Ignore on read.'),
276 Field('h', 'checksum', help='Checksum over this header and the wave header.'),
279 BinHeader5 = Structure(
282 Field('h', 'version', help='Version number for backwards compatibility.'),
283 Field('h', 'checksum', help='Checksum over this header and the wave header.'),
284 Field('l', 'wfmSize', help='The size of the WaveHeader5 data structure plus the wave data.'),
285 Field('l', 'formulaSize', help='The size of the dependency formula, if any.'),
286 Field('l', 'noteSize', help='The size of the note text.'),
287 Field('l', 'dataEUnitsSize', help='The size of optional extended data units.'),
288 Field('l', 'dimEUnitsSize', help='The size of optional extended dimension units.', count=MAXDIMS),
289 Field('l', 'dimLabelsSize', help='The size of optional dimension labels.', count=MAXDIMS),
290 Field('l', 'sIndicesSize', help='The size of string indicies if this is a text wave.'),
291 Field('l', 'optionsSize1', default=0, help='Reserved. Write zero. Ignore on read.'),
292 Field('l', 'optionsSize2', default=0, help='Reserved. Write zero. Ignore on read.'),
297 MAX_WAVE_NAME2 = 18 # Maximum length of wave name in version 1 and 2
298 # files. Does not include the trailing null.
299 MAX_WAVE_NAME5 = 31 # Maximum length of wave name in version 5
300 # files. Does not include the trailing null.
303 # Header to an array of waveform data.
305 WaveHeader2 = Structure(
308 Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'),
309 Field('P', 'next', default=0, help='Used in memory only. Write zero. Ignore on read.'),
310 Field('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME2+2),
311 Field('h', 'whVersion', default=0, help='Write 0. Ignore on read.'),
312 Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'),
313 Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'),
314 Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1),
315 Field('c', 'xUnits', default=0, help='Natural x-axis units go here - null if none.', count=MAX_UNIT_CHARS+1),
316 Field('l', 'npnts', help='Number of data points in wave.'),
317 Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
318 Field('d', 'hsA', help='X value for point p = hsA*p + hsB'),
319 Field('d', 'hsB', help='X value for point p = hsA*p + hsB'),
320 Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
321 Field('h', 'swModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
322 Field('h', 'fsValid', help='True if full scale values have meaning.'),
323 Field('d', 'topFullScale', help='The min full scale value for wave.'), # sic, 'min' should probably be 'max'
324 Field('d', 'botFullScale', help='The min full scale value for wave.'),
325 Field('c', 'useBits', default=0, help='Used in memory only. Write zero. Ignore on read.'),
326 Field('c', 'kindBits', default=0, help='Reserved. Write zero. Ignore on read.'),
327 Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'),
328 Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'),
329 Field('L', 'creationDate', help='DateTime of creation. Not used in version 1 files.'),
330 Field('c', 'wUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=2),
331 Field('L', 'modDate', help='DateTime of last modification.'),
332 Field('P', 'waveNoteH', help='Used in memory only. Write zero. Ignore on read.'),
333 Field('f', 'wData', help='The start of the array of waveform data.', count=4),
336 WaveHeader5 = Structure(
339 Field('P', 'next', help='link to next wave in linked list.'),
340 Field('L', 'creationDate', help='DateTime of creation.'),
341 Field('L', 'modDate', help='DateTime of last modification.'),
342 Field('l', 'npnts', help='Total number of points (multiply dimensions up to first zero).'),
343 Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'),
344 Field('h', 'dLock', default=0, help='Reserved. Write zero. Ignore on read.'),
345 Field('c', 'whpad1', default=0, help='Reserved. Write zero. Ignore on read.', count=6),
346 Field('h', 'whVersion', default=1, help='Write 1. Ignore on read.'),
347 Field('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME5+1),
348 Field('l', 'whpad2', default=0, help='Reserved. Write zero. Ignore on read.'),
349 Field('P', 'dFolder', default=0, help='Used in memory only. Write zero. Ignore on read.'),
350 # Dimensioning info. [0] == rows, [1] == cols etc
351 Field('l', 'nDim', help='Number of of items in a dimension -- 0 means no data.', count=MAXDIMS),
352 Field('d', 'sfA', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS),
353 Field('d', 'sfB', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS),
355 Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1),
356 Field('c', 'dimUnits', default=0, help='Natural dimension units go here - null if none.', count=(MAXDIMS, MAX_UNIT_CHARS+1)),
357 Field('h', 'fsValid', help='TRUE if full scale values have meaning.'),
358 Field('h', 'whpad3', default=0, help='Reserved. Write zero. Ignore on read.'),
359 Field('d', 'topFullScale', help='The max and max full scale value for wave'), # sic, probably "max and min"
360 Field('d', 'botFullScale', help='The max and max full scale value for wave.'), # sic, probably "max and min"
361 Field('P', 'dataEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.'),
362 Field('P', 'dimEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS),
363 Field('P', 'dimLabels', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS),
364 Field('P', 'waveNoteH', default=0, help='Used in memory only. Write zero. Ignore on read.'),
365 Field('l', 'whUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=16),
366 # The following stuff is considered private to Igor.
367 Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
368 Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
369 Field('h', 'swModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
370 Field('c', 'useBits', default=0, help='Used in memory only. Write zero. Ignore on read.'),
371 Field('c', 'kindBits', default=0, help='Reserved. Write zero. Ignore on read.'),
372 Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'),
373 Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'),
374 Field('h', 'whpad4', default=0, help='Reserved. Write zero. Ignore on read.'),
375 Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'),
376 Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'),
377 Field('P', 'sIndices', default=0, help='Used in memory only. Write zero. Ignore on read.'),
378 Field('f', 'wData', help='The start of the array of data. Must be 64 bit aligned.', count=1),
381 # End IGOR constants and typedefs from IgorBin.h
383 # Begin functions from ReadWave.c
385 def need_to_reorder_bytes(version):
386 # If the low order byte of the version field of the BinHeader
387 # structure is zero then the file is from a platform that uses
388 # different byte-ordering and therefore all data will need to be
390 return version & 0xFF == 0
392 def byte_order(needToReorderBytes):
393 little_endian = sys.byteorder == 'little'
394 if needToReorderBytes:
395 little_endian = not little_endian
397 return '<' # little-endian
398 return '>' # big-endian
400 def version_structs(version, byte_order):
414 raise ValueError('This does not appear to be a valid Igor binary wave file. The version field = %d.\n', version);
415 checkSumSize = bin.size + wave.size
417 checkSumSize -= 4 # Version 5 checksum does not include the wData field.
418 bin.set_byte_order(byte_order)
419 wave.set_byte_order(byte_order)
420 return (bin, wave, checkSumSize)
422 def checksum(buffer, byte_order, oldcksum, numbytes):
424 (numbytes/2,), # 2 bytes to a short -- ignore trailing odd byte
425 dtype=numpy.dtype(byte_order+'h'),
428 if oldcksum > 2**31: # fake the C implementation's int rollover
432 return oldcksum & 0xffff
434 # Translated from ReadWave()
435 def loadibw(filename, strict=True):
436 if hasattr(filename, 'read'):
437 f = filename # filename is actually a stream object
439 f = open(filename, 'rb')
441 b = buffer(f.read(BinHeaderCommon.size))
442 version = BinHeaderCommon.unpack_dict_from(b)['version']
443 needToReorderBytes = need_to_reorder_bytes(version)
444 byteOrder = byte_order(needToReorderBytes)
446 if needToReorderBytes:
447 BinHeaderCommon.set_byte_order(byteOrder)
448 version = BinHeaderCommon.unpack_dict_from(b)['version']
449 bin_struct,wave_struct,checkSumSize = version_structs(version, byteOrder)
451 b = buffer(b + f.read(bin_struct.size + wave_struct.size - BinHeaderCommon.size))
452 c = checksum(b, byteOrder, 0, checkSumSize)
454 raise ValueError('Error in checksum - should be 0, is %d. This does not appear to be a valid Igor binary wave file.' % c)
455 bin_info = bin_struct.unpack_dict_from(b)
456 wave_info = wave_struct.unpack_dict_from(b, offset=bin_struct.size)
457 if wave_info['type'] == 0:
458 raise NotImplementedError('Text wave')
459 if version in [1,2,3]:
460 tail = 16 # 16 = size of wData field in WaveHeader2 structure
461 waveDataSize = bin_info['wfmSize'] - wave_struct.size
462 # = bin_info['wfmSize']-16 - (wave_struct.size - tail)
464 assert version == 5, version
465 tail = 4 # 4 = size of wData field in WaveHeader5 structure
466 waveDataSize = bin_info['wfmSize'] - (wave_struct.size - tail)
467 # dtype() wrapping to avoid numpy.generic and
468 # getset_descriptor issues with the builtin Numpy types
469 # (e.g. int32). It has no effect on our local complex
471 t = numpy.dtype(TYPE_TABLE[wave_info['type']])
472 assert waveDataSize == wave_info['npnts'] * t.itemsize, \
473 ('%d, %d, %d, %s' % (waveDataSize, wave_info['npnts'], t.itemsize, t))
474 tail_data = array.array('f', b[-tail:])
475 data_b = buffer(buffer(tail_data) + f.read(waveDataSize-tail))
477 shape = [n for n in wave_info['nDim'] if n > 0]
479 shape = (wave_info['npnts'],)
480 data = numpy.ndarray(
482 dtype=t.newbyteorder(byteOrder),
488 pass # No post-data information
491 # * 16 bytes of padding
492 # * Optional wave note data
493 pad_b = buffer(f.read(16)) # skip the padding
496 assert max(pad_b) == 0, pad_b
498 print sys.stderr, 'warning: post-data padding not zero: %s.' % pad_b
499 bin_info['note'] = str(f.read(bin_info['noteSize'])).strip()
502 # * 16 bytes of padding
503 # * Optional wave note data
504 # * Optional wave dependency formula
505 """Excerpted from TN003:
507 A wave has a dependency formula if it has been bound by a
508 statement such as "wave0 := sin(x)". In this example, the
509 dependency formula is "sin(x)". The formula is stored with
510 no trailing null byte.
512 pad_b = buffer(f.read(16)) # skip the padding
515 assert max(pad_b) == 0, pad_b
517 print sys.stderr, 'warning: post-data padding not zero: %s.' % pad_b
518 bin_info['note'] = str(f.read(bin_info['noteSize'])).strip()
519 bin_info['formula'] = str(f.read(bin_info['formulaSize'])).strip()
522 # * Optional wave dependency formula
523 # * Optional wave note data
524 # * Optional extended data units data
525 # * Optional extended dimension units data
526 # * Optional dimension label data
527 # * String indices used for text waves only
528 """Excerpted from TN003:
530 dataUnits - Present in versions 1, 2, 3, 5. The dataUnits
531 field stores the units for the data represented by the
532 wave. It is a C string terminated with a null
533 character. This field supports units of 0 to 3 bytes. In
534 version 1, 2 and 3 files, longer units can not be
535 represented. In version 5 files, longer units can be
536 stored using the optional extended data units section of
539 xUnits - Present in versions 1, 2, 3. The xUnits field
540 stores the X units for a wave. It is a C string
541 terminated with a null character. This field supports
542 units of 0 to 3 bytes. In version 1, 2 and 3 files,
543 longer units can not be represented.
545 dimUnits - Present in version 5 only. This field is an
546 array of 4 strings, one for each possible wave
547 dimension. Each string supports units of 0 to 3
548 bytes. Longer units can be stored using the optional
549 extended dimension units section of the file.
551 bin_info['formula'] = str(f.read(bin_info['formulaSize'])).strip()
552 bin_info['note'] = str(f.read(bin_info['noteSize'])).strip()
553 bin_info['dataEUnits'] = str(f.read(bin_info['dataEUnitsSize'])).strip()
554 bin_info['dimEUnits'] = [
555 str(f.read(size)).strip() for size in bin_info['dimEUnitsSize']]
556 bin_info['dimLabels'] = []
557 for size in bin_info['dimLabelsSize']:
558 labels = str(f.read(size)).split(chr(0)) # split null-delimited strings
559 bin_info['dimLabels'].append([L for L in labels if len(L) > 0])
560 if wave_info['type'] == 0: # text wave
561 bin_info['sIndices'] = f.read(bin_info['sIndicesSize'])
564 if not hasattr(filename, 'read'):
567 return data, bin_info, wave_info
570 def saveibw(filename):
571 raise NotImplementedError
574 if __name__ == '__main__':
575 """IBW -> ASCII conversion
580 p = optparse.OptionParser(version=__version__)
582 p.add_option('-f', '--infile', dest='infile', metavar='FILE',
583 default='-', help='Input IGOR Binary Wave (.ibw) file.')
584 p.add_option('-o', '--outfile', dest='outfile', metavar='FILE',
585 default='-', help='File for ASCII output.')
586 p.add_option('-v', '--verbose', dest='verbose', default=0,
587 action='count', help='Increment verbosity')
588 p.add_option('-n', '--not-strict', dest='strict', default=True,
589 action='store_false', help='Attempt to parse invalid IBW files.')
590 p.add_option('-t', '--test', dest='test', default=False,
591 action='store_true', help='Run internal tests and exit.')
593 options,args = p.parse_args()
595 if options.test == True:
597 num_failures,num_tests = doctest.testmod(verbose=options.verbose)
598 sys.exit(min(num_failures, 127))
600 if len(args) > 0 and options.infile == None:
601 options.infile = args[0]
602 if options.infile == '-':
603 options.infile = sys.stdin
604 if options.outfile == '-':
605 options.outfile = sys.stdout
607 data,bin_info,wave_info = loadibw(options.infile, strict=options.strict)
608 numpy.savetxt(options.outfile, data, fmt='%g', delimiter='\t')
609 if options.verbose > 0:
611 pprint.pprint(bin_info)
612 pprint.pprint(wave_info)