From d20d217c2d2806c32bb40dbd68c4b0aea35e5fd6 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 16 Jul 2012 23:23:47 -0400 Subject: [PATCH] Split struct and util modules out of binarywave. --- igor/binarywave.py | 495 +++++++++++++-------------------------------- igor/struct.py | 181 +++++++++++++++++ igor/util.py | 53 +++++ 3 files changed, 376 insertions(+), 353 deletions(-) create mode 100644 igor/struct.py create mode 100644 igor/util.py diff --git a/igor/binarywave.py b/igor/binarywave.py index 856c85f..a79c6be 100644 --- a/igor/binarywave.py +++ b/igor/binarywave.py @@ -26,181 +26,15 @@ # share. We hope IGOR Technical Notes will provide you with lots of # valuable information while you are developing IGOR applications. -import array -import struct -import sys -import types - -import numpy - - -_buffer = buffer # save builtin buffer for clobbered situations - - -class Field (object): - """Represent a Structure field. - - See Also - -------- - Structure - """ - def __init__(self, format, name, default=None, help=None, count=1): - self.format = format # See the struct documentation - self.name = name - self.default = None - self.help = help - self.count = count - self.total_count = numpy.prod(count) - -class Structure (struct.Struct): - """Represent a C structure. - - A convenient wrapper around struct.Struct that uses Fields and - adds dict-handling methods for transparent name assignment. - - See Also - -------- - Field - - Examples - -------- - - Represent the C structure:: - - struct thing { - short version; - long size[3]; - } - - As - - >>> from pprint import pprint - >>> thing = Structure(name='thing', - ... fields=[Field('h', 'version'), Field('l', 'size', count=3)]) - >>> thing.set_byte_order('>') - >>> b = array.array('b', range(2+4*3)) - >>> d = thing.unpack_dict_from(buffer=b) - >>> pprint(d) - {'size': array([ 33752069, 101124105, 168496141]), 'version': 1} - >>> [hex(x) for x in d['size']] - ['0x2030405L', '0x6070809L', '0xa0b0c0dL'] - - You can even get fancy with multi-dimensional arrays. - - >>> thing = Structure(name='thing', - ... fields=[Field('h', 'version'), Field('l', 'size', count=(3,2))]) - >>> thing.set_byte_order('>') - >>> b = array.array('b', range(2+4*3*2)) - >>> d = thing.unpack_dict_from(buffer=b) - >>> d['size'].shape - (3, 2) - >>> pprint(d) - {'size': array([[ 33752069, 101124105], - [168496141, 235868177], - [303240213, 370612249]]), - 'version': 1} - """ - def __init__(self, name, fields, byte_order='='): - # '=' for native byte order, standard size and alignment - # See http://docs.python.org/library/struct for details - self.name = name - self.fields = fields - self.set_byte_order(byte_order) - - def __str__(self): - return self.name - - def set_byte_order(self, byte_order): - """Allow changing the format byte_order on the fly. - """ - if (hasattr(self, 'format') and self.format != None - and self.format.startswith(byte_order)): - return # no need to change anything - format = [] - for field in self.fields: - format.extend([field.format]*field.total_count) - struct.Struct.__init__(self, format=byte_order+''.join(format).replace('P', 'L')) - - def _flatten_args(self, args): - # handle Field.count > 0 - flat_args = [] - for a,f in zip(args, self.fields): - if f.total_count > 1: - flat_args.extend(a) - else: - flat_args.append(a) - return flat_args - - def _unflatten_args(self, args): - # handle Field.count > 0 - unflat_args = [] - i = 0 - for f in self.fields: - if f.total_count > 1: - data = numpy.array(args[i:i+f.total_count]) - data = data.reshape(f.count) - unflat_args.append(data) - else: - unflat_args.append(args[i]) - i += f.total_count - return unflat_args - - def pack(self, *args): - return struct.Struct.pack(self, *self._flatten_args(args)) - - def pack_into(self, buffer, offset, *args): - return struct.Struct.pack_into(self, buffer, offset, - *self._flatten_args(args)) - - def _clean_dict(self, dict): - for f in self.fields: - if f.name not in dict: - if f.default != None: - dict[f.name] = f.default - else: - raise ValueError('{} field not set for {}'.format( - f.name, self.__class__.__name__)) - return dict - - def pack_dict(self, dict): - dict = self._clean_dict(dict) - return self.pack(*[dict[f.name] for f in self.fields]) - - def pack_dict_into(self, buffer, offset, dict={}): - dict = self._clean_dict(dict) - return self.pack_into(buffer, offset, - *[dict[f.name] for f in self.fields]) - - def unpack(self, string): - return self._unflatten_args(struct.Struct.unpack(self, string)) - - def unpack_from(self, buffer, offset=0): - try: - args = struct.Struct.unpack_from(self, buffer, offset) - except struct.error as e: - if not self.name in ('WaveHeader2', 'WaveHeader5'): - raise - # HACK! For WaveHeader5, when npnts is 0, wData is - # optional. If we couldn't unpack the structure, fill in - # wData with zeros and try again, asserting that npnts is - # zero. - if len(buffer) - offset < self.size: - # missing wData? Pad with zeros - buffer += _buffer('\x00'*(self.size + offset - len(buffer))) - args = struct.Struct.unpack_from(self, buffer, offset) - unpacked = self._unflatten_args(args) - data = dict(zip([f.name for f in self.fields], - unpacked)) - assert data['npnts'] == 0, data['npnts'] - return self._unflatten_args(args) - - def unpack_dict(self, string): - return dict(zip([f.name for f in self.fields], - self.unpack(string))) - - def unpack_dict_from(self, buffer, offset=0): - return dict(zip([f.name for f in self.fields], - self.unpack_from(buffer, offset))) +import array as _array +import sys as _sys +import types as _types + +import numpy as _numpy + +from .struct import Structure as _Structure +from .struct import Field as _Field +from .util import assert_null as _assert_null # Numpy doesn't support complex integers by default, see @@ -209,40 +43,41 @@ class Structure (struct.Struct): # So we roll our own types. See # http://docs.scipy.org/doc/numpy/user/basics.rec.html # http://docs.scipy.org/doc/numpy/reference/generated/numpy.dtype.html -complexInt8 = numpy.dtype([('real', numpy.int8), ('imag', numpy.int8)]) -complexInt16 = numpy.dtype([('real', numpy.int16), ('imag', numpy.int16)]) -complexInt32 = numpy.dtype([('real', numpy.int32), ('imag', numpy.int32)]) -complexUInt8 = numpy.dtype([('real', numpy.uint8), ('imag', numpy.uint8)]) -complexUInt16 = numpy.dtype([('real', numpy.uint16), ('imag', numpy.uint16)]) -complexUInt32 = numpy.dtype([('real', numpy.uint32), ('imag', numpy.uint32)]) - +complexInt8 = _numpy.dtype([('real', _numpy.int8), ('imag', _numpy.int8)]) +complexInt16 = _numpy.dtype([('real', _numpy.int16), ('imag', _numpy.int16)]) +complexInt32 = _numpy.dtype([('real', _numpy.int32), ('imag', _numpy.int32)]) +complexUInt8 = _numpy.dtype([('real', _numpy.uint8), ('imag', _numpy.uint8)]) +complexUInt16 = _numpy.dtype( + [('real', _numpy.uint16), ('imag', _numpy.uint16)]) +complexUInt32 = _numpy.dtype( + [('real', _numpy.uint32), ('imag', _numpy.uint32)]) # Begin IGOR constants and typedefs from IgorBin.h # From IgorMath.h -TYPE_TABLE = { # (key: integer flag, value: numpy dtype) - 0:None, # Text wave, not handled in ReadWave.c - 1:numpy.complex, # NT_CMPLX, makes number complex. - 2:numpy.float32, # NT_FP32, 32 bit fp numbers. - 3:numpy.complex64, - 4:numpy.float64, # NT_FP64, 64 bit fp numbers. - 5:numpy.complex128, - 8:numpy.int8, # NT_I8, 8 bit signed integer. Requires Igor Pro - # 2.0 or later. +TYPE_TABLE = { # (key: integer flag, value: numpy dtype) + 0:None, # Text wave, not handled in ReadWave.c + 1:_numpy.complex, # NT_CMPLX, makes number complex. + 2:_numpy.float32, # NT_FP32, 32 bit fp numbers. + 3:_numpy.complex64, + 4:_numpy.float64, # NT_FP64, 64 bit fp numbers. + 5:_numpy.complex128, + 8:_numpy.int8, # NT_I8, 8 bit signed integer. Requires Igor Pro + # 2.0 or later. 9:complexInt8, - 0x10:numpy.int16,# NT_I16, 16 bit integer numbers. Requires Igor - # Pro 2.0 or later. + 0x10:_numpy.int16,# NT_I16, 16 bit integer numbers. Requires Igor + # Pro 2.0 or later. 0x11:complexInt16, - 0x20:numpy.int32,# NT_I32, 32 bit integer numbers. Requires Igor - # Pro 2.0 or later. + 0x20:_numpy.int32,# NT_I32, 32 bit integer numbers. Requires Igor + # Pro 2.0 or later. 0x21:complexInt32, -# 0x40:None, # NT_UNSIGNED, Makes above signed integers -# # unsigned. Requires Igor Pro 3.0 or later. - 0x48:numpy.uint8, +# 0x40:None, # NT_UNSIGNED, Makes above signed integers +# # unsigned. Requires Igor Pro 3.0 or later. + 0x48:_numpy.uint8, 0x49:complexUInt8, - 0x50:numpy.uint16, + 0x50:_numpy.uint16, 0x51:complexUInt16, - 0x60:numpy.uint32, + 0x60:_numpy.uint32, 0x61:complexUInt32, } @@ -250,55 +85,55 @@ TYPE_TABLE = { # (key: integer flag, value: numpy dtype) MAXDIMS = 4 # From binary.h -BinHeaderCommon = Structure( # WTK: this one is mine. +BinHeaderCommon = _Structure( # WTK: this one is mine. name='BinHeaderCommon', fields=[ - Field('h', 'version', help='Version number for backwards compatibility.'), + _Field('h', 'version', help='Version number for backwards compatibility.'), ]) -BinHeader1 = Structure( +BinHeader1 = _Structure( name='BinHeader1', fields=[ - Field('h', 'version', help='Version number for backwards compatibility.'), - Field('l', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'), - Field('h', 'checksum', help='Checksum over this header and the wave header.'), + _Field('h', 'version', help='Version number for backwards compatibility.'), + _Field('l', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'), + _Field('h', 'checksum', help='Checksum over this header and the wave header.'), ]) -BinHeader2 = Structure( +BinHeader2 = _Structure( name='BinHeader2', fields=[ - Field('h', 'version', help='Version number for backwards compatibility.'), - Field('l', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'), - Field('l', 'noteSize', help='The size of the note text.'), - Field('l', 'pictSize', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('h', 'checksum', help='Checksum over this header and the wave header.'), + _Field('h', 'version', help='Version number for backwards compatibility.'), + _Field('l', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'), + _Field('l', 'noteSize', help='The size of the note text.'), + _Field('l', 'pictSize', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('h', 'checksum', help='Checksum over this header and the wave header.'), ]) -BinHeader3 = Structure( +BinHeader3 = _Structure( name='BinHeader3', fields=[ - Field('h', 'version', help='Version number for backwards compatibility.'), - Field('h', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'), - Field('l', 'noteSize', help='The size of the note text.'), - Field('l', 'formulaSize', help='The size of the dependency formula, if any.'), - Field('l', 'pictSize', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('h', 'checksum', help='Checksum over this header and the wave header.'), + _Field('h', 'version', help='Version number for backwards compatibility.'), + _Field('h', 'wfmSize', help='The size of the WaveHeader2 data structure plus the wave data plus 16 bytes of padding.'), + _Field('l', 'noteSize', help='The size of the note text.'), + _Field('l', 'formulaSize', help='The size of the dependency formula, if any.'), + _Field('l', 'pictSize', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('h', 'checksum', help='Checksum over this header and the wave header.'), ]) -BinHeader5 = Structure( +BinHeader5 = _Structure( name='BinHeader5', fields=[ - Field('h', 'version', help='Version number for backwards compatibility.'), - Field('h', 'checksum', help='Checksum over this header and the wave header.'), - Field('l', 'wfmSize', help='The size of the WaveHeader5 data structure plus the wave data.'), - Field('l', 'formulaSize', help='The size of the dependency formula, if any.'), - Field('l', 'noteSize', help='The size of the note text.'), - Field('l', 'dataEUnitsSize', help='The size of optional extended data units.'), - Field('l', 'dimEUnitsSize', help='The size of optional extended dimension units.', count=MAXDIMS), - Field('l', 'dimLabelsSize', help='The size of optional dimension labels.', count=MAXDIMS), - Field('l', 'sIndicesSize', help='The size of string indicies if this is a text wave.'), - Field('l', 'optionsSize1', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('l', 'optionsSize2', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('h', 'version', help='Version number for backwards compatibility.'), + _Field('h', 'checksum', help='Checksum over this header and the wave header.'), + _Field('l', 'wfmSize', help='The size of the WaveHeader5 data structure plus the wave data.'), + _Field('l', 'formulaSize', help='The size of the dependency formula, if any.'), + _Field('l', 'noteSize', help='The size of the note text.'), + _Field('l', 'dataEUnitsSize', help='The size of optional extended data units.'), + _Field('l', 'dimEUnitsSize', help='The size of optional extended dimension units.', count=MAXDIMS), + _Field('l', 'dimLabelsSize', help='The size of optional dimension labels.', count=MAXDIMS), + _Field('l', 'sIndicesSize', help='The size of string indicies if this is a text wave.'), + _Field('l', 'optionsSize1', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('l', 'optionsSize2', default=0, help='Reserved. Write zero. Ignore on read.'), ]) @@ -311,80 +146,80 @@ MAX_UNIT_CHARS = 3 # Header to an array of waveform data. -WaveHeader2 = Structure( +WaveHeader2 = _Structure( name='WaveHeader2', fields=[ - Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'), - Field('P', 'next', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME2+2), - Field('h', 'whVersion', default=0, help='Write 0. Ignore on read.'), - Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1), - Field('c', 'xUnits', default=0, help='Natural x-axis units go here - null if none.', count=MAX_UNIT_CHARS+1), - Field('l', 'npnts', help='Number of data points in wave.'), - Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('d', 'hsA', help='X value for point p = hsA*p + hsB'), - Field('d', 'hsB', help='X value for point p = hsA*p + hsB'), - Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('h', 'swModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('h', 'fsValid', help='True if full scale values have meaning.'), - Field('d', 'topFullScale', help='The min full scale value for wave.'), # sic, 'min' should probably be 'max' - Field('d', 'botFullScale', help='The min full scale value for wave.'), - Field('c', 'useBits', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('c', 'kindBits', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('L', 'creationDate', help='DateTime of creation. Not used in version 1 files.'), - Field('c', 'wUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=2), - Field('L', 'modDate', help='DateTime of last modification.'), - Field('P', 'waveNoteH', help='Used in memory only. Write zero. Ignore on read.'), - Field('f', 'wData', help='The start of the array of waveform data.', count=4), + _Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'), + _Field('P', 'next', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME2+2), + _Field('h', 'whVersion', default=0, help='Write 0. Ignore on read.'), + _Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1), + _Field('c', 'xUnits', default=0, help='Natural x-axis units go here - null if none.', count=MAX_UNIT_CHARS+1), + _Field('l', 'npnts', help='Number of data points in wave.'), + _Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('d', 'hsA', help='X value for point p = hsA*p + hsB'), + _Field('d', 'hsB', help='X value for point p = hsA*p + hsB'), + _Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('h', 'swModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('h', 'fsValid', help='True if full scale values have meaning.'), + _Field('d', 'topFullScale', help='The min full scale value for wave.'), # sic, 'min' should probably be 'max' + _Field('d', 'botFullScale', help='The min full scale value for wave.'), + _Field('c', 'useBits', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('c', 'kindBits', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('L', 'creationDate', help='DateTime of creation. Not used in version 1 files.'), + _Field('c', 'wUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=2), + _Field('L', 'modDate', help='DateTime of last modification.'), + _Field('P', 'waveNoteH', help='Used in memory only. Write zero. Ignore on read.'), + _Field('f', 'wData', help='The start of the array of waveform data.', count=4), ]) -WaveHeader5 = Structure( +WaveHeader5 = _Structure( name='WaveHeader5', fields=[ - Field('P', 'next', help='link to next wave in linked list.'), - Field('L', 'creationDate', help='DateTime of creation.'), - Field('L', 'modDate', help='DateTime of last modification.'), - Field('l', 'npnts', help='Total number of points (multiply dimensions up to first zero).'), - Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'), - Field('h', 'dLock', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('c', 'whpad1', default=0, help='Reserved. Write zero. Ignore on read.', count=6), - Field('h', 'whVersion', default=1, help='Write 1. Ignore on read.'), - Field('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME5+1), - Field('l', 'whpad2', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('P', 'dFolder', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('P', 'next', help='link to next wave in linked list.'), + _Field('L', 'creationDate', help='DateTime of creation.'), + _Field('L', 'modDate', help='DateTime of last modification.'), + _Field('l', 'npnts', help='Total number of points (multiply dimensions up to first zero).'), + _Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'), + _Field('h', 'dLock', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('c', 'whpad1', default=0, help='Reserved. Write zero. Ignore on read.', count=6), + _Field('h', 'whVersion', default=1, help='Write 1. Ignore on read.'), + _Field('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME5+1), + _Field('l', 'whpad2', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('P', 'dFolder', default=0, help='Used in memory only. Write zero. Ignore on read.'), # Dimensioning info. [0] == rows, [1] == cols etc - Field('l', 'nDim', help='Number of of items in a dimension -- 0 means no data.', count=MAXDIMS), - Field('d', 'sfA', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS), - Field('d', 'sfB', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS), + _Field('l', 'nDim', help='Number of of items in a dimension -- 0 means no data.', count=MAXDIMS), + _Field('d', 'sfA', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS), + _Field('d', 'sfB', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS), # SI units - Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1), - Field('c', 'dimUnits', default=0, help='Natural dimension units go here - null if none.', count=(MAXDIMS, MAX_UNIT_CHARS+1)), - Field('h', 'fsValid', help='TRUE if full scale values have meaning.'), - Field('h', 'whpad3', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('d', 'topFullScale', help='The max and max full scale value for wave'), # sic, probably "max and min" - Field('d', 'botFullScale', help='The max and max full scale value for wave.'), # sic, probably "max and min" - Field('P', 'dataEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('P', 'dimEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS), - Field('P', 'dimLabels', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS), - Field('P', 'waveNoteH', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('l', 'whUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=16), + _Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1), + _Field('c', 'dimUnits', default=0, help='Natural dimension units go here - null if none.', count=(MAXDIMS, MAX_UNIT_CHARS+1)), + _Field('h', 'fsValid', help='TRUE if full scale values have meaning.'), + _Field('h', 'whpad3', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('d', 'topFullScale', help='The max and max full scale value for wave'), # sic, probably "max and min" + _Field('d', 'botFullScale', help='The max and max full scale value for wave.'), # sic, probably "max and min" + _Field('P', 'dataEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('P', 'dimEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS), + _Field('P', 'dimLabels', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS), + _Field('P', 'waveNoteH', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('l', 'whUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=16), # The following stuff is considered private to Igor. - Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('h', 'swModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('c', 'useBits', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('c', 'kindBits', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('h', 'whpad4', default=0, help='Reserved. Write zero. Ignore on read.'), - Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('P', 'sIndices', default=0, help='Used in memory only. Write zero. Ignore on read.'), - Field('f', 'wData', help='The start of the array of data. Must be 64 bit aligned.', count=1), + _Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('h', 'swModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('c', 'useBits', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('c', 'kindBits', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('h', 'whpad4', default=0, help='Reserved. Write zero. Ignore on read.'), + _Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('P', 'sIndices', default=0, help='Used in memory only. Write zero. Ignore on read.'), + _Field('f', 'wData', help='The start of the array of data. Must be 64 bit aligned.', count=1), ]) # End IGOR constants and typedefs from IgorBin.h @@ -399,7 +234,7 @@ def need_to_reorder_bytes(version): return version & 0xFF == 0 def byte_order(needToReorderBytes): - little_endian = sys.byteorder == 'little' + little_endian = _sys.byteorder == 'little' if needToReorderBytes: little_endian = not little_endian if little_endian: @@ -431,9 +266,9 @@ def version_structs(version, byte_order): return (bin, wave, checkSumSize) def checksum(buffer, byte_order, oldcksum, numbytes): - x = numpy.ndarray( + x = _numpy.ndarray( (numbytes/2,), # 2 bytes to a short -- ignore trailing odd byte - dtype=numpy.dtype(byte_order+'h'), + dtype=_numpy.dtype(byte_order+'h'), buffer=buffer) oldcksum += x.sum() if oldcksum > 2**31: # fake the C implementation's int rollover @@ -442,52 +277,6 @@ def checksum(buffer, byte_order, oldcksum, numbytes): oldcksum -= 2**31 return oldcksum & 0xffff -def hex_bytes(buffer, spaces=None): - r"""Pretty-printing for binary buffers. - - >>> hex_bytes(buffer('\x00\x01\x02\x03\x04')) - '0001020304' - >>> hex_bytes(buffer('\x00\x01\x02\x03\x04'), spaces=1) - '00 01 02 03 04' - >>> hex_bytes(buffer('\x00\x01\x02\x03\x04'), spaces=2) - '0001 0203 04' - >>> hex_bytes(buffer('\x00\x01\x02\x03\x04\x05\x06'), spaces=2) - '0001 0203 0405 06' - >>> hex_bytes(buffer('\x00\x01\x02\x03\x04\x05\x06'), spaces=3) - '000102 030405 06' - """ - hex_bytes = ['{:02x}'.format(ord(x)) for x in buffer] - if spaces is None: - return ''.join(hex_bytes) - elif spaces is 1: - return ' '.join(hex_bytes) - for i in range(len(hex_bytes)//spaces): - hex_bytes.insert((spaces+1)*(i+1)-1, ' ') - return ''.join(hex_bytes) - -def assert_null(buffer, strict=True): - r"""Ensure an input buffer is entirely zero. - - >>> assert_null(buffer('')) - >>> assert_null(buffer('\x00\x00')) - >>> assert_null(buffer('\x00\x01\x02\x03')) - Traceback (most recent call last): - ... - ValueError: 00 01 02 03 - >>> stderr = sys.stderr - >>> sys.stderr = sys.stdout - >>> assert_null(buffer('\x00\x01\x02\x03'), strict=False) - warning: post-data padding not zero: 00 01 02 03 - >>> sys.stderr = stderr - """ - if buffer and ord(max(buffer)) != 0: - hex_string = hex_bytes(buffer, spaces=1) - if strict: - raise ValueError(hex_string) - else: - sys.stderr.write( - 'warning: post-data padding not zero: {}\n'.format(hex_string)) - # Translated from ReadWave() def loadibw(filename, strict=True): if hasattr(filename, 'read'): @@ -523,18 +312,18 @@ def loadibw(filename, strict=True): tail = 4 # 4 = size of wData field in WaveHeader5 structure waveDataSize = bin_info['wfmSize'] - (wave_struct.size - tail) # dtype() wrapping to avoid numpy.generic and - # getset_descriptor issues with the builtin Numpy types + # getset_descriptor issues with the builtin numpy types # (e.g. int32). It has no effect on our local complex # integers. if version == 5: shape = [n for n in wave_info['nDim'] if n > 0] or (0,) else: shape = (wave_info['npnts'],) - t = numpy.dtype(numpy.int8) # setup a safe default + t = _numpy.dtype(_numpy.int8) # setup a safe default if wave_info['type'] == 0: # text wave shape = (waveDataSize,) elif wave_info['type'] in TYPE_TABLE or wave_info['npnts']: - t = numpy.dtype(TYPE_TABLE[wave_info['type']]) + t = _numpy.dtype(TYPE_TABLE[wave_info['type']]) assert waveDataSize == wave_info['npnts'] * t.itemsize, ( '{}, {}, {}, {}'.format( waveDataSize, wave_info['npnts'], t.itemsize, t)) @@ -543,9 +332,9 @@ def loadibw(filename, strict=True): if wave_info['npnts'] == 0: data_b = buffer('') else: - tail_data = array.array('f', b[-tail:]) + tail_data = _array.array('f', b[-tail:]) data_b = buffer(buffer(tail_data) + f.read(waveDataSize-tail)) - data = numpy.ndarray( + data = _numpy.ndarray( shape=shape, dtype=t.newbyteorder(byteOrder), buffer=data_b, @@ -559,7 +348,7 @@ def loadibw(filename, strict=True): # * 16 bytes of padding # * Optional wave note data pad_b = buffer(f.read(16)) # skip the padding - assert_null(pad_b, strict=strict) + _assert_null(pad_b, strict=strict) bin_info['note'] = str(f.read(bin_info['noteSize'])).strip() elif version == 3: # Post-data info: @@ -574,7 +363,7 @@ def loadibw(filename, strict=True): no trailing null byte. """ pad_b = buffer(f.read(16)) # skip the padding - assert_null(pad_b, strict=strict) + _assert_null(pad_b, strict=strict) bin_info['note'] = str(f.read(bin_info['noteSize'])).strip() bin_info['formula'] = str(f.read(bin_info['formulaSize'])).strip() elif version == 5: @@ -632,7 +421,7 @@ def loadibw(filename, strict=True): start = offset else: assert offset == 0, offset - data = numpy.array(strings) + data = _numpy.array(strings) shape = [n for n in wave_info['nDim'] if n > 0] or (0,) data.reshape(shape) finally: diff --git a/igor/struct.py b/igor/struct.py new file mode 100644 index 0000000..d16ca8a --- /dev/null +++ b/igor/struct.py @@ -0,0 +1,181 @@ +# Copyright + +"Structure and Field classes for declaring structures " + +from __future__ import absolute_import +import struct as _struct + +import numpy as _numpy + + +_buffer = buffer # save builtin buffer for clobbered situations + + +class Field (object): + """Represent a Structure field. + + See Also + -------- + Structure + """ + def __init__(self, format, name, default=None, help=None, count=1): + self.format = format # See the struct documentation + self.name = name + self.default = None + self.help = help + self.count = count + self.total_count = _numpy.prod(count) + + +class Structure (_struct.Struct): + """Represent a C structure. + + A convenient wrapper around struct.Struct that uses Fields and + adds dict-handling methods for transparent name assignment. + + See Also + -------- + Field + + Examples + -------- + + Represent the C structure:: + + struct thing { + short version; + long size[3]; + } + + As + + >>> import array + >>> from pprint import pprint + >>> thing = Structure(name='thing', + ... fields=[Field('h', 'version'), Field('l', 'size', count=3)]) + >>> thing.set_byte_order('>') + >>> b = array.array('b', range(2+4*3)) + >>> d = thing.unpack_dict_from(buffer=b) + >>> pprint(d) + {'size': array([ 33752069, 101124105, 168496141]), 'version': 1} + >>> [hex(x) for x in d['size']] + ['0x2030405L', '0x6070809L', '0xa0b0c0dL'] + + You can even get fancy with multi-dimensional arrays. + + >>> thing = Structure(name='thing', + ... fields=[Field('h', 'version'), Field('l', 'size', count=(3,2))]) + >>> thing.set_byte_order('>') + >>> b = array.array('b', range(2+4*3*2)) + >>> d = thing.unpack_dict_from(buffer=b) + >>> d['size'].shape + (3, 2) + >>> pprint(d) + {'size': array([[ 33752069, 101124105], + [168496141, 235868177], + [303240213, 370612249]]), + 'version': 1} + """ + def __init__(self, name, fields, byte_order='='): + # '=' for native byte order, standard size and alignment + # See http://docs.python.org/library/struct for details + self.name = name + self.fields = fields + self.set_byte_order(byte_order) + + def __str__(self): + return self.name + + def set_byte_order(self, byte_order): + """Allow changing the format byte_order on the fly. + """ + if (hasattr(self, 'format') and self.format != None + and self.format.startswith(byte_order)): + return # no need to change anything + format = [] + for field in self.fields: + format.extend([field.format]*field.total_count) + super(Structure, self).__init__( + format=byte_order+''.join(format).replace('P', 'L')) + + def _flatten_args(self, args): + # handle Field.count > 0 + flat_args = [] + for a,f in zip(args, self.fields): + if f.total_count > 1: + flat_args.extend(a) + else: + flat_args.append(a) + return flat_args + + def _unflatten_args(self, args): + # handle Field.count > 0 + unflat_args = [] + i = 0 + for f in self.fields: + if f.total_count > 1: + data = _numpy.array(args[i:i+f.total_count]) + data = data.reshape(f.count) + unflat_args.append(data) + else: + unflat_args.append(args[i]) + i += f.total_count + return unflat_args + + def pack(self, *args): + return super(Structure, self)(*self._flatten_args(args)) + + def pack_into(self, buffer, offset, *args): + return super(Structure, self).pack_into( + buffer, offset, *self._flatten_args(args)) + + def _clean_dict(self, dict): + for f in self.fields: + if f.name not in dict: + if f.default != None: + dict[f.name] = f.default + else: + raise ValueError('{} field not set for {}'.format( + f.name, self.__class__.__name__)) + return dict + + def pack_dict(self, dict): + dict = self._clean_dict(dict) + return self.pack(*[dict[f.name] for f in self.fields]) + + def pack_dict_into(self, buffer, offset, dict={}): + dict = self._clean_dict(dict) + return self.pack_into(buffer, offset, + *[dict[f.name] for f in self.fields]) + + def unpack(self, string): + return self._unflatten_args( + super(Structure, self).unpack(string)) + + def unpack_from(self, buffer, offset=0): + try: + args = super(Structure, self).unpack_from(buffer, offset) + except _struct.error as e: + if not self.name in ('WaveHeader2', 'WaveHeader5'): + raise + # HACK! For WaveHeader5, when npnts is 0, wData is + # optional. If we couldn't unpack the structure, fill in + # wData with zeros and try again, asserting that npnts is + # zero. + if len(buffer) - offset < self.size: + # missing wData? Pad with zeros + buffer += _buffer('\x00'*(self.size + offset - len(buffer))) + args = super(Structure, self).unpack_from(buffer, offset) + unpacked = self._unflatten_args(args) + data = dict(zip([f.name for f in self.fields], + unpacked)) + assert data['npnts'] == 0, data['npnts'] + return self._unflatten_args(args) + + def unpack_dict(self, string): + return dict(zip([f.name for f in self.fields], + self.unpack(string))) + + def unpack_dict_from(self, buffer, offset=0): + return dict(zip([f.name for f in self.fields], + self.unpack_from(buffer, offset))) diff --git a/igor/util.py b/igor/util.py new file mode 100644 index 0000000..7b2c34f --- /dev/null +++ b/igor/util.py @@ -0,0 +1,53 @@ +# Copyright + +"Utility functions for handling buffers" + +import sys as _sys + + +def hex_bytes(buffer, spaces=None): + r"""Pretty-printing for binary buffers. + + >>> hex_bytes(buffer('\x00\x01\x02\x03\x04')) + '0001020304' + >>> hex_bytes(buffer('\x00\x01\x02\x03\x04'), spaces=1) + '00 01 02 03 04' + >>> hex_bytes(buffer('\x00\x01\x02\x03\x04'), spaces=2) + '0001 0203 04' + >>> hex_bytes(buffer('\x00\x01\x02\x03\x04\x05\x06'), spaces=2) + '0001 0203 0405 06' + >>> hex_bytes(buffer('\x00\x01\x02\x03\x04\x05\x06'), spaces=3) + '000102 030405 06' + """ + hex_bytes = ['{:02x}'.format(ord(x)) for x in buffer] + if spaces is None: + return ''.join(hex_bytes) + elif spaces is 1: + return ' '.join(hex_bytes) + for i in range(len(hex_bytes)//spaces): + hex_bytes.insert((spaces+1)*(i+1)-1, ' ') + return ''.join(hex_bytes) + +def assert_null(buffer, strict=True): + r"""Ensure an input buffer is entirely zero. + + >>> import sys + >>> assert_null(buffer('')) + >>> assert_null(buffer('\x00\x00')) + >>> assert_null(buffer('\x00\x01\x02\x03')) + Traceback (most recent call last): + ... + ValueError: 00 01 02 03 + >>> stderr = sys.stderr + >>> sys.stderr = sys.stdout + >>> assert_null(buffer('\x00\x01\x02\x03'), strict=False) + warning: post-data padding not zero: 00 01 02 03 + >>> sys.stderr = stderr + """ + if buffer and ord(max(buffer)) != 0: + hex_string = hex_bytes(buffer, spaces=1) + if strict: + raise ValueError(hex_string) + else: + _sys.stderr.write( + 'warning: post-data padding not zero: {}\n'.format(hex_string)) -- 2.26.2