From 3ae7034c746b13a46b9e4523e35c45542cf59f32 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 21 Jul 2012 12:20:32 -0400 Subject: [PATCH] Use an explicit .array attribute to determine if fields are arrays. This improves on the old method of assuming they were scalar if .item_count was 1. --- igor/binarywave.py | 63 ++++++++++++++++++++++------------------ igor/record/variables.py | 9 ++++-- igor/struct.py | 48 ++++++++++++++++++------------ 3 files changed, 70 insertions(+), 50 deletions(-) diff --git a/igor/binarywave.py b/igor/binarywave.py index c9d1ff9..bcccda2 100644 --- a/igor/binarywave.py +++ b/igor/binarywave.py @@ -63,6 +63,11 @@ complexUInt32 = _numpy.dtype( class StaticStringField (_DynamicField): _null_terminated = False _array_size_field = None + def __init__(self, *args, **kwargs): + if 'array' not in kwargs: + kwargs['array'] = True + super(StaticStringField, self).__init__(*args, **kwargs) + def post_unpack(self, parents, data): wave_structure = parents[-1] wave_data = self._get_structure_data(parents, data, wave_structure) @@ -164,8 +169,8 @@ BinHeader5 = _Structure( # `version` field pulled out into Wave _Field('l', 'formulaSize', help='The size of the dependency formula, if any.'), _Field('l', 'noteSize', help='The size of the note text.'), _Field('l', 'dataEUnitsSize', help='The size of optional extended data units.'), - _Field('l', 'dimEUnitsSize', help='The size of optional extended dimension units.', count=MAXDIMS), - _Field('l', 'dimLabelsSize', help='The size of optional dimension labels.', count=MAXDIMS), + _Field('l', 'dimEUnitsSize', help='The size of optional extended dimension units.', count=MAXDIMS, array=True), + _Field('l', 'dimLabelsSize', help='The size of optional dimension labels.', count=MAXDIMS, array=True), _Field('l', 'sIndicesSize', help='The size of string indicies if this is a text wave.'), _Field('l', 'optionsSize1', default=0, help='Reserved. Write zero. Ignore on read.'), _Field('l', 'optionsSize2', default=0, help='Reserved. Write zero. Ignore on read.'), @@ -191,8 +196,8 @@ WaveHeader2 = _DynamicStructure( _Field('h', 'whVersion', default=0, help='Write 0. Ignore on read.'), _Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'), _Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'), - _Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1), - _Field('c', 'xUnits', default=0, help='Natural x-axis units go here - null if none.', count=MAX_UNIT_CHARS+1), + _Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1, array=True), + _Field('c', 'xUnits', default=0, help='Natural x-axis units go here - null if none.', count=MAX_UNIT_CHARS+1, array=True), _Field('l', 'npnts', help='Number of data points in wave.'), _Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), _Field('d', 'hsA', help='X value for point p = hsA*p + hsB'), @@ -207,7 +212,7 @@ WaveHeader2 = _DynamicStructure( _Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'), _Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'), _Field('L', 'creationDate', help='DateTime of creation. Not used in version 1 files.'), - _Field('c', 'wUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=2), + _Field('c', 'wUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=2, array=True), _Field('L', 'modDate', help='DateTime of last modification.'), _Field('P', 'waveNoteH', help='Used in memory only. Write zero. Ignore on read.'), ]) @@ -224,27 +229,27 @@ WaveHeader5 = _DynamicStructure( _Field('l', 'npnts', help='Total number of points (multiply dimensions up to first zero).'), _Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'), _Field('h', 'dLock', default=0, help='Reserved. Write zero. Ignore on read.'), - _Field('c', 'whpad1', default=0, help='Reserved. Write zero. Ignore on read.', count=6), + _Field('c', 'whpad1', default=0, help='Reserved. Write zero. Ignore on read.', count=6, array=True), _Field('h', 'whVersion', default=1, help='Write 1. Ignore on read.'), NullStaticStringField('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME5+1), _Field('l', 'whpad2', default=0, help='Reserved. Write zero. Ignore on read.'), _Field('P', 'dFolder', default=0, help='Used in memory only. Write zero. Ignore on read.'), # Dimensioning info. [0] == rows, [1] == cols etc - _Field('l', 'nDim', help='Number of of items in a dimension -- 0 means no data.', count=MAXDIMS), - _Field('d', 'sfA', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS), - _Field('d', 'sfB', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS), + _Field('l', 'nDim', help='Number of of items in a dimension -- 0 means no data.', count=MAXDIMS, array=True), + _Field('d', 'sfA', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS, array=True), + _Field('d', 'sfB', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS, array=True), # SI units - _Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1), - _Field('c', 'dimUnits', default=0, help='Natural dimension units go here - null if none.', count=(MAXDIMS, MAX_UNIT_CHARS+1)), + _Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1, array=True), + _Field('c', 'dimUnits', default=0, help='Natural dimension units go here - null if none.', count=(MAXDIMS, MAX_UNIT_CHARS+1), array=True), _Field('h', 'fsValid', help='TRUE if full scale values have meaning.'), _Field('h', 'whpad3', default=0, help='Reserved. Write zero. Ignore on read.'), _Field('d', 'topFullScale', help='The max and max full scale value for wave'), # sic, probably "max and min" _Field('d', 'botFullScale', help='The max and max full scale value for wave.'), # sic, probably "max and min" _Field('P', 'dataEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.'), - _Field('P', 'dimEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS), - _Field('P', 'dimLabels', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS), + _Field('P', 'dimEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS, array=True), + _Field('P', 'dimLabels', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS, array=True), _Field('P', 'waveNoteH', default=0, help='Used in memory only. Write zero. Ignore on read.'), - _Field('l', 'whUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=16), + _Field('l', 'whUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=16, array=True), # The following stuff is considered private to Igor. _Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), _Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'), @@ -577,7 +582,7 @@ Wave1 = _DynamicStructure( fields=[ _Field(BinHeader1, 'bin_header', help='Binary wave header'), _Field(WaveHeader2, 'wave_header', help='Wave header'), - DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0), + DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0, array=True), ]) Wave2 = _DynamicStructure( @@ -585,9 +590,9 @@ Wave2 = _DynamicStructure( fields=[ _Field(BinHeader2, 'bin_header', help='Binary wave header'), _Field(WaveHeader2, 'wave_header', help='Wave header'), - DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0), - _Field('x', 'padding', help='16 bytes of padding in versions 2 and 3.', count=16), - DynamicWaveNoteField('c', 'note', help='Optional wave note data', count=0), + DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0, array=True), + _Field('x', 'padding', help='16 bytes of padding in versions 2 and 3.', count=16, array=True), + DynamicWaveNoteField('c', 'note', help='Optional wave note data', count=0, array=True), ]) Wave3 = _DynamicStructure( @@ -595,10 +600,10 @@ Wave3 = _DynamicStructure( fields=[ _Field(BinHeader3, 'bin_header', help='Binary wave header'), _Field(WaveHeader2, 'wave_header', help='Wave header'), - DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0), - _Field('x', 'padding', help='16 bytes of padding in versions 2 and 3.', count=16), - DynamicWaveNoteField('c', 'note', help='Optional wave note data', count=0), - DynamicDependencyFormulaField('c', 'formula', help='Optional wave dependency formula', count=0), + DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0, array=True), + _Field('x', 'padding', help='16 bytes of padding in versions 2 and 3.', count=16, array=True), + DynamicWaveNoteField('c', 'note', help='Optional wave note data', count=0, array=True), + DynamicDependencyFormulaField('c', 'formula', help='Optional wave dependency formula', count=0, array=True), ]) Wave5 = _DynamicStructure( @@ -606,13 +611,13 @@ Wave5 = _DynamicStructure( fields=[ _Field(BinHeader5, 'bin_header', help='Binary wave header'), _Field(WaveHeader5, 'wave_header', help='Wave header'), - DynamicWaveDataField5('f', 'wData', help='The start of the array of waveform data.', count=0), - DynamicDependencyFormulaField('c', 'formula', help='Optional wave dependency formula.', count=0), - DynamicWaveNoteField('c', 'note', help='Optional wave note data.', count=0), - DynamicDataUnitsField('c', 'data_units', help='Optional extended data units data.', count=0), - DynamicDimensionUnitsField('c', 'dimension_units', help='Optional dimension label data', count=0), - DynamicLabelsField('c', 'labels', help="Optional dimension label data", count=0), - DynamicStringIndicesDataField('P', 'sIndices', help='Dynamic string indices for text waves.', count=0), + DynamicWaveDataField5('f', 'wData', help='The start of the array of waveform data.', count=0, array=True), + DynamicDependencyFormulaField('c', 'formula', help='Optional wave dependency formula.', count=0, array=True), + DynamicWaveNoteField('c', 'note', help='Optional wave note data.', count=0, array=True), + DynamicDataUnitsField('c', 'data_units', help='Optional extended data units data.', count=0, array=True), + DynamicDimensionUnitsField('c', 'dimension_units', help='Optional dimension label data', count=0, array=True), + DynamicLabelsField('c', 'labels', help="Optional dimension label data", count=0, array=True), + DynamicStringIndicesDataField('P', 'sIndices', help='Dynamic string indices for text waves.', count=0, array=True), ]) Wave = _DynamicStructure( diff --git a/igor/record/variables.py b/igor/record/variables.py index cdcdd89..a8eaccf 100644 --- a/igor/record/variables.py +++ b/igor/record/variables.py @@ -75,6 +75,11 @@ class ListedDynamicStrDataField (_DynamicStringField, ListedStaticStringField): class DynamicVarDataField (_DynamicField): + def __init__(self, *args, **kwargs): + if 'array' not in kwargs: + kwargs['array'] = True + super(DynamicVarDataField, self).__init__(*args, **kwargs) + def pre_pack(self, parents, data): raise NotImplementedError() @@ -247,8 +252,8 @@ Variables2 = _DynamicStructure( DynamicSysVarField('f', 'sysVars', help='System variables', count=0), DynamicUserVarField(UserNumVarRec, 'userVars', help='User numeric variables', count=0), DynamicUserStrField(UserStrVarRec2, 'userStrs', help='User string variables', count=0), - _Field(UserDependentVarRec, 'dependentVars', help='Dependent numeric variables.', count=0), - _Field(UserDependentVarRec, 'dependentStrs', help='Dependent string variables.', count=0), + _Field(UserDependentVarRec, 'dependentVars', help='Dependent numeric variables.', count=0, array=True), + _Field(UserDependentVarRec, 'dependentStrs', help='Dependent string variables.', count=0, array=True), ]) diff --git a/igor/struct.py b/igor/struct.py index 2885da4..6a19e2c 100644 --- a/igor/struct.py +++ b/igor/struct.py @@ -65,7 +65,7 @@ class Field (object): Example of a multi-dimensional float field: >>> data = Field( - ... 'f', 'data', help='example data', count=(2,3,4)) + ... 'f', 'data', help='example data', count=(2,3,4), array=True) >>> data.arg_count 24 >>> list(data.indexes()) # doctest: +ELLIPSIS @@ -91,7 +91,7 @@ class Field (object): Example of a nested structure field: >>> run = Structure('run', fields=[time, data]) - >>> runs = Field(run, 'runs', help='pair of runs', count=2) + >>> runs = Field(run, 'runs', help='pair of runs', count=2, array=True) >>> runs.arg_count # = 2 * (1 + 24) 50 >>> data1 = numpy.arange(data.arg_count).reshape(data.count) @@ -148,12 +148,14 @@ class Field (object): -------- Structure """ - def __init__(self, format, name, default=None, help=None, count=1): + def __init__(self, format, name, default=None, help=None, count=1, + array=False): self.format = format self.name = name self.default = default self.help = help self.count = count + self.array = array self.setup() def setup(self): @@ -164,6 +166,10 @@ class Field (object): """ _LOG.debug('setup {}'.format(self)) self.item_count = _numpy.prod(self.count) # number of item repeats + if not self.array and self.item_count != 1: + raise ValueError( + '{} must be an array field to have a count of {}'.format( + self, self.count)) if isinstance(self.format, Structure): self.structure_count = sum( f.arg_count for f in self.format.fields) @@ -182,7 +188,7 @@ class Field (object): def indexes(self): """Iterate through indexes to a possibly multi-dimensional array""" - assert self.item_count > 1, self + assert self.array, self try: i = [0] * len(self.count) except TypeError: # non-iterable count @@ -202,7 +208,7 @@ class Field (object): If the field is repeated (count > 1), the incoming data should be iterable with each iteration returning a single item. """ - if self.item_count > 1: + if self.array: if data is None: data = [] if hasattr(data, 'flat'): # take advantage of numpy's ndarray.flat @@ -230,7 +236,7 @@ class Field (object): item = None for arg in self.pack_item(item): yield arg - elif self.item_count: + else: for arg in self.pack_item(data): yield arg @@ -272,7 +278,8 @@ class Field (object): count = self.count else: count = 0 # padding bytes, etc. - if count == 1: + if not self.array: + assert count == 1, (self, self.count) return unpacked[0] if isinstance(self.format, Structure): try: @@ -377,11 +384,12 @@ class Structure (_struct.Struct): >>> time = Field('I', 'time', default=0, help='POSIX time') >>> data = Field( - ... 'h', 'data', default=0, help='example data', count=(2,3)) + ... 'h', 'data', default=0, help='example data', count=(2,3), + ... array=True) >>> run = Structure('run', fields=[time, data]) >>> version = Field( ... 'H', 'version', default=1, help='example version') - >>> runs = Field(run, 'runs', help='pair of runs', count=2) + >>> runs = Field(run, 'runs', help='pair of runs', count=2, array=True) >>> experiment = Structure('experiment', fields=[version, runs]) The structures automatically calculate the flattened data format: @@ -453,7 +461,8 @@ class Structure (_struct.Struct): If you set ``count=0``, the field is ignored. >>> experiment2 = Structure('experiment', fields=[ - ... version, Field('f', 'ignored', count=0), runs], byte_order='>') + ... version, Field('f', 'ignored', count=0, array=True), runs], + ... byte_order='>') >>> experiment2.format '>HIhhhhhhIhhhhhh' >>> d = experiment2.unpack(b) @@ -656,7 +665,7 @@ class DynamicStructure (Structure): >>> dynamic_length_vector = DynamicStructure('vector', ... fields=[ ... DynamicLengthField('I', 'length'), - ... Field('h', 'data', count=0), + ... Field('h', 'data', count=0, array=True), ... ], ... byte_order='>') >>> class DynamicDataField (DynamicField): @@ -667,7 +676,7 @@ class DynamicStructure (Structure): >>> dynamic_data_vector = DynamicStructure('vector', ... fields=[ ... Field('I', 'length'), - ... DynamicDataField('h', 'data', count=0), + ... DynamicDataField('h', 'data', count=0, array=True), ... ], ... byte_order='>') @@ -746,18 +755,18 @@ class DynamicStructure (Structure): f.setup() f.format.setup() if isinstance(f.format, DynamicStructure): - if f.item_count == 1: - # TODO, fix in case we *want* an array - d[f.name] = {} - f.format.unpack_stream( - stream, parents=parents, data=data, d=d[f.name]) - else: + if f.array: d[f.name] = [] for i in range(f.item_count): x = {} d[f.name].append(x) f.format.unpack_stream( stream, parents=parents, data=data, d=x) + else: + assert f.item_count == 1, (f, f.count) + d[f.name] = {} + f.format.unpack_stream( + stream, parents=parents, data=data, d=d[f.name]) if hasattr(f, 'post_unpack'): _LOG.debug('post-unpack {}'.format(f)) repeat = f.post_unpack(parents=parents, data=data) @@ -774,7 +783,8 @@ class DynamicStructure (Structure): f.setup() f.format.setup() x = [f.format.unpack_from(b) for b in bs] - if len(x) == 1: # TODO, fix in case we *want* an array + if not f.array: + assert len(x) == 1, (f, f.count, x) x = x[0] return x else: -- 2.26.2