From: W. Trevor King Date: Thu, 19 Jul 2012 12:14:04 +0000 (-0400) Subject: Split record handling into modules and implement VariablesRecord. X-Git-Tag: v0.2~34 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=d400cacfa6910a3b83414e4d8ad99e3b9b1ec465;p=igor.git Split record handling into modules and implement VariablesRecord. --- diff --git a/igor/packed.py b/igor/packed.py index 69d7353..48f933e 100644 --- a/igor/packed.py +++ b/igor/packed.py @@ -1,94 +1,15 @@ # Copyright -from io import BytesIO as _BytesIO +"Read IGOR Packed Experiment files files into records." -from .binarywave import load as _loadibw from .struct import Structure as _Structure from .struct import Field as _Field +from .util import byte_order as _byte_order +from .util import need_to_reorder_bytes as _need_to_reorder_bytes +from .record import RECORD_TYPE as _RECORD_TYPE +from .record.base import UnknownRecord as _UnknownRecord +from .record.base import UnusedRecord as _UnusedRecord -"Read IGOR Packed Experiment files files into records." - - -class Record (object): - def __init__(self, header, data): - self.header = header - self.data = data - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return '<{} {}>'.format(self.__class__.__name__, id(self)) - - -class UnknownRecord (Record): - def __repr__(self): - return '<{}-{} {}>'.format( - self.__class__.__name__, self.header['recordType'], id(self)) - - -class UnusedRecord (Record): - pass - - -class VariablesRecord (Record): - pass - - -class HistoryRecord (Record): - pass - - -class WaveRecord (Record): - def __init__(self, *args, **kwargs): - super(WaveRecord, self).__init__(*args, **kwargs) - self.wave = _loadibw(_BytesIO(bytes(self.data)), strict=False) - - def __str__(self): - return str(self.wave) - - def __repr__(self): - return str(self.wave) - - -class RecreationRecord (Record): - pass - - -class ProcedureRecord (Record): - pass - - -class GetHistoryRecord (Record): - pass - - -class PackedFileRecord (Record): - pass - - -class FolderStartRecord (Record): - pass - - -class FolderEndRecord (Record): - pass - - -# From PackedFile.h -RECORD_TYPE = { - 0: UnusedRecord, - 1: VariablesRecord, - 2: HistoryRecord, - 3: WaveRecord, - 4: RecreationRecord, - 5: ProcedureRecord, - 6: UnusedRecord, - 7: GetHistoryRecord, - 8: PackedFileRecord, - 9: FolderStartRecord, - 10: FolderEndRecord, - } # Igor writes other kinds of records in a packed experiment file, for # storing things like pictures, page setup records, and miscellaneous @@ -118,21 +39,29 @@ def load(filename, strict=True, ignore_unknown=True): f = filename # filename is actually a stream object else: f = open(filename, 'rb') + byte_order = None + initial_byte_order = '=' try: while True: - PackedFileRecordHeader.set_byte_order('=') b = buffer(f.read(PackedFileRecordHeader.size)) if not b: break + PackedFileRecordHeader.set_byte_order(initial_byte_order) header = PackedFileRecordHeader.unpack_from(b) + if header['version'] and not byte_order: + need_to_reorder = _need_to_reorder_bytes(header['version']) + byte_order = initial_byte_order = _byte_order(need_to_reorder) + if need_to_reorder: + PackedFileRecordHeader.set_byte_order(byte_order) + header = PackedFileRecordHeader.unpack_from(b) data = buffer(f.read(header['numDataBytes'])) - record_type = RECORD_TYPE.get( - header['recordType'] & PACKEDRECTYPE_MASK, UnknownRecord) - if record_type in [UnknownRecord, UnusedRecord + record_type = _RECORD_TYPE.get( + header['recordType'] & PACKEDRECTYPE_MASK, _UnknownRecord) + if record_type in [_UnknownRecord, _UnusedRecord ] and not ignore_unknown: raise KeyError('unkown record type {}'.format( header['recordType'])) - records.append(record_type(header, data)) + records.append(record_type(header, data, byte_order=byte_order)) finally: if not hasattr(filename, 'read'): f.close() diff --git a/igor/record/__init__.py b/igor/record/__init__.py new file mode 100644 index 0000000..ffa6456 --- /dev/null +++ b/igor/record/__init__.py @@ -0,0 +1,30 @@ +# Copyright + +"Record parsers for IGOR's packed experiment files." + + +from .base import Record, UnknownRecord, UnusedRecord +from .variables import VariablesRecord +from .history import HistoryRecord +from .wave import WaveRecord +from .recreation import RecreationRecord +from .procedure import ProcedureRecord +from .gethistory import GetHistoryRecord +from .packedfile import PackedFileRecord +from .folder import FolderStartRecord, FolderEndRecord + + +# From PackedFile.h +RECORD_TYPE = { + 0: UnusedRecord, + 1: VariablesRecord, + 2: HistoryRecord, + 3: WaveRecord, + 4: RecreationRecord, + 5: ProcedureRecord, + 6: UnusedRecord, + 7: GetHistoryRecord, + 8: PackedFileRecord, + 9: FolderStartRecord, + 10: FolderEndRecord, + } diff --git a/igor/record/base.py b/igor/record/base.py new file mode 100644 index 0000000..a6990f8 --- /dev/null +++ b/igor/record/base.py @@ -0,0 +1,24 @@ +# Copyright + + +class Record (object): + def __init__(self, header, data, byte_order=None): + self.header = header + self.data = data + self.byte_order = byte_order + + def __str__(self): + return self.__repr__() + + def __repr__(self): + return '<{} {}>'.format(self.__class__.__name__, id(self)) + + +class UnknownRecord (Record): + def __repr__(self): + return '<{}-{} {}>'.format( + self.__class__.__name__, self.header['recordType'], id(self)) + + +class UnusedRecord (Record): + pass diff --git a/igor/record/folder.py b/igor/record/folder.py new file mode 100644 index 0000000..b03e283 --- /dev/null +++ b/igor/record/folder.py @@ -0,0 +1,11 @@ +# Copyright + +from .base import Record + + +class FolderStartRecord (Record): + pass + + +class FolderEndRecord (Record): + pass diff --git a/igor/record/gethistory.py b/igor/record/gethistory.py new file mode 100644 index 0000000..d2e5c20 --- /dev/null +++ b/igor/record/gethistory.py @@ -0,0 +1,7 @@ +# Copyright + +from .base import Record + + +class GetHistoryRecord (Record): + pass diff --git a/igor/record/history.py b/igor/record/history.py new file mode 100644 index 0000000..0974cac --- /dev/null +++ b/igor/record/history.py @@ -0,0 +1,7 @@ +# Copyright + +from .base import Record + + +class HistoryRecord (Record): + pass diff --git a/igor/record/packedfile.py b/igor/record/packedfile.py new file mode 100644 index 0000000..9e12437 --- /dev/null +++ b/igor/record/packedfile.py @@ -0,0 +1,7 @@ +# Copyright + +from .base import Record + + +class PackedFileRecord (Record): + pass diff --git a/igor/record/procedure.py b/igor/record/procedure.py new file mode 100644 index 0000000..3c1e764 --- /dev/null +++ b/igor/record/procedure.py @@ -0,0 +1,7 @@ +# Copyright + +from .base import Record + + +class ProcedureRecord (Record): + pass diff --git a/igor/record/recreation.py b/igor/record/recreation.py new file mode 100644 index 0000000..3bc9cb4 --- /dev/null +++ b/igor/record/recreation.py @@ -0,0 +1,7 @@ +# Copyright + +from .base import Record + + +class RecreationRecord (Record): + pass diff --git a/igor/record/variables.py b/igor/record/variables.py new file mode 100644 index 0000000..e55d800 --- /dev/null +++ b/igor/record/variables.py @@ -0,0 +1,210 @@ +# Copyright + +from ..binarywave import TYPE_TABLE as _TYPE_TABLE +from ..struct import Structure as _Structure +from ..struct import Field as _Field +from ..util import byte_order as _byte_order +from ..util import need_to_reorder_bytes as _need_to_reorder_bytes +from .base import Record + + +VarHeaderCommon = _Structure( + name='VarHeaderCommon', + fields=[ + _Field('h', 'version', help='Version number for this header.'), + ]) + +# From Variables.h +VarHeader1 = _Structure( + name='VarHeader1', + fields=[ + _Field('h', 'version', help='Version number is 1 for this header.'), + _Field('h', 'numSysVars', help='Number of system variables (K0, K1, ...).'), + _Field('h', 'numUserVars', help='Number of user numeric variables -- may be zero.'), + _Field('h', 'numUserStrs', help='Number of user string variables -- may be zero.'), + ]) + +# From Variables.h +VarHeader2 = _Structure( + name='VarHeader2', + fields=[ + _Field('h', 'version', help='Version number is 2 for this header.'), + _Field('h', 'numSysVars', help='Number of system variables (K0, K1, ...).'), + _Field('h', 'numUserVars', help='Number of user numeric variables -- may be zero.'), + _Field('h', 'numUserStrs', help='Number of user string variables -- may be zero.'), + _Field('h', 'numDependentVars', help='Number of dependent numeric variables -- may be zero.'), + _Field('h', 'numDependentStrs', help='Number of dependent string variables -- may be zero.'), + ]) + +# From Variables.h +UserStrVarRec1 = _Structure( + name='UserStrVarRec1', + fields=[ + _Field('c', 'name', help='Name of the string variable.', count=32), + _Field('h', 'strLen', help='The real size of the following array.'), + _Field('c', 'data'), + ]) + +# From Variables.h +UserStrVarRec2 = _Structure( + name='UserStrVarRec2', + fields=[ + _Field('c', 'name', help='Name of the string variable.', count=32), + _Field('l', 'strLen', help='The real size of the following array.'), + _Field('c', 'data'), + ]) + +# From Variables.h +VarNumRec = _Structure( + name='VarNumRec', + fields=[ + _Field('h', 'numType', help='Type from binarywave.TYPE_TABLE'), + _Field('d', 'realPart', help='The real part of the number.'), + _Field('d', 'imagPart', help='The imag part if the number is complex.'), + _Field('l', 'reserved', help='Reserved - set to zero.'), + ]) + +# From Variables.h +UserNumVarRec = _Structure( + name='UserNumVarRec', + fields=[ + _Field('c', 'name', help='Name of the string variable.', count=32), + _Field('h', 'type', help='0 = string, 1 = numeric.'), + _Field(VarNumRec, 'num', help='Type and value of the variable if it is numeric. Not used for string.'), + ]) + +# From Variables.h +UserDependentVarRec = _Structure( + name='UserDependentVarRec', + fields=[ + _Field('c', 'name', help='Name of the string variable.', count=32), + _Field('h', 'type', help='0 = string, 1 = numeric.'), + _Field(VarNumRec, 'num', help='Type and value of the variable if it is numeric. Not used for string.'), + _Field('h', 'formulaLen', help='The length of the dependency formula.'), + _Field('c', 'formula', help='Start of the dependency formula. A C string including null terminator.'), + ]) + + +class VariablesRecord (Record): + def __init__(self, *args, **kwargs): + super(VariablesRecord, self).__init__(*args, **kwargs) + # self.header['version'] # record version always 0? + version = self._set_byte_order_and_get_version() + self.structure = self._get_structure(version) + self.variables = self.structure.unpack_from(self.data) + self.variables.update(self._unpack_variable_length_structures(version)) + self._normalize_variables() + + def _set_byte_order_and_get_version(self): + if self.byte_order: + VarHeaderCommon.set_byte_order(self.byte_order) + else: + VarHeaderCommon.set_byte_order('=') + version = VarHeaderCommon.unpack_from(self.data)['version'] + if not self.byte_order: + need_to_reorder = _need_to_reorder_bytes(version) + self.byte_order = _byte_order(need_to_reorder) + if need_to_reorder: + VarHeaderCommon.set_byte_order(self.byte_order) + version = VarHeaderCommon.unpack_from(self.data)['version'] + return version + + def _get_structure(self, version): + if version == 1: + header_struct = VarHeader1 + elif version == 2: + header_struct = VarHeader2 + else: + raise NotImplementedError( + 'Variables record version {}'.format(version)) + header = header_struct.unpack_from(self.data) + fields = [ + _Field(header_struct, 'header', help='VarHeader'), + _Field('f', 'sysVars', help='system variables', + count=header['numSysVars']), + _Field(UserNumVarRec, 'userVars', help='user variables', + count=header['numUserVars']), + ] + return _Structure(name='variables', fields=fields) + + def _unpack_variable_length_structures(self, version): + data = {'userStrs': []} + offset = self.structure.size + + if version == 1: + user_str_var_struct = UserStrVarRec1 + elif version == 2: + user_str_var_struct = UserStrVarRec2 + else: + raise NotImplementedError( + 'Variables record version {}'.format(version)) + user_str_var_struct.set_byte_order(self.byte_order) + for i in range(self.variables['header']['numUserStrs']): + d = user_str_var_struct.unpack_from(self.data, offset) + offset += user_str_var_struct.size + end = offset + d['strLen'] - 1 # one character already in struct + if d['strLen']: + d['data'] = d['data'] + self.data[offset:end] + else: + d['data'] = '' + offset = end + data['userStrs'].append(d) + + if version == 2: + data.update({'dependentVars': [], 'dependentStrs': []}) + UserDependentVarRec.set_byte_order(self.byte_order) + for i in range(self.variables['header']['numDependentVars']): + d,offset = self._unpack_dependent_variable(offset) + data['dependentVars'].append(d) + for i in range(self.variables['header']['numDependentStrs']): + d,offset = self._unpack_dependent_variable(offset) + data['dependentStrs'].append(d) + + if offset != len(self.data): + raise ValueError('too much data ({} extra bytes)'.format( + len(self.data)-offset)) + return data + + def _unpack_dependent_variable(self, offset): + d = UserDependentVarRec.unpack_from(self.data, offset) + offset += UserDependentVarRec.size + end = offset + d['formulaLen'] - 1 # one character already in struct + if d['formulaLen']: + d['formula'] = d['formula'] + self.data[offset:end] + else: + d['formula'] = '' + offset = end + return (d, offset) + + def _normalize_variables(self): + user_vars = {} + for num_var in self.variables['userVars']: + key,value = self._normalize_user_numeric_variable(num_var) + user_vars[key] = value + self.variables['userVars'] = user_vars + user_strs = {} + for str_var in self.variables['userStrs']: + name = self._normalize_null_terminated_string(str_var['name']) + user_strs[name] = str_var['data'] + if self.variables['header']['version'] == 2: + raise NotImplementedError('normalize dependent variables') + self.variables['userStrs'] = user_strs + + def _normalize_null_terminated_string(self, string): + return string.tostring().split('\x00', 1)[0] + + def _normalize_user_numeric_variable(self, user_num_var): + user_num_var['name'] = self._normalize_null_terminated_string( + user_num_var['name']) + if user_num_var['type']: # numeric + value = self._normalize_numeric_variable(user_num_var['num']) + else: # string + value = None + return (user_num_var['name'], value) + + def _normalize_numeric_variable(self, num_var): + t = _TYPE_TABLE[num_var['numType']] + if num_var['numType'] % 2: # complex number + return t(complex(num_var['realPart'], num_var['imagPart'])) + else: + return t(num_var['realPart']) diff --git a/igor/record/wave.py b/igor/record/wave.py new file mode 100644 index 0000000..53d3af1 --- /dev/null +++ b/igor/record/wave.py @@ -0,0 +1,15 @@ +# Copyright + +from io import BytesIO as _BytesIO + +from ..binarywave import load as _loadibw +from . import Record + + +class WaveRecord (Record): + def __init__(self, *args, **kwargs): + super(WaveRecord, self).__init__(*args, **kwargs) + self.wave = _loadibw(_BytesIO(bytes(self.data)), strict=False) + + def __str__(self): + return str(self.wave) diff --git a/setup.py b/setup.py index c63b48b..446771e 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup(name=package_name, ], packages=[ 'igor', + 'igor.record', ], scripts=[ 'bin/igorbinarywave.py', diff --git a/test/test.py b/test/test.py index 2c8c60e..3fd208f 100644 --- a/test/test.py +++ b/test/test.py @@ -1,3 +1,5 @@ +# Copyright + r"""Test the igor module by loading sample files. >>> dumpibw('mac-double.ibw', strict=False) # doctest: +REPORT_UDIFF @@ -652,7 +654,15 @@ record 28: record 29: record 30: - +{'header': {'numSysVars': 21, + 'numUserStrs': 0, + 'numUserVars': 0, + 'version': 1}, + 'sysVars': array([ 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 128.]), + 'userStrs': {}, + 'userVars': {}} record 31: record 32: @@ -1242,13 +1252,70 @@ record 40: record 41: record 42: - +{'header': {'numSysVars': 21, + 'numUserStrs': 6, + 'numUserVars': 0, + 'version': 1}, + 'sysVars': array([ 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 128.]), + 'userStrs': {'u_dataBase': ';PolarGraph0:,...,useCircles=2,maxArcLine=6;', + 'u_dbBadStringChars': ',;=:', + 'u_dbCurrBag': 'PolarGraph1', + 'u_dbCurrContents': ',appendRadius=radiusQ1,...,useCircles=2,maxArcLine=6;', + 'u_dbReplaceBadChars': '\xa9\xae\x99\x9f', + 'u_str': '2'}, + 'userVars': {}} record 43: record 44: record 45: - +{'header': {'numSysVars': 21, + 'numUserStrs': 10, + 'numUserVars': 28, + 'version': 1}, + 'sysVars': array([ 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 128.]), + 'userStrs': {'u_colorList': 'black;blue;green;cyan;red;magenta;yellow;white;special', + 'u_debugStr': 'Turn Debugging On', + 'u_polAngleAxesWherePop': 'Off;Radius Start;Radius End;Radius Start and End;All Major Radii;At Listed Radii', + 'u_polAngleUnitsPop': 'deg;rad', + 'u_polLineStylePop': 'solid;dash 1;dash 2;dash 3;dash 4;dash 5;dash 6;dash 7;dash 8;dash 9;dash 10;dash 11;dash 12;dash 13;dash 14;dash 15;dash 16;dash 17;', + 'u_polOffOn': 'Off;On', + 'u_polRadAxesWherePop': ' Off; Angle Start; Angle Middle; Angle End; Angle Start and End; 0; 90; 180; -90; 0, 90; 90, 180; -180, -90; -90, 0; 0, 180; 90, -90; 0, 90, 180, -90; All Major Angles; At Listed Angles', + 'u_polRotPop': ' -90; 0; +90; +180', + 'u_popup': '', + 'u_prompt': ''}, + 'userVars': {'V_bottom': 232.0, + 'V_left': 1.0, + 'V_max': 2.4158518093414401, + 'V_min': -2.1848498883412, + 'V_right': 232.0, + 'V_top': 1.0, + 'u_UniqWaveNdx': 8.0, + 'u_UniqWinNdx': 3.0, + 'u_angle0': 0.0, + 'u_angleRange': 6.2831853071795862, + 'u_debug': 0.0, + 'u_majorDelta': 0.0, + 'u_numPlaces': 0.0, + 'u_polAngle0': 0.26179938779914941, + 'u_polAngleRange': 1.0471975511965976, + 'u_polInnerRadius': -20.0, + 'u_polMajorAngleInc': 0.26179938779914941, + 'u_polMajorRadiusInc': 10.0, + 'u_polMinorAngleTicks': 3.0, + 'u_polMinorRadiusTicks': 1.0, + 'u_polOuterRadius': 0.0, + 'u_segsPerMinorArc': 3.0, + 'u_tickDelta': 0.0, + 'u_var': 0.0, + 'u_x1': 11.450159535018935, + 'u_x2': 12.079591517721363, + 'u_y1': 42.732577139459856, + 'u_y2': 45.081649278814126}} record 46: record 47: @@ -1267,7 +1334,8 @@ import sys from igor.binarywave import load as loadibw from igor.packed import load as loadpxp -from igor.packed import WaveRecord +from igor.record.variables import VariablesRecord +from igor.record.wave import WaveRecord _this_dir = os.path.dirname(__file__) @@ -1287,7 +1355,9 @@ def dumppxp(filename, strict=True): records = loadpxp(path, strict=strict) for i,record in enumerate(records): print('record {}:'.format(i)) - if isinstance(record, WaveRecord): + if isinstance(record, VariablesRecord): + pprint(record.variables) + elif isinstance(record, WaveRecord): pprint(record.wave) else: pprint(record)