+#!/usr/bin/env python
+
'''
-FLATFILTS
+flatfilts.py
-Force spectroscopy curves filtering of flat curves
-Licensed under the GNU LGPL version 2
+Force spectroscopy files filtering of flat files.
-Other plugin dependencies:
+Plugin dependencies:
procplots.py (plot processing plugin)
-'''
-from hooke.libhooke import WX_GOOD
+Copyright 2008 Massimo Sandal, Fabrizio Benedetti
+with modifications by Dr. Rolf Schmidt (Concordia University, Canada)
-import wxversion
-wxversion.select(WX_GOOD)
-import xml.dom.minidom
-import wx
-import scipy
-import numpy
-from numpy import diff
-#import pickle
+This program is released under the GNU General Public License version 2.
+'''
-from .. import libpeakspot as lps
-from .. import libhookecurve as lhc
+import lib.libhooke as lh
+import wxversion
+wxversion.select(lh.WX_GOOD)
+import copy
+from numpy import diff, mean
-class flatfiltsCommands(object):
+import lib.peakspot as lps
+import lib.curve
- def _plug_init(self):
- #configurate convfilt variables
- convfilt_configurator=ConvfiltConfig()
- self.convfilt_config=convfilt_configurator.load_config('convfilt.conf')
+class flatfiltsCommands:
- def do_flatfilt(self,args):
+ def do_flatfilt(self):
'''
FLATFILT
(flatfilts.py)
- Filters out flat (featureless) curves of the current playlist,
- creating a playlist containing only the curves with potential
+ Filters out flat (featureless) files of the current playlist,
+ creating a playlist containing only the files with potential
features.
------------
Syntax:
If called without arguments, it uses default values, that
should work most of the times.
'''
- median_filter=7
- min_npks=4
- min_deviation=9
-
- args=args.split(' ')
- if len(args) == 2:
- min_npks=int(args[0])
- min_deviation=int(args[1])
- else:
- pass
-
- print 'Processing playlist...'
- notflat_list=[]
-
- c=0
- for item in self.current_list:
- c+=1
+ self.AppendToOutput('Processing playlist...')
+ self.AppendToOutput('(Please wait)')
+ features = []
+ playlist = self.GetActivePlaylist()
+ files = playlist.files
+ file_index = 0
+ for current_file in files:
+ file_index += 1
try:
- notflat=self.has_features(item, median_filter, min_npks, min_deviation)
- print 'Curve',item.path, 'is',c,'of',len(self.current_list),': features are ',notflat
+ current_file.identify(self.drivers)
+ notflat = self.has_features(copy.deepcopy(current_file))
+ feature_string = ''
+ if notflat != 1:
+ if notflat > 0:
+ feature_string = str(notflat) + ' features'
+ else:
+ feature_string = 'no features'
+ else:
+ feature_string = '1 feature'
+ output_string = ''.join(['Curve ', current_file.name, '(', str(file_index), '/', str(len(files)), '): ', feature_string])
except:
- notflat=False
- print 'Curve',item.path, 'is',c,'of',len(self.current_list),': cannot be filtered. Probably unable to retrieve force data from corrupt file.'
-
+ notflat = False
+ output_string = ''.join(['Curve ', current_file.name, '(', str(file_index), '/', str(len(files)), '): cannot be filtered. Probably unable to retrieve force data from corrupt file.'])
+ self.AppendToOutput(output_string)
if notflat:
- item.features=notflat
- item.curve=None #empty the item object, to further avoid memory leak
- notflat_list.append(item)
-
- if len(notflat_list)==0:
- print 'Found nothing interesting. Check your playlist, could be a bug or criteria could be too much stringent'
- return
+ current_file.features = notflat
+ features.append(file_index - 1)
+ if not features:
+ self.AppendToOutput('Found nothing interesting. Check the playlist, could be a bug or criteria could be too stringent.')
else:
- print 'Found ',len(notflat_list),' potentially interesting curves'
- print 'Regenerating playlist...'
- self.pointer=0
- self.current_list=notflat_list
- self.current=self.current_list[self.pointer]
- self.do_plot(0)
-
- def has_features(self,item,median_filter,min_npks,min_deviation):
+ if len(features) < playlist.count:
+ self.AppendToOutput(''.join(['Found ', str(len(features)), ' potentially interesting files.']))
+ self.AppendToOutput('Regenerating playlist...')
+ playlist_filtered = playlist.filter_curves(features)
+ self.AddPlaylist(playlist_filtered, name='flatfilt')
+ else:
+ self.AppendToOutput('No files filtered. Try different filtering criteria.')
+
+ def has_features(self, current_file):
'''
decides if a curve is flat enough to be rejected from analysis: it sees if there
are at least min_npks points that are higher than min_deviation times the absolute value
Algorithm original idea by Francesco Musiani, with my tweaks and corrections.
'''
- retvalue=False
+ #TODO: shoudl medianfilter be variable?
+ medianfilter = 7
+ #medianfilter = self.GetIntFromConfig('flatfilts', 'flatfilt', 'median_filter')
+ mindeviation = self.GetIntFromConfig('flatfilts', 'flatfilt', 'min_deviation')
+ minpeaks = self.GetIntFromConfig('flatfilts', 'flatfilt', 'min_npks')
+
+ retvalue = 0
- item.identify(self.drivers)
#we assume the first is the plot with the force curve
#do the median to better resolve features from noise
- flat_plot=self.plotmanip_median(item.curve.default_plots()[0], item, customvalue=median_filter)
- flat_vects=flat_plot.vectors
- item.curve.close_all()
- #needed to avoid *big* memory leaks!
- del item.curve
- del item
+ flat_curve = self.plotmanip_median(current_file.plot, current_file, customvalue=medianfilter)
#absolute value of derivate
- yretdiff=diff(flat_vects[1][1])
- yretdiff=[abs(value) for value in yretdiff]
+ yretdiff = diff(flat_curve.curves[lh.RETRACTION].y)
+ yretdiff = [abs(value) for value in yretdiff]
#average of derivate values
- diffmean=numpy.mean(yretdiff)
+ diffmean = mean(yretdiff)
yretdiff.sort()
yretdiff.reverse()
- c_pks=0
+ c_pks = 0
for value in yretdiff:
- if value/diffmean > min_deviation:
- c_pks+=1
+ if value / diffmean > mindeviation:
+ c_pks += 1
else:
break
- if c_pks>=min_npks:
+ if c_pks >= minpeaks:
retvalue = c_pks
- del flat_plot, flat_vects, yretdiff
-
return retvalue
################################################################
#-----CONVFILT-------------------------------------------------
#-----Convolution-based peak recognition and filtering.
- #Requires the libpeakspot.py library
+ #Requires the peakspot.py library
- def has_peaks(self, plot, abs_devs=None):
+ def has_peaks(self, plot=None, plugin=None):
'''
Finds peak position in a force curve.
- FIXME: should be moved in libpeakspot.py
'''
- if abs_devs==None:
- abs_devs=self.convfilt_config['mindeviation']
-
- xret=plot.vectors[1][0]
- yret=plot.vectors[1][1]
- #Calculate convolution.
- convoluted=lps.conv_dx(yret, self.convfilt_config['convolution'])
+ if plugin is None:
+ blindwindow = self.GetFloatFromConfig('flatfilts', 'convfilt', 'blindwindow')
+ #need to convert the string that contains the list into a list
+ convolution = eval(self.GetStringFromConfig('flatfilts', 'convfilt', 'convolution'))
+ maxcut = self.GetFloatFromConfig('flatfilts', 'convfilt', 'maxcut')
+ mindeviation = self.GetFloatFromConfig('flatfilts', 'convfilt', 'mindeviation')
+ positive = self.GetBoolFromConfig('flatfilts', 'convfilt', 'positive')
+ seedouble = self.GetIntFromConfig('flatfilts', 'convfilt', 'seedouble')
+ stable = self.GetFloatFromConfig('flatfilts', 'convfilt', 'stable')
+ else:
+ blindwindow = self.GetFloatFromConfig(plugin.name, plugin.section, plugin.prefix + 'blindwindow')
+ #need to convert the string that contains the list into a list
+ convolution = eval(self.GetStringFromConfig(plugin.name, plugin.section, plugin.prefix + 'convolution'))
+ maxcut = self.GetFloatFromConfig(plugin.name, plugin.section, plugin.prefix + 'maxcut')
+ mindeviation = self.GetFloatFromConfig(plugin.name, plugin.section, plugin.prefix + 'mindeviation')
+ positive = self.GetBoolFromConfig(plugin.name, plugin.section, plugin.prefix + 'positive')
+ seedouble = self.GetIntFromConfig(plugin.name, plugin.section, plugin.prefix + 'seedouble')
+ stable = self.GetFloatFromConfig(plugin.name, plugin.section, plugin.prefix + 'stable')
+
+ if plot is None:
+ plot = self.GetDisplayedPlotCorrected()
+
+ retraction = plot.curves[lh.RETRACTION]
+ #Calculate convolution
+ convoluted = lps.conv_dx(retraction.y, convolution)
#surely cut everything before the contact point
- cut_index=self.find_contact_point(plot)
+ cut_index = self.find_contact_point(plot)
#cut even more, before the blind window
- start_x=xret[cut_index]
- blind_index=0
- for value in xret[cut_index:]:
- if abs((value) - (start_x)) > self.convfilt_config['blindwindow']*(10**-9):
+ start_x = retraction.x[cut_index]
+ blind_index = 0
+ for value in retraction.x[cut_index:]:
+ if abs((value) - (start_x)) > blindwindow * (10 ** -9):
break
- blind_index+=1
- cut_index+=blind_index
+ blind_index += 1
+ cut_index += blind_index
#do the dirty convolution-peak finding stuff
- noise_level=lps.noise_absdev(convoluted[cut_index:], self.convfilt_config['positive'], self.convfilt_config['maxcut'], self.convfilt_config['stable'])
- above=lps.abovenoise(convoluted,noise_level,cut_index,abs_devs)
- peak_location,peak_size=lps.find_peaks(above,seedouble=self.convfilt_config['seedouble'])
-
+ noise_level = lps.noise_absdev(convoluted[cut_index:], positive, maxcut, stable)
+ above = lps.abovenoise(convoluted, noise_level, cut_index, mindeviation)
+ peak_location, peak_size = lps.find_peaks(above, seedouble=seedouble)
#take the maximum
for i in range(len(peak_location)):
- peak=peak_location[i]
- maxpk=min(yret[peak-10:peak+10])
- index_maxpk=yret[peak-10:peak+10].index(maxpk)+(peak-10)
- peak_location[i]=index_maxpk
+ peak = peak_location[i]
+ maxpk = min(retraction.y[peak - 10:peak + 10])
+ index_maxpk = retraction.y[peak - 10:peak + 10].index(maxpk) + (peak - 10)
+ peak_location[i] = index_maxpk
- return peak_location,peak_size
-
-
- def exec_has_peaks(self,item,abs_devs):
- '''
- encapsulates has_peaks for the purpose of correctly treating the curve objects in the convfilt loop,
- to avoid memory leaks
- '''
- item.identify(self.drivers)
- #we assume the first is the plot with the force curve
- plot=item.curve.default_plots()[0]
-
- if 'flatten' in self.config['plotmanips']:
- #If flatten is present, use it for better recognition of peaks...
- flatten=self._find_plotmanip('flatten') #extract flatten plot manipulator
- plot=flatten(plot, item, customvalue=1)
-
- peak_location,peak_size=self.has_peaks(plot,abs_devs)
- #close all open files
- item.curve.close_all()
- #needed to avoid *big* memory leaks!
- del item.curve
- del item
return peak_location, peak_size
- #------------------------
- #------commands----------
- #------------------------
- def do_peaks(self,args):
+ def do_peaks(self, plugin=None, peak_location=None, peak_size=None):
'''
- PEAKS
- (flatfilts.py)
- Test command for convolution filter / test.
+ Test command for convolution filter.
----
Syntax: peaks [deviations]
absolute deviation = number of times the convolution signal is above the noise absolute deviation.
Default is 5.
'''
- if len(args)==0:
- args=self.convfilt_config['mindeviation']
- try:
- abs_devs=float(args)
- except:
- print 'Wrong argument, using config value'
- abs_devs=float(self.convfilt_config['mindeviation'])
-
- defplots=self.current.curve.default_plots()[0] #we need the raw, uncorrected plots
-
- if 'flatten' in self.config['plotmanips']:
- flatten=self._find_plotmanip('flatten') #extract flatten plot manipulator
- defplots=flatten(defplots, self.current)
- else:
- print 'You have the flatten plot manipulator not loaded. Enabling it could give you better results.'
-
- peak_location,peak_size=self.has_peaks(defplots,abs_devs)
- print 'Found '+str(len(peak_location))+' peaks.'
- to_dump='peaks '+self.current.path+' '+str(len(peak_location))
- self.outlet.push(to_dump)
- #print peak_location
-
- #if no peaks, we have nothing to plot. exit.
- if len(peak_location)==0:
- return
-
- #otherwise, we plot the peak locations.
- xplotted_ret=self.plots[0].vectors[1][0]
- yplotted_ret=self.plots[0].vectors[1][1]
- xgood=[xplotted_ret[index] for index in peak_location]
- ygood=[yplotted_ret[index] for index in peak_location]
-
- recplot=self._get_displayed_plot()
- recplot.vectors.append([xgood,ygood])
- if recplot.styles==[]:
- recplot.styles=[None,None,'scatter']
- recplot.colors=[None,None,None]
+ if plugin is None:
+ color = self.GetColorFromConfig('flatfilts', 'peaks', 'color')
+ size = self.GetIntFromConfig('flatfilts', 'peaks', 'size')
else:
- recplot.styles+=['scatter']
- recplot.colors+=[None]
+ color = self.GetColorFromConfig(plugin.name, plugin.section, plugin.prefix + 'color')
+ size = self.GetIntFromConfig(plugin.name, plugin.section, plugin.prefix + 'size')
- self._send_plot([recplot])
-
- def do_convfilt(self,args):
- '''
- CONVFILT
- (flatfilts.py)
- Filters out flat (featureless) curves of the current playlist,
- creating a playlist containing only the curves with potential
- features.
- ------------
- Syntax:
- convfilt [min_npks min_deviation]
+ plot = self.GetDisplayedPlotCorrected()
- min_npks = minmum number of peaks
- (to set the default, see convfilt.conf file; CONVCONF and SETCONF commands)
+ if peak_location is None and peak_size is None:
+ if not self.AppliesPlotmanipulator('flatten'):
+ self.AppendToOutput('The flatten plot manipulator is not loaded. Enabling it could give better results.')
- min_deviation = minimum signal/noise ratio *in the convolution*
- (to set the default, see convfilt.conf file; CONVCONF and SETCONF commands)
+ peak_location, peak_size = self.has_peaks(plot)
+ if len(peak_location) != 1:
+ peak_str = ' peaks.'
+ else:
+ peak_str = ' peak.'
+ self.AppendToOutput('Found ' + str(len(peak_location)) + peak_str)
- If called without arguments, it uses default values.
- '''
+ if peak_location:
+ retraction = plot.curves[lh.RETRACTION]
- min_npks=self.convfilt_config['minpeaks']
- min_deviation=self.convfilt_config['mindeviation']
+ peaks = lib.curve.Curve()
+ peaks.color = color
+ peaks.size = size
+ peaks.style = 'scatter'
+ peaks.title = 'Peaks'
+ peaks.x = [retraction.x[index] for index in peak_location]
+ peaks.y = [retraction.y[index] for index in peak_location]
- args=args.split(' ')
- if len(args) == 2:
- min_npks=int(args[0])
- min_deviation=int(args[1])
- else:
- pass
+ plot.curves.append(peaks)
- print 'Processing playlist...'
- print '(Please wait)'
- notflat_list=[]
+ self.UpdatePlot(plot)
- c=0
- for item in self.current_list:
- c+=1
+ def do_convfilt(self):
+ '''
+ Filters out flat (featureless) files of the current playlist,
+ creating a playlist containing only the files with potential
+ features.
+ ------------
+ min_npks: minmum number of peaks
+ min_deviation: minimum signal/noise ratio *in the convolution*
+ '''
+ self.AppendToOutput('Processing playlist...')
+ self.AppendToOutput('(Please wait)')
+ apply_plotmanipulators = self.GetStringFromConfig('flatfilts', 'convfilt', 'apply_plotmanipulators')
+ minpeaks = self.GetIntFromConfig('flatfilts', 'convfilt', 'minpeaks')
+ features = []
+ playlist = self.GetActivePlaylist()
+
+ files = self.GetActivePlaylist().files
+ file_index = 0
+ for current_file in files:
+ number_of_peaks = 0
+ file_index += 1
try:
- peak_location,peak_size=self.exec_has_peaks(item,min_deviation)
- if len(peak_location)>=min_npks:
- isok='+'
+ current_file.identify(self.drivers)
+ if apply_plotmanipulators == 'all':
+ plot = self.ApplyPlotmanipulators(current_file.plot, current_file)
+ if apply_plotmanipulators == 'flatten':
+ plotmanipulator = self.GetPlotmanipulator('flatten')
+ plot = plotmanipulator.method(current_file.plot, current_file)
+ if apply_plotmanipulators == 'none':
+ plot = copy.deepcopy(current_file.plot)
+
+ peak_location, peak_size = self.has_peaks(plot)
+ number_of_peaks = len(peak_location)
+ if number_of_peaks != 1:
+ if number_of_peaks > 0:
+ feature_string = str(number_of_peaks) + ' features'
+ else:
+ feature_string = 'no features'
else:
- isok=''
- print 'Curve',item.path, 'is',c,'of',len(self.current_list),': found '+str(len(peak_location))+' peaks.'+isok
+ feature_string = '1 feature'
+ if number_of_peaks >= minpeaks:
+ feature_string += '+'
+ output_string = ''.join(['Curve ', current_file.name, '(', str(file_index), '/', str(len(files)), '): ', feature_string])
except:
- peak_location,peak_size=[],[]
- print 'Curve',item.path, 'is',c,'of',len(self.current_list),': cannot be filtered. Probably unable to retrieve force data from corrupt file.'
-
- if len(peak_location)>=min_npks:
- item.peak_location=peak_location
- item.peak_size=peak_size
- item.curve=None #empty the item object, to further avoid memory leak
- notflat_list.append(item)
+ peak_location = []
+ peak_size = []
+ output_string = ''.join(['Curve ', current_file.name, '(', str(file_index), '/', str(len(files)), '): cannot be filtered. Probably unable to retrieve force data from corrupt file.'])
+ self.AppendToOutput(output_string)
+ if number_of_peaks >= minpeaks:
+ current_file.peak_location = peak_location
+ current_file.peak_size = peak_size
+ features.append(file_index - 1)
#Warn that no flattening had been done.
- if not ('flatten' in self.config['plotmanips']):
- print 'Flatten manipulator was not found. Processing was done without flattening.'
- print 'Try to enable it in your configuration file for better results.'
-
- if len(notflat_list)==0:
- print 'Found nothing interesting. Check your playlist, could be a bug or criteria could be too much stringent'
- return
+ if not self.HasPlotmanipulator('plotmanip_flatten'):
+ self.AppendToOutput('Flatten manipulator was not found. Processing was done without flattening.')
else:
- print 'Found ',len(notflat_list),' potentially interesting curves'
- print 'Regenerating playlist...'
- self.pointer=0
- self.current_list=notflat_list
- self.current=self.current_list[self.pointer]
- self.do_plot(0)
+ if not self.AppliesPlotmanipulator('flatten'):
+ self.AppendToOutput('Flatten manipulator was not applied.')
+ self.AppendToOutput('Try to enable the flatten plotmanipulator for better results.')
-
- def do_setconv(self,args):
- '''
- SETCONV
- (flatfilts.py)
- Sets the convfilt configuration variables
- ------
- Syntax: setconv variable value
- '''
- args=args.split()
- #FIXME: a general "set dictionary" function has to be built
- if len(args)==0:
- print self.convfilt_config
+ if not features:
+ self.AppendToOutput('Found nothing interesting. Check the playlist, could be a bug or criteria could be too stringent.')
else:
- if not (args[0] in self.convfilt_config.keys()):
- print 'This is not an internal convfilt variable!'
- print 'Run "setconv" without arguments to see a list of defined variables.'
- return
-
- if len(args)==1:
- print self.convfilt_config[args[0]]
- elif len(args)>1:
- try:
- self.convfilt_config[args[0]]=eval(args[1])
- except NameError: #we have a string argument
- self.convfilt_config[args[0]]=args[1]
-
-
-#########################
-#HANDLING OF CONFIGURATION FILE
-class ConvfiltConfig(object):
- '''
- Handling of convfilt configuration file
-
- Mostly based on the simple-yet-useful examples of the Python Library Reference
- about xml.dom.minidom
-
- FIXME: starting to look a mess, should require refactoring
- '''
-
- def __init__(self):
- self.config={}
-
-
- def load_config(self, filename):
- myconfig=file(filename)
- #the following 3 lines are needed to strip newlines. otherwise, since newlines
- #are XML elements too, the parser would read them (and re-save them, multiplying
- #newlines...)
- #yes, I'm an XML n00b
- the_file=myconfig.read()
- the_file_lines=the_file.split('\n')
- the_file=''.join(the_file_lines)
-
- self.config_tree=xml.dom.minidom.parseString(the_file)
-
- def getText(nodelist):
- #take the text from a nodelist
- #from Python Library Reference 13.7.2
- rc = ''
- for node in nodelist:
- if node.nodeType == node.TEXT_NODE:
- rc += node.data
- return rc
-
- def handleConfig(config):
- noiseabsdev_elements=config.getElementsByTagName("noise_absdev")
- convfilt_elements=config.getElementsByTagName("convfilt")
- handleAbsdev(noiseabsdev_elements)
- handleConvfilt(convfilt_elements)
-
- def handleAbsdev(noiseabsdev_elements):
- for element in noiseabsdev_elements:
- for attribute in element.attributes.keys():
- self.config[attribute]=element.getAttribute(attribute)
-
- def handleConvfilt(convfilt_elements):
- for element in convfilt_elements:
- for attribute in element.attributes.keys():
- self.config[attribute]=element.getAttribute(attribute)
-
- handleConfig(self.config_tree)
- #making items in the dictionary machine-readable
- for item in self.config.keys():
- try:
- self.config[item]=eval(self.config[item])
- except NameError: #if it's an unreadable string, keep it as a string
- pass
+ if len(features) < playlist.count:
+ self.AppendToOutput(''.join(['Found ', str(len(features)), ' potentially interesting files.']))
+ self.AppendToOutput('Regenerating playlist...')
+ playlist_filtered = playlist.filter_curves(features)
+ self.AddPlaylist(playlist_filtered, name='convfilt')
+ else:
+ self.AppendToOutput('No files filtered. Try different filtering criteria.')
- return self.config
projects / hooke.git / blobdiff