updated to new bench-pitch, fixed gettruth

updated to new bench-pitch, fixed gettruth

diff --git a/python/aubio/bench/node.py b/python/aubio/bench/node.py
index 15969d39a258da45cdedd4424ff351a89340d24c..7a42211af05593dfc4d5c9d9f30f67e07cdc08fb 100644 (file)
--- a/python/aubio/bench/node.py
+++ b/python/aubio/bench/node.py
@@ -25,7 +25,7 @@ def list_files(datapath,filter='f', maxdepth = -1):
                sys.exit(1)
        if maxdepth >= 0: maxstring = " -maxdepth %d " % maxdepth       
        else: maxstring = ""
-        cmd = '%s' * 5 % ('find ',datapath,maxstring,' -type ',filter)
+        cmd = '%s' * 6 % ('find ',datapath,maxstring,' -type ',filter, "| sort -n")
         return runcommand(cmd)
 
 def list_wav_files(datapath,maxdepth = -1):

diff --git a/python/aubio/task/pitch.py b/python/aubio/task/pitch.py
index a2010aa5f0cce89b081c1528d6198f10de38dd98..ed33a7ce9422d73a4d0946b1446d1ccc77353822 100644 (file)
--- a/python/aubio/task/pitch.py
+++ b/python/aubio/task/pitch.py
@@ -60,9 +60,9 @@ class taskpitch(task):
                        return False,False
                elif floatpit:
                        try:
-                               self.truth = aubio_miditofreq(float(floatpit))
-                               print "ground truth found in filename:", self.truth
-                               tasksil = tasksilence(self.input)
+                               self.truth = float(floatpit)
+                               #print "ground truth found in filename:", self.truth
+                               tasksil = tasksilence(self.input,params=self.params)
                                time,pitch =[],[]
                                while(tasksil.readsize==tasksil.params.hopsize):
                                        tasksil()
@@ -83,10 +83,10 @@ class taskpitch(task):
                                        time, pitch = [], []
                                        for i in range(len(values)):
                                                time.append(values[i][0])
-                                               pitch.append(values[i][1])
+                                               pitch.append(aubio_freqtomidi(values[i][1]))
                                        return time,pitch
 
-       def eval(self,results):
+       def oldeval(self,results):
                def mmean(l):
                        return sum(l)/max(float(len(l)),1)
 
@@ -104,23 +104,44 @@ class taskpitch(task):
                        med = percental(res,len(res)/2) 
                return self.truth, self.truth-med, self.truth-avg
 
-       def neweval(self,results):
+       def eval(self,pitch,tol=0.9):
                timet,pitcht = self.gettruth()
-               for i in timet:
-                       print results[i]
-               return self.truth, self.truth-med, self.truth-avg
+               pitch = [aubio_freqtomidi(i) for i in pitch]
+               for i in range(len(pitch)):
+                       if pitch[i] == "nan" or pitch[i] == -1:
+                               pitch[i] = -1
+               time = [ i*self.params.step for i in range(len(pitch)) ]
+               assert len(timet) == len(time) 
+               assert len(pitcht) == len(pitch)
+               osil, esil, opit, epit, echr = 0, 0, 0, 0, 0
+               for i in range(len(pitcht)):
+                       if pitcht[i] == -1: # currently silent
+                               osil += 1 # count a silence
+                               if pitch[i] == -1. or pitch[i] == "nan": 
+                                       esil += 1 # found a silence
+                       else:
+                               opit +=1
+                               if abs(pitcht[i] - pitch[i]) < tol:
+                                       epit += 1
+                                       echr += 1
+                               elif abs(pitcht[i] - pitch[i]) % 12. < tol:
+                                       echr += 1
+                               #else:
+                               #       print timet[i], pitcht[i], time[i], pitch[i]
+               #print "origsilence", "foundsilence", "origpitch", "foundpitch", "orig pitchroma", "found pitchchroma"
+               #print 100.*esil/float(osil), 100.*epit/float(opit), 100.*echr/float(opit)
+               return osil, esil, opit, epit, echr
 
        def plot(self,pitch,wplot,oplots,outplot=None):
                import numarray
                import Gnuplot
 
-               self.eval(pitch)
                downtime = self.params.step*numarray.arange(len(pitch))
-               oplots.append(Gnuplot.Data(downtime,pitch,with='lines',
+               oplots.append(Gnuplot.Data(downtime,pitch,with='linespoints',
                        title=self.params.pitchmode))
 
                        
-       def plotplot(self,wplot,oplots,outplot=None,multiplot = 1):
+       def plotplot(self,wplot,oplots,outplot=None,multiplot = 0):
                from aubio.gnuplot import gnuplot_init, audio_to_array, make_audio_plot
                import re
                import Gnuplot
@@ -131,6 +152,7 @@ class taskpitch(task):
                # check if ground truth exists
                timet,pitcht = self.gettruth()
                if timet and pitcht:
+                       pitcht = [aubio_miditofreq(i) for i in pitcht]
                        oplots = [Gnuplot.Data(timet,pitcht,with='lines',
                                title='ground truth')] + oplots
 

diff --git a/python/test/bench/pitch/bench-pitch b/python/test/bench/pitch/bench-pitch
index 7ad73517481aa03b4845ecafac62c242be3a829e..21b7093503ed418081d35c6bae094e8a8969a7f7 100755 (executable)
--- a/python/test/bench/pitch/bench-pitch
+++ b/python/test/bench/pitch/bench-pitch
@@ -8,61 +8,104 @@ class benchpitch(bench):
        """ list of values to store per file """
        valuenames = ['mode']
        """ list of lists to store per file """
-       valuelists = ['orig', 'mean', 'med']
+       valuelists = ['truth', 'osil', 'esil', 'opit', 'epit', 'echr', 
+               'Msil', 'Mpit', 'Mchr']
        """ list of values to print per dir """
-       printnames = [ 'mode']
+       printnames = [ 'mode', 'truth', 'Msil', 'Mpit', 'Mchr']
 
        """ per dir """
-       formats = {'mode': "%12s" , 'thres': "%5.4s", 
-               'dist':  "%5.4s", 'prec': "%5.4s", 'recl':  "%5.4s",
-               'Ttrue': "%5.4s", 'Tfp':   "%5.4s", 'Tfn':   "%5.4s", 
-               'Tm':    "%5.4s", 'Td':    "%5.4s",
-               'aTtrue':"%5.4s", 'aTfp':  "%5.4s", 'aTfn':  "%5.4s", 
-               'aTm':   "%5.4s", 'aTd':   "%5.4s",
-               'mean':  "%5.40s", 'smean': "%5.40s", 
-               'amean':  "%5.40s", 'samean': "%5.40s"}
+       formats = {'mode': "%12s" , 
+               'truth': "%s",
+               'osil': "%s", 'esil': "%s", 
+               'opit': "%s", 'epit': "%s", 'echr': "%s",
+               'Msil': "%s", 'Mpit': "%s", 'Mchr': "%s"}
 
        def dir_eval(self):
                """ evaluate statistical data over the directory """
                v = self.v
-
                v['mode']      = self.params.pitchmode
 
        def file_exec(self,input,output):
                filetask = self.task(input,params=self.params)
                computed_data = filetask.compute_all()
-               orig,mean,med = filetask.eval(computed_data)
+               osil, esil, opit, epit, echr = filetask.eval(computed_data)
+               self.v['truth'].append(int(filetask.truth))
+               assert opit > 0
                
-               self.v['orig'].append(orig)
-               self.v['mean'].append(mean)
-               self.v['med'].append(med)
+               self.v['osil'].append(osil)
+               self.v['esil'].append(esil)
+               self.v['opit'].append(opit)
+               self.v['epit'].append(epit)
+               self.v['echr'].append(echr)
+
+               self.v['Msil'].append(esil/float(osil)*100.)
+               self.v['Mpit'].append(epit/float(opit)*100.)
+               self.v['Mchr'].append(echr/float(opit)*100.)
                #print results#, computed_data
                #print input, results, results - float(input.split('.')[-2])
                        
        def run_bench(self,modes=['schmitt']):
                from os.path import basename
-               d = []
                self.modes = modes
                self.pretty_titles()
+               d = []
                for mode in self.modes:
                        self.params.pitchmode = mode
                        self.dir_eval_print()
-                       self.plotpitchtessiture(d,
-                               self.v['orig'], 
-                               self.v['med'],
-                               plottitle=self.v['mode'],
-                               plotmode='points')
-               #d.append('beta = .25,orig(x) title \"-2 octave\"')
-               d.append('beta = .50,orig(x) title \"-1 octave\"')
-               d.append('beta = 1.0,orig(x) title \"original\"')
-               d.append('beta = 2.0,orig(x) title \"+1 octave\"')
-               title = basename(self.datadir)
+                       truth   = [i for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       allOsil = [0 for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       allEsil = [0 for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       allOpit = [0 for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       allEpit = [0 for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       allEchr = [0 for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       allMsil = [0 for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       allMpit = [0 for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       allMchr = [0 for i in range(min(self.v['truth']),max(self.v['truth'])+1)]
+                       for i in range(len(self.v['truth'])):
+                               allOsil[self.v['truth'][i]-min(self.v['truth'])] += self.v['osil'][i]
+                               allEsil[self.v['truth'][i]-min(self.v['truth'])] += self.v['esil'][i]
+                               allOpit[self.v['truth'][i]-min(self.v['truth'])] += self.v['opit'][i]
+                               allEpit[self.v['truth'][i]-min(self.v['truth'])] += self.v['epit'][i]
+                               allEchr[self.v['truth'][i]-min(self.v['truth'])] += self.v['echr'][i]
+                       for i in range(len(truth)):
+                               allMsil[i] = allEsil[i]/float(allOsil[i])*100.
+                               allMpit[i] = allEpit[i]/float(allOpit[i])*100.
+                               allMchr[i] = allEchr[i]/float(allOpit[i])*100.
+
+                       plot = []
+                       self.plotpitchtessiture(plot,
+                               truth, 
+                               allMpit,
+                               plottitle="%s %s" % (self.v['mode'],self.params.bufsize),
+                               plotmode='lines')
+                       self.plotpitchtessiture(plot,
+                               truth, 
+                               allMchr,
+                               plottitle="%s %s" % (self.v['mode'],"%12"),
+                               plotmode='lines')
+                       """
+                       self.plotpitchtessiture(plot,
+                               truth, 
+                               allMsil,
+                               plottitle="%s %s" % (self.v['mode'],"sil"),
+                               plotmode='lines')
+                       """
+                       title = basename(self.datadir)
+                       d.append(plot)
                outplot = "_-_".join(('pitchtessiture',title))
+               self.xmin = 20. #min(self.v['truth'])
+               self.xmax = 110. #max(self.v['truth'])
                for ext in ('ps','png','svg',''):
                        self.plotplotpitchtessiture(d,
-                               plottitle=title,
+                               plottitle="".join(['Performance against MIDI Note number (',
+                                       title,
+                                       ", %s" % len(self.sndlist), " samples)"]),
                                outplot=outplot,
                                extension=ext)
+               #d.append('beta = .25,orig(x) title \"-2 octave\"')
+               #d.append('beta = .50,orig(x) title \"-1 octave\"')
+               #d.append('beta = 1.0,orig(x) title \"original\"')
+               #d.append('beta = 2.0,orig(x) title \"+1 octave\"')
 
        """
        Plot functions 
@@ -72,18 +115,38 @@ class benchpitch(bench):
                import Gnuplot, Gnuplot.funcutils
                d.append(Gnuplot.Data(lx, ly, with=plotmode, title="%s" % (plottitle) ))
 
-       def plotplotpitchtessiture(self,d,plottitle='',outplot=0,extension=''):
+       def plotplotpitchtessiture(self,d,plottitle='',outplot=0,extension='',multiplot=1):
                from aubio.gnuplot import gnuplot_create
                g = gnuplot_create(outplot=outplot,extension=extension) 
-               g.title(plottitle)
-               g('orig(x) = beta*x')
-               g.xlabel('original pitch (Hz)')
-               g.ylabel('detected pitch (Hz)')
-               g('set key left top')
-               g('set log xy')
-               g('set xrange [50:2000]')
-               g('set yrange [50:2000]')
-               g.plot(*d)
+               #g.title(plottitle)
+               #g('orig(x) = beta*x')
+               #g.xlabel('original pitch (Hz)')
+               #g.ylabel('detected pitch (Hz)')
+               #g('set key left top')
+               #g('set log xy')
+               #g('set xrange [50:2000]')
+               g('set yrange [0:100]')
+               #g.plot(*d)
+               if multiplot:
+                       g('set multiplot')
+                       for i in range(len(d)):
+                               # plot onset detection functions
+                               g('set size   1,%f' % ( 1.0/float(len(d)) ) )
+                               g('set origin 0,%f' % ( 1.0*float(len(d)-i-1)/float(len(d)) ) )
+                               # erase axis
+                               g('set border 3')
+                               g('set xtics nomirror')
+                               g('set ytics nomirror')
+                               g('set key left top')
+                               g('set xrange [%f:%f]' % (self.xmin,self.xmax)) #(self.xmax - (self.xmax-self.xmin)*5./4.,self.xmax))
+                               #g.ylabel('%Correct detections')
+                               if i == len(d)-1:
+                                       g.xlabel(plottitle)
+                               g.plot(*d[i])
+                               g('unset title')
+                       g('unset multiplot')
+               else:
+                       g.plot(*d)
 
 
 if __name__ == "__main__":
@@ -93,13 +156,15 @@ if __name__ == "__main__":
        if len(sys.argv) > 2:
                for each in sys.argv[3:-1]: print each
        modes = ['schmitt', 'yin', 'mcomb', 'fcomb']
-       #modes = ['fcomb']
+       modes = ['schmitt', 'yin', 'fcomb']
 
        params = taskparams()
-       params.bufsize = 4096 
-       params.hopsize = params.bufsize/4
-       params.silence = -1000.
-       params.pitchsmooth = 50
+       params.bufsize = 2048 
+       params.hopsize = params.bufsize/8
+       params.silence = -60.
+       params.pitchsmooth = 0 
+       params.pitchmax = 20000
+       params.pitchmin = 20
        benchpitch = benchpitch(datapath,params=params)
        benchpitch.task = taskpitch