diff options
Diffstat (limited to 'mfcc_match_fltk_demo.csd')
| -rw-r--r-- | mfcc_match_fltk_demo.csd | 330 |
1 files changed, 330 insertions, 0 deletions
diff --git a/mfcc_match_fltk_demo.csd b/mfcc_match_fltk_demo.csd new file mode 100644 index 0000000..47e7373 --- /dev/null +++ b/mfcc_match_fltk_demo.csd @@ -0,0 +1,330 @@ +<CsoundSynthesizer> +<CsOptions> +-odac +--m-amps=0 +</CsOptions> +<CsInstruments> +sr = 44100 +ksmps = 64 +nchnls = 2 +0dbfs = 1 + +/* + * MFCC matching / concatenative resynthesis example + * By Richard Knight 2021 + * + * Interactive FLTK interface example + * + * See README.md for overview and usage details. + * + */ + +; put your own sounds here and they will be analysed/added to available sounds at startup. +;gSsounds[] fillarray "/path/to/sound1.wav", "/path/to/sound2.wav" + +; default is to use any sounds in the sounds subdirectory +gSsounds[] directory "sounds", ".wav" + +; FFT size for MFCC analysis (lower = more CPU) +gifftsize = 1024 + +; Number of MFCC bands to use (^2, ideally 8, 16, 32) +gimfccbands = 16 + + + + +; instrument numbers for updateui and player, as used in FL opcode calls +iupdateui nstrnum "updateui" +iplayer nstrnum "player" + + +/* + * Return element after last slash + */ +opcode strfilename, S, S + Sinput xin + xout strsub(Sinput, strrindex(Sinput, "/") + 1) +endop + + +; the UI +FLpanel "MFCC Matching", 800, 500 + gkcorpusindex, gicorpusindex FLcount "Corpus", 0, lenarray(gSsounds)-1, 1, 1, 1, 200, 50, 0, 0, 0, iupdateui, 0, 1, 1 ; updateui item 1 + gicorpusbox FLbox strfilename(gSsounds[0]), 1, 1, 18, 300, 50, 0, 100 + FLsetTextSize 24, gicorpusindex + FLsetFont 2, gicorpusindex + + gkdriverindex, gidriverindex FLcount "Driver", 0, lenarray(gSsounds)-1, 1, 1, 1, 200, 50, 600, 0, 0, iupdateui, 0, 1, 2 ; updateui item 2 + gidriverbox FLbox strfilename(gSsounds[0]), 1, 1, 18, 300, 50, 500, 100 + FLsetTextSize 24, gidriverindex + FLsetFont 2, gidriverindex + + gkxfade, gixfade FLslider "Crossfade", 0, 1, 0, 5, -1, 400, 50, 200, 150 + FLsetTextSize 24, gixfade + FLsetFont 2, gixfade + FLsetVal_i 0.5, gixfade + + gkblur, giblur FLbutton "Blur", 1, 0, 3, 100, 30, 0, 300, 0, iupdateui, 0, 1, 3 ; updateui item 3 + gkblurtime, giblurtime FLslider "Blur time", 0, 1, 0, 5, -1, 300, 30, 100, 300 + + gkenvelope, gienvelope FLslider "Grain envelope", 0.1, 0.9, 0, 5, -1, 400, 20, 0, 350 + FLsetVal_i 0.4, gienvelope + + gkgrainsize, gigrainsize FLslider "Grain size", 0.01, 3, 0, 5, -1, 400, 20, 0, 400 + FLsetVal_i 1.2, gigrainsize + + gkpitchshift, gipitchshift FLbutton "Repitch", 1, 0, 3, 100, 30, 0, 450, -1 + gkmatchamps, gimatchamps FLbutton "Match amps", 1, 0, 3, 100, 30, 200, 450, -1 + + gkreadmode, gireadmode FLcount "Read mode", 0, 1, 1, 1, 1, 150, 30, 0, 250, 0, iupdateui, 0, 1, 4 ; updateui item 4 + gkstretch, gistretch FLslider "Stretch", 0, 0.999, 0, 5, -1, 250, 30, 150, 250 + + + gkplay, giplay FLbutton "Play/Stop", 1, 0, 1, 200, 50, 300, 0, 0, iplayer, 0, 200 + FLsetTextSize 24, giplay + FLsetColor 40, 140, 40, giplay + + giloadingbox FLbox "Analysing", 1, 1, 72, 800, 500, 0, 0 + +FLpanelEnd +FLrun + +; not shown to begin with +FLhide giblurtime +FLhide gistretch + + + + +/* + * Alter UI elements according to specified item/mode + */ +instr updateui + item = p4 + if (item == 1) then ; corpus sound index changed + FLsetText strfilename(gSsounds[i(gkcorpusindex)]), gicorpusbox + + elseif (item == 2) then ; driver sound index changed + FLsetText strfilename(gSsounds[i(gkdriverindex)]), gidriverbox + + elseif (item == 3) then ; blur checkbox changed + if (i(gkblur) == 1) then + FLshow giblurtime + else + FLhide giblurtime + endif + elseif (item == 4) then ; read mode changed + if (i(gkreadmode) == 1) then + FLshow gistretch + FLhide gienvelope + FLhide gigrainsize + else + FLhide gistretch + FLshow gienvelope + FLshow gigrainsize + endif + endif + turnoff +endin + + + + +; The corpus sounds +gicorpuswave[] init lenarray(gSsounds) + +; ftables for MFCC corpus data, initialise to -1 and create in analysis instrument +gicorpus[] init lenarray(gSsounds) + +; allow only one playing instance +giplaying = 0 + +; include analysis and matching opcodes +#include "mfcc_matching.udo" + + +/* + * Step through gSsounds by index (p4), load to ftable and perform mfcc analysis + */ +instr analyseloop + index = p4 + if (index > lenarray(gSsounds) - 1) then + + ; everything analysed, show UI accordingly + FLhide giloadingbox + else + + ; wave still to analyse + gicorpuswave[index] ftgen 0, 0, 0, 1, gSsounds[index], 0, 0, 1 + event_i "i", "analysecorpus", 0, 1, index + endif + turnoff +endin + + +/* + * Analyse corpus sound in gicorpuswave, in one k-cycle, + * then invoke the "analyseloop" instrument again to continue + */ +instr analysecorpus + index = p4 + ilen = ftlen(gicorpuswave[index]) / ftsr(gicorpuswave[index]) + imaxitems = gimfccbands * (ftlen(gicorpuswave[index]) / gifftsize) + gicorpus[index] ftgen 0, 0, -imaxitems, 2, 0 + ktimek timeinstk + if (ktimek == 1) then + kcycles = ilen*kr + kcount init 0 +loop: + asig loscil 1, 1, gicorpuswave[index], 1 + kdx init 0 + kmfcc[], ktrig getmfccs asig, gifftsize, gimfccbands + if (ktrig == 1) then + kfb = 0 + while (kfb < gimfccbands) do + tabw kmfcc[kfb], kdx, gicorpus[index] + kfb += 1 + kdx += 1 + od + endif + loop_lt kcount, 1, kcycles, loop + else + schedkwhen 1, 1, 1, "analyseloop", 0, 1, index+1 + turnoff + endif +endin + + +/* + * Play the driver sound, obtain the nearest matching index of the corpus sound, and + * then play that from the relevant starting index with the "segment" instrument. + * Output only the left channel for audible comparison purposes. + */ +instr player + ; if playing, turn off, otherwise play + if (giplaying == 1) then + + ; set play/stop button colour to green and turn off + FLsetColor 40, 140, 40, giplay + giplaying = 0 + turnoff2 p1, 0, 0 + else + + ; set play/stop button colour to red + FLsetColor 140, 40, 40, giplay + giplaying = 1 + + idriverfn = gicorpuswave[i(gkdriverindex)] + iduration = ftlen(idriverfn) / ftsr(idriverfn) + p3 = iduration + + ; after duration, call same instrument to set giplaying to 0 + ktime timeinsts + if (ktime >= iduration) then + event "i", "player", 0, 0.1 + endif + + ; read the driving sound + adriver loscil 1, 1, idriverfn, 1 + + ; do the actual matching to find the best sample point + kdx, ktrig nearest adriver, gifftsize, gimfccbands, gicorpus[i(gkcorpusindex)] + + ; segment time according to the MFCC analysis FFT size + isegmenttime = (1/sr) * gifftsize + ksegmenttime = isegmenttime * gkgrainsize + + if (gkreadmode == 0) then + + ; each grain is an instrument call that chnmixes to "segments" + schedkwhen ktrig, 0, 0, "segment", random:k(0, isegmenttime), ksegmenttime, gicorpuswave[i(gkcorpusindex)], kdx + amatched chnget "segments" + chnclear "segments" + else + + ; grains are read using sndwarp + ilen = ftlen(gicorpuswave[i(gkcorpusindex)]) + icsr = ftsr(gicorpuswave[i(gkcorpusindex)]) + icduration = ilen / icsr + icps = 1/(ilen/icsr) + aphs, a_ syncphasor icps*(1-gkstretch), a(ktrig) + apos = (((aphs * ilen) + kdx) / ilen) * icduration + amatched sndwarp 0.7, apos, 1, gicorpuswave[i(gkcorpusindex)], 0, gifftsize/2, 64, 4, 99, 1 + endif + + ; delay to account for matching + adriver vdelay adriver, (1/sr)*gifftsize, 1 + + ; if pvs modifications are required + if (gkblur == 1 || gkpitchshift == 1) then + ir = 1024 + fsegments pvsanal amatched, ir, ir/4, ir, 1 + + if (gkpitchshift == 1) then + + ; additional pitch matching; rough at the moment + fsource pvsanal adriver, ir, ir/4, ir, 1 + kfrsrc, kasrc pvspitch fsource, 0.1 + kfrseg, kaseg pvspitch fsegments, 0.1 + kpscale init 1 + + if (kfrseg != 0) then + kpscale = kfrsrc / kfrseg + endif + + fpitchmatched pvscale fsegments, kpscale + + + if (gkblur == 1) then + fouts pvsblur fpitchmatched, (gkblurtime*ksegmenttime*2)+(ksegmenttime*0.5), isegmenttime*6 + else + fouts = fpitchmatched + endif + + elseif (gkblur == 1) then + fouts pvsblur fsegments, (gkblurtime*ksegmenttime*2)+(ksegmenttime*0.5), isegmenttime*6 + adriver vdelay adriver, (1/sr)*ir, 1 + endif + amatched pvsynth fouts + ;asegments balance asegmentstemp, asegments + endif + + if (gkmatchamps == 1) then + amatched balance amatched, adriver + endif + ; output with crossfader + aout = (adriver*gkxfade) + (amatched*(1-gkxfade)) + outs aout, aout + fout "D:/mfxx.wav", 14, aout + endif +endin + +/* + * Play sound grain from table p4 with the start index p5, with basic enveloping. + */ +instr segment + imidpoint = i(gkenvelope) + ifadepoint = (1-imidpoint)*0.5 + iamp = 1 + ifn = p4 + il = ftlen(ifn) + isec = il/sr + ist = p5 + icps = 1/isec + aphs phasor icps + andx = aphs * il + aout tablei andx+ist, ifn + aout *= linseg:a(0, p3*ifadepoint, iamp, p3*imidpoint, iamp, p3*ifadepoint, 0) + chnmix aout, "segments" +endin + + + +</CsInstruments> +<CsScore> +f99 0 512 9 0.5 1 0 ; half sine +f0 3600 +i"analyseloop" 0 1 0 +</CsScore> +</CsoundSynthesizer> |