-odac --m-amps=0 sr = 44100 ksmps = 64 nchnls = 2 0dbfs = 1 /* * MFCC matching / concatenative resynthesis example * By Richard Knight 2021 * * Interactive FLTK interface example * * See README.md for overview and usage details. * */ ; put your own sounds here and they will be analysed/added to available sounds at startup. ;gSsounds[] fillarray "/path/to/sound1.wav", "/path/to/sound2.wav" ; default is to use any sounds in the sounds subdirectory gSsounds[] directory "sounds", ".wav" ; FFT size for MFCC analysis (lower = more CPU) gifftsize = 1024 ; Number of MFCC bands to use (^2, ideally 8, 16, 32) gimfccbands = 16 ; instrument numbers for updateui and player, as used in FL opcode calls iupdateui nstrnum "updateui" iplayer nstrnum "player" /* * Return element after last slash */ opcode strfilename, S, S Sinput xin xout strsub(Sinput, strrindex(Sinput, "/") + 1) endop ; the UI FLpanel "MFCC Matching", 800, 500 gkcorpusindex, gicorpusindex FLcount "Corpus", 0, lenarray(gSsounds)-1, 1, 1, 1, 200, 50, 0, 0, 0, iupdateui, 0, 1, 1 ; updateui item 1 gicorpusbox FLbox strfilename(gSsounds[0]), 1, 1, 18, 300, 50, 0, 100 FLsetTextSize 24, gicorpusindex FLsetFont 2, gicorpusindex gkdriverindex, gidriverindex FLcount "Driver", 0, lenarray(gSsounds)-1, 1, 1, 1, 200, 50, 600, 0, 0, iupdateui, 0, 1, 2 ; updateui item 2 gidriverbox FLbox strfilename(gSsounds[0]), 1, 1, 18, 300, 50, 500, 100 FLsetTextSize 24, gidriverindex FLsetFont 2, gidriverindex gkxfade, gixfade FLslider "Crossfade", 0, 1, 0, 5, -1, 400, 50, 200, 150 FLsetTextSize 24, gixfade FLsetFont 2, gixfade FLsetVal_i 0.5, gixfade gkblur, giblur FLbutton "Blur", 1, 0, 3, 100, 30, 0, 300, 0, iupdateui, 0, 1, 3 ; updateui item 3 gkblurtime, giblurtime FLslider "Blur time", 0, 1, 0, 5, -1, 300, 30, 100, 300 gkenvelope, gienvelope FLslider "Grain envelope", 0.1, 0.9, 0, 5, -1, 400, 20, 0, 350 FLsetVal_i 0.4, gienvelope gkgrainsize, gigrainsize FLslider "Grain size", 0.01, 3, 0, 5, -1, 400, 20, 0, 400 FLsetVal_i 1.2, gigrainsize gkpitchshift, gipitchshift FLbutton "Repitch", 1, 0, 3, 100, 30, 0, 450, -1 gkmatchamps, gimatchamps FLbutton "Match amps", 1, 0, 3, 100, 30, 200, 450, -1 gkreadmode, gireadmode FLcount "Read mode", 0, 1, 1, 1, 1, 150, 30, 0, 250, 0, iupdateui, 0, 1, 4 ; updateui item 4 gkstretch, gistretch FLslider "Stretch", 0, 0.999, 0, 5, -1, 250, 30, 150, 250 gkplay, giplay FLbutton "Play/Stop", 1, 0, 1, 200, 50, 300, 0, 0, iplayer, 0, 200 FLsetTextSize 24, giplay FLsetColor 40, 140, 40, giplay giloadingbox FLbox "Analysing", 1, 1, 72, 800, 500, 0, 0 FLpanelEnd FLrun ; not shown to begin with FLhide giblurtime FLhide gistretch /* * Alter UI elements according to specified item/mode */ instr updateui item = p4 if (item == 1) then ; corpus sound index changed FLsetText strfilename(gSsounds[i(gkcorpusindex)]), gicorpusbox elseif (item == 2) then ; driver sound index changed FLsetText strfilename(gSsounds[i(gkdriverindex)]), gidriverbox elseif (item == 3) then ; blur checkbox changed if (i(gkblur) == 1) then FLshow giblurtime else FLhide giblurtime endif elseif (item == 4) then ; read mode changed if (i(gkreadmode) == 1) then FLshow gistretch FLhide gienvelope FLhide gigrainsize else FLhide gistretch FLshow gienvelope FLshow gigrainsize endif endif turnoff endin ; The corpus sounds gicorpuswave[] init lenarray(gSsounds) ; ftables for MFCC corpus data, initialise to -1 and create in analysis instrument gicorpus[] init lenarray(gSsounds) ; allow only one playing instance giplaying = 0 ; include analysis and matching opcodes #include "mfcc_matching.udo" /* * Step through gSsounds by index (p4), load to ftable and perform mfcc analysis */ instr analyseloop index = p4 if (index > lenarray(gSsounds) - 1) then ; everything analysed, show UI accordingly FLhide giloadingbox else ; wave still to analyse gicorpuswave[index] ftgen 0, 0, 0, 1, gSsounds[index], 0, 0, 1 event_i "i", "analysecorpus", 0, 1, index endif turnoff endin /* * Analyse corpus sound in gicorpuswave, in one k-cycle, * then invoke the "analyseloop" instrument again to continue */ instr analysecorpus index = p4 ilen = ftlen(gicorpuswave[index]) / ftsr(gicorpuswave[index]) imaxitems = gimfccbands * (ftlen(gicorpuswave[index]) / gifftsize) gicorpus[index] ftgen 0, 0, -imaxitems, 2, 0 ktimek timeinstk if (ktimek == 1) then kcycles = ilen*kr kcount init 0 loop: asig loscil 1, 1, gicorpuswave[index], 1 kdx init 0 kmfcc[], ktrig getmfccs asig, gifftsize, gimfccbands if (ktrig == 1) then kfb = 0 while (kfb < gimfccbands) do tabw kmfcc[kfb], kdx, gicorpus[index] kfb += 1 kdx += 1 od endif loop_lt kcount, 1, kcycles, loop else schedkwhen 1, 1, 1, "analyseloop", 0, 1, index+1 turnoff endif endin /* * Play the driver sound, obtain the nearest matching index of the corpus sound, and * then play that from the relevant starting index with the "segment" instrument. * Output only the left channel for audible comparison purposes. */ instr player ; if playing, turn off, otherwise play if (giplaying == 1) then ; set play/stop button colour to green and turn off FLsetColor 40, 140, 40, giplay giplaying = 0 turnoff2 p1, 0, 0 else ; set play/stop button colour to red FLsetColor 140, 40, 40, giplay giplaying = 1 idriverfn = gicorpuswave[i(gkdriverindex)] iduration = ftlen(idriverfn) / ftsr(idriverfn) p3 = iduration ; after duration, call same instrument to set giplaying to 0 ktime timeinsts if (ktime >= iduration) then event "i", "player", 0, 0.1 endif ; read the driving sound adriver loscil 1, 1, idriverfn, 1 ; do the actual matching to find the best sample point kdx, ktrig nearest adriver, gifftsize, gimfccbands, gicorpus[i(gkcorpusindex)] ; segment time according to the MFCC analysis FFT size isegmenttime = (1/sr) * gifftsize ksegmenttime = isegmenttime * gkgrainsize if (gkreadmode == 0) then ; each grain is an instrument call that chnmixes to "segments" schedkwhen ktrig, 0, 0, "segment", random:k(0, isegmenttime), ksegmenttime, gicorpuswave[i(gkcorpusindex)], kdx amatched chnget "segments" chnclear "segments" else ; grains are read using sndwarp ilen = ftlen(gicorpuswave[i(gkcorpusindex)]) icsr = ftsr(gicorpuswave[i(gkcorpusindex)]) icduration = ilen / icsr icps = 1/(ilen/icsr) aphs, a_ syncphasor icps*(1-gkstretch), a(ktrig) apos = (((aphs * ilen) + kdx) / ilen) * icduration amatched sndwarp 0.7, apos, 1, gicorpuswave[i(gkcorpusindex)], 0, gifftsize/2, 64, 4, 99, 1 endif ; delay to account for matching adriver vdelay adriver, (1/sr)*gifftsize, 1 ; if pvs modifications are required if (gkblur == 1 || gkpitchshift == 1) then ir = 1024 fsegments pvsanal amatched, ir, ir/4, ir, 1 if (gkpitchshift == 1) then ; additional pitch matching; rough at the moment fsource pvsanal adriver, ir, ir/4, ir, 1 kfrsrc, kasrc pvspitch fsource, 0.1 kfrseg, kaseg pvspitch fsegments, 0.1 kpscale init 1 if (kfrseg != 0) then kpscale = kfrsrc / kfrseg endif fpitchmatched pvscale fsegments, kpscale if (gkblur == 1) then fouts pvsblur fpitchmatched, (gkblurtime*ksegmenttime*2)+(ksegmenttime*0.5), isegmenttime*6 else fouts = fpitchmatched endif elseif (gkblur == 1) then fouts pvsblur fsegments, (gkblurtime*ksegmenttime*2)+(ksegmenttime*0.5), isegmenttime*6 adriver vdelay adriver, (1/sr)*ir, 1 endif amatched pvsynth fouts ;asegments balance asegmentstemp, asegments endif if (gkmatchamps == 1) then amatched balance amatched, adriver endif ; output with crossfader aout = (adriver*gkxfade) + (amatched*(1-gkxfade)) outs aout, aout fout "D:/mfxx.wav", 14, aout endif endin /* * Play sound grain from table p4 with the start index p5, with basic enveloping. */ instr segment imidpoint = i(gkenvelope) ifadepoint = (1-imidpoint)*0.5 iamp = 1 ifn = p4 il = ftlen(ifn) isec = il/sr ist = p5 icps = 1/isec aphs phasor icps andx = aphs * il aout tablei andx+ist, ifn aout *= linseg:a(0, p3*ifadepoint, iamp, p3*imidpoint, iamp, p3*ifadepoint, 0) chnmix aout, "segments" endin f99 0 512 9 0.5 1 0 ; half sine f0 3600 i"analyseloop" 0 1 0