From d15e78188a9cdbd70640ac57e42d4a598c89b532 Mon Sep 17 00:00:00 2001 From: John Glover Date: Fri, 6 Jul 2012 14:53:01 +0100 Subject: [residual] Add C++ implementation of SMSResidual. --- simpl/residual.pxd | 11 ++++- simpl/residual.pyx | 95 ++++++++++++++++++++++++++++++++++------ src/simpl/residual.cpp | 81 +++++++++++++++++++++++++++++++++- src/simpl/residual.h | 33 ++++++++++++-- src/sms/sms.h | 2 +- src/sms/synthesis.c | 4 ++ tests/create_libsms_test_data.py | 74 +++++++++++++++++++++++++++++++ tests/test_residual.py | 90 +++++++++++++++++++++++++++++++++---- 8 files changed, 362 insertions(+), 28 deletions(-) diff --git a/simpl/residual.pxd b/simpl/residual.pxd index eac4c32..5980e33 100644 --- a/simpl/residual.pxd +++ b/simpl/residual.pxd @@ -12,7 +12,7 @@ from base import dtype cdef extern from "../src/simpl/residual.h" namespace "simpl": cdef cppclass c_Residual "simpl::Residual": - c_Synthesis() + c_Residual() int frame_size() void frame_size(int new_frame_size) int next_frame_size() @@ -27,4 +27,11 @@ cdef extern from "../src/simpl/residual.h" namespace "simpl": int original_size, double* original, int residual_size, double* residual) void synth_frame(c_Frame* frame) - vector[c_Frame*] synth(vector[c_Frame*] frames) + vector[c_Frame*] synth(int synth_size, double* synth, + int original_size, double* original) + + cdef cppclass c_SMSResidual "simpl::SMSResidual"(c_Residual): + c_SMSResidual() + int num_stochastic_coeffs() + void num_stochastic_coeffs(int new_num_stochastic_coeffs) + # int stochastic_type() diff --git a/simpl/residual.pyx b/simpl/residual.pyx index 58bc1ca..315a5a3 100644 --- a/simpl/residual.pyx +++ b/simpl/residual.pyx @@ -12,8 +12,12 @@ from base cimport c_Frame cdef class Residual: cdef c_Residual* thisptr - def __cinit__(self): self.thisptr = new c_Residual() - def __dealloc__(self): del self.thisptr + def __cinit__(self): + self.thisptr = new c_Residual() + + def __dealloc__(self): + if self.thisptr: + del self.thisptr property sampling_rate: def __get__(self): return self.thisptr.sampling_rate() @@ -25,7 +29,9 @@ cdef class Residual: property hop_size: def __get__(self): return self.thisptr.hop_size() - def __set__(self, int i): self.thisptr.hop_size(i) + def __set__(self, int i): + print 'setting hop size...' + self.thisptr.hop_size(i) def residual_frame(self, np.ndarray[dtype_t, ndim=1] synth, np.ndarray[dtype_t, ndim=1] original): @@ -47,17 +53,80 @@ cdef class Residual: self.thisptr.synth_frame(frame.thisptr) return frame.audio - def synth(self, frames): - cdef vector[c_Frame*] c_frames - for frame in frames: - c_frames.push_back((frame).thisptr) - cdef vector[c_Frame*] output_frames = self.thisptr.synth(c_frames) - cdef np.ndarray[dtype_t, ndim=1] output = np.zeros( - output_frames.size() * self.thisptr.hop_size() + def synth(self, np.ndarray[dtype_t, ndim=1] synth, np.ndarray[dtype_t, ndim=1] original): + # cdef vector[c_Frame*] c_frames + # cdef int i = 0 + # cdef int j = 0 + cdef int hop = self.thisptr.hop_size() + # cdef int n = min(len(synth), len(original)) + # cdef Frame f + + # while i < (n - hop): + # f = Frame(hop, alloc_memory=True) + # frame_audio = np.zeros(hop) + # frame_synth = np.zeros(hop) + # for j in xrange(hop): + # frame_audio[j] = original[i+j] + # frame_synth[j] = synth[i+j] + # f.audio = frame_audio + + # f.audio = original[i:i + hop] + # f.synth = synth[i:i + hop] + # f.synth = frame_synth + + # c_frames.push_back(f.thisptr) + + # c_frames.push_back(Frame(hop, alloc_memory=True).thisptr) + # c_frames[i / hop].audio = original[i:i + hop] + # for j in xrange(hop): + # c_frames[i / hop].audio()[j] = original[i + j] + + # print original[i:i + hop].data + # c_frames[i / hop].audio((original[i:i + hop].data)) + # c_frames[i / hop].synth = synth[i:i + hop] + # i += hop + + # for i in range(c_frames.size()): + # print c_frames[i].size() + + # print + # f = Fra + + cdef vector[c_Frame*] output_frames = self.thisptr.synth( + len(synth), synth.data, + len(original), original.data ) + + cdef np.ndarray[dtype_t, ndim=1] output = np.zeros(output_frames.size() * hop) cdef np.npy_intp shape[1] - shape[0] = self.thisptr.hop_size() + shape[0] = hop + for i in range(output_frames.size()): - frame_audio = np.PyArray_SimpleNewFromData(1, shape, np.NPY_DOUBLE, output_frames[i].synth()) - output[i * self.thisptr.hop_size():(i + 1) * self.thisptr.hop_size()] = frame_audio + frame_audio = np.PyArray_SimpleNewFromData(1, shape, np.NPY_DOUBLE, output_frames[i].synth_residual()) + output[i * hop:(i + 1) * hop] = frame_audio + return output + + +cdef class SMSResidual(Residual): + def __cinit__(self): + if self.thisptr: + del self.thisptr + self.thisptr = new c_SMSResidual() + + def __dealloc__(self): + if self.thisptr: + del self.thisptr + self.thisptr = 0 + + property hop_size: + def __get__(self): return self.thisptr.hop_size() + def __set__(self, int i): self.thisptr.hop_size(i) + + property num_stochastic_coeffs: + def __get__(self): return (self.thisptr).num_stochastic_coeffs() + def __set__(self, int i): (self.thisptr).num_stochastic_coeffs(i) + + # property stochastic_type: + # def __get__(self): return (self.thisptr).stochastic_type() + # def __set__(self, int i): (self.thisptr).stochastic_type(i) diff --git a/src/simpl/residual.cpp b/src/simpl/residual.cpp index a6ed226..5d3cb14 100644 --- a/src/simpl/residual.cpp +++ b/src/simpl/residual.cpp @@ -56,9 +56,88 @@ void Residual::synth_frame(Frame* frame) { } // Calculate and return a synthesised residual signal -Frames Residual::synth(Frames frames) { +Frames Residual::synth(Frames& frames) { for(int i = 0; i < frames.size(); i++) { synth_frame(frames[i]); } return frames; } + +Frames Residual::synth(int synth_size, sample* synth, + int original_size, sample* original) { + Frames frames; + + for(int i = 0; i < min(synth_size, original_size) - _hop_size; i += _hop_size) { + Frame* f = new Frame(_hop_size, true); + f->audio(&original[i]); + f->synth(&synth[i]); + synth_frame(f); + frames.push_back(f); + } + + return frames; +} + + +// --------------------------------------------------------------------------- +// SMSResidual +// --------------------------------------------------------------------------- + +SMSResidual::SMSResidual() { + sms_init(); + + sms_initResidualParams(&_residual_params); + _residual_params.hopSize = _hop_size; + sms_initResidual(&_residual_params); +} + +SMSResidual::~SMSResidual() { + sms_freeResidual(&_residual_params); + sms_free(); +} + +void SMSResidual::hop_size(int new_hop_size) { + _hop_size = new_hop_size; + + sms_freeResidual(&_residual_params); + _residual_params.hopSize = _hop_size; + sms_initResidual(&_residual_params); +} + +int SMSResidual::num_stochastic_coeffs() { + return _residual_params.nCoeffs; +} + +void SMSResidual::num_stochastic_coeffs(int new_num_stochastic_coeffs) { + sms_freeResidual(&_residual_params); + _residual_params.nCoeffs = new_num_stochastic_coeffs; + sms_initResidual(&_residual_params); +} + +// int SMSResidual::stochastic_type() { +// return _residual_params. +// } + +// void SMSResidual::stochastic_type(int new_stochastic_type) { +// } + +void SMSResidual::residual_frame(int synth_size, sample* synth, + int original_size, sample* original, + int residual_size, sample* residual) { + + sms_findResidual(synth_size, synth, original_size, original, &_residual_params); + + for(int i = 0; i < residual_size; i++) { + residual[i] = _residual_params.residual[i]; + } +} + +// Calculate and return one frame of the synthesised residual signal +void SMSResidual::synth_frame(Frame* frame) { + residual_frame(frame->size(), frame->synth(), + frame->size(), frame->audio(), + frame->size(), frame->residual()); + sms_approxResidual(frame->size(), frame->residual(), + frame->size(), frame->synth_residual(), + &_residual_params); +} diff --git a/src/simpl/residual.h b/src/simpl/residual.h index 5dd53f5..0125417 100644 --- a/src/simpl/residual.h +++ b/src/simpl/residual.h @@ -3,6 +3,10 @@ #include "base.h" +extern "C" { + #include "sms.h" +} + using namespace std; namespace simpl @@ -16,7 +20,7 @@ namespace simpl // --------------------------------------------------------------------------- class Residual { - private: + protected: int _frame_size; int _hop_size; int _sampling_rate; @@ -26,7 +30,7 @@ class Residual { int frame_size(); void frame_size(int new_frame_size); int hop_size(); - void hop_size(int new_hop_size); + virtual void hop_size(int new_hop_size); int sampling_rate(); void sampling_rate(int new_sampling_rate); @@ -38,10 +42,33 @@ class Residual { int residual_size, sample* residual); virtual void synth_frame(Frame* frame); - virtual Frames synth(Frames frames); + virtual Frames synth(Frames& frames); + virtual Frames synth(int synth_size, sample* synth, + int original_size, sample* original); }; +// --------------------------------------------------------------------------- +// SMSResidual +// --------------------------------------------------------------------------- +class SMSResidual : public Residual { + private: + SMSResidualParams _residual_params; + + public: + SMSResidual(); + ~SMSResidual(); + void hop_size(int new_hop_size); + int num_stochastic_coeffs(); + void num_stochastic_coeffs(int new_num_stochastic_coeffs); + // int stochastic_type(); + // void stochastic_type(int new_stochastic_type); + void residual_frame(int synth_size, sample* synth, + int original_size, sample* original, + int residual_size, sample* residual); + void synth_frame(Frame* frame); +}; + } // end of namespace Simpl #endif diff --git a/src/sms/sms.h b/src/sms/sms.h index 7372a8a..c72b6c1 100644 --- a/src/sms/sms.h +++ b/src/sms/sms.h @@ -193,7 +193,7 @@ typedef struct * This structure contains all the necessary settings and memory for residual synthesis. * */ -typedef struct +typedef struct SMSResidualParams { int samplingRate; int hopSize; diff --git a/src/sms/synthesis.c b/src/sms/synthesis.c index bc739d0..35e91a4 100644 --- a/src/sms/synthesis.c +++ b/src/sms/synthesis.c @@ -195,7 +195,9 @@ void sms_approxResidual(int sizeResidual, sfloat* residual, /* generate random phases */ for(i = 0; i < residualParams->sizeStocMagSpectrum; i++) + { residualParams->stocPhaseSpectrum[i] = TWO_PI * sms_random(); + } /* IFFT with 50% overlap */ sms_invQuickSpectrumW(residualParams->stocMagSpectrum, @@ -208,7 +210,9 @@ void sms_approxResidual(int sizeResidual, sfloat* residual, /* output */ for(i = 0; i < sizeApprox; i++) + { approx[i] = residualParams->approx[i] * residualParams->windowScale; + } } /*! \brief synthesizes one frame of SMS data diff --git a/tests/create_libsms_test_data.py b/tests/create_libsms_test_data.py index 553c292..05b8cee 100644 --- a/tests/create_libsms_test_data.py +++ b/tests/create_libsms_test_data.py @@ -251,11 +251,83 @@ def _harmonic_synthesis(det_synth_type): return synth_audio +def _residual_synthesis(): + pysms.sms_init() + snd_header = pysms.SMS_SndHeader() + + if(pysms.sms_openSF(audio_path, snd_header)): + raise NameError(pysms.sms_errorString()) + + analysis_params = _pysms_analysis_params(sampling_rate) + analysis_params.nStochasticCoeff = 128 + analysis_params.iStochasticType = pysms.SMS_STOC_APPROX + if pysms.sms_initAnalysis(analysis_params, snd_header) != 0: + raise Exception("Error allocating memory for analysis_params") + analysis_params.iSizeSound = num_samples + analysis_params.nFrames = num_frames + sms_header = pysms.SMS_Header() + pysms.sms_fillHeader(sms_header, analysis_params, "pysms") + + sample_offset = 0 + size_new_data = 0 + current_frame = 0 + analysis_frames = [] + do_analysis = True + + while do_analysis and (current_frame < num_frames): + sample_offset += size_new_data + size_new_data = analysis_params.sizeNextRead + + frame_audio = audio[sample_offset:sample_offset + size_new_data] + frame_audio = np.array(frame_audio, dtype=np.float32) + if len(frame_audio) < size_new_data: + frame_audio = np.hstack(( + frame_audio, np.zeros(size_new_data - len(frame_audio), + dtype=np.float32) + )) + + analysis_data = pysms.SMS_Data() + pysms.sms_allocFrameH(sms_header, analysis_data) + status = pysms.sms_analyze(frame_audio, analysis_data, + analysis_params) + + analysis_frames.append(analysis_data) + current_frame += 1 + + if status == -1: + do_analysis = False + + sms_header.nFrames = len(analysis_frames) + synth_params = _pysms_synthesis_params(sampling_rate) + synth_params.iStochasticType = pysms.SMS_STOC_APPROX + synth_params.iSynthesisType = pysms.SMS_STYPE_STOC + pysms.sms_initSynth(sms_header, synth_params) + + synth_frame = np.zeros(synth_params.sizeHop, dtype=np.float32) + synth_audio = np.array([], dtype=np.float32) + + for i in range(len(analysis_frames)): + pysms.sms_synthesize(analysis_frames[i], synth_frame, synth_params) + synth_audio = np.hstack((synth_audio, synth_frame)) + + synth_audio = np.asarray(synth_audio * 32768, np.int16) + + for frame in analysis_frames: + pysms.sms_freeFrame(frame) + pysms.sms_freeAnalysis(analysis_params) + pysms.sms_closeSF() + pysms.sms_freeSynth(synth_params) + pysms.sms_free() + + return synth_audio + + if __name__ == '__main__': size_next_read = _size_next_read() partial_tracking = _partial_tracking() harmonic_synthesis_ifft = _harmonic_synthesis('ifft') harmonic_synthesis_sin = _harmonic_synthesis('sin') + residual_synthesis = _residual_synthesis() test_data = {'size_next_read': size_next_read, 'peak_detection': partial_tracking, @@ -269,3 +341,5 @@ if __name__ == '__main__': harmonic_synthesis_ifft) wav.write('libsms_harmonic_synthesis_sin.wav', sampling_rate, harmonic_synthesis_sin) + wav.write('libsms_residual_synthesis.wav', sampling_rate, + residual_synthesis) diff --git a/tests/test_residual.py b/tests/test_residual.py index 205ee59..e3992a8 100644 --- a/tests/test_residual.py +++ b/tests/test_residual.py @@ -7,12 +7,28 @@ import simpl.partial_tracking as partial_tracking import simpl.synthesis as synthesis import simpl.residual as residual -float_precision = 5 +float_precision = 2 frame_size = 512 hop_size = 512 +max_peaks = 10 +max_partials = 10 +num_frames = 30 +num_samples = num_frames * hop_size audio_path = os.path.join( os.path.dirname(__file__), 'audio/flute.wav' ) +libsms_residual_synthesis_path = os.path.join( + os.path.dirname(__file__), 'libsms_residual_synthesis.wav' +) + +PeakDetection = peak_detection.PeakDetection +SMSPeakDetection = peak_detection.SMSPeakDetection +PartialTracking = partial_tracking.PartialTracking +SMSPartialTracking = partial_tracking.SMSPartialTracking +Synthesis = synthesis.Synthesis +SMSSynthesis = synthesis.SMSSynthesis +Residual = residual.Residual +SMSResidual = residual.SMSResidual class TestResidual(object): @@ -20,16 +36,74 @@ class TestResidual(object): def setup_class(cls): cls.audio = simpl.read_wav(audio_path)[0] - def test_synthesis(self): - pd = peak_detection.PeakDetection() + def test_basic(self): + pd = PeakDetection() + frames = pd.find_peaks(self.audio) + + pt = PartialTracking() + frames = pt.find_partials(frames) + + synth = Synthesis() + synth_audio = synth.synth(frames) + + res = Residual() + residual_audio = res.find_residual(synth_audio, self.audio) + assert len(residual_audio) + + +class TestSMSResidual(object): + @classmethod + def setup_class(cls): + cls.audio = simpl.read_wav(audio_path)[0] + cls.audio = cls.audio[0:num_samples] + + def test_basic(self): + pd = SMSPeakDetection() + pd.hop_size = hop_size frames = pd.find_peaks(self.audio) - pt = partial_tracking.PartialTracking() + pt = SMSPartialTracking() + pt.max_partials = max_partials frames = pt.find_partials(frames) - s = synthesis.Synthesis() - synth_audio = s.synth(frames) + synth = SMSSynthesis() + synth.hop_size = hop_size + synth_audio = synth.synth(frames) - r = residual.Residual() - residual_audio = r.find_residual(synth_audio, self.audio) + res = SMSResidual() + residual_audio = res.find_residual(synth_audio, self.audio) assert len(residual_audio) + + def test_residual_synthesis(self): + pd = SMSPeakDetection() + pd.max_peaks = max_peaks + pd.hop_size = hop_size + frames = pd.find_peaks(self.audio) + + pt = SMSPartialTracking() + pt.max_partials = max_partials + frames = pt.find_partials(frames) + + synth = SMSSynthesis() + synth.hop_size = hop_size + synth.max_partials = max_partials + simpl_harmonic = synth.synth(frames) + + res = SMSResidual() + res.hop_size = hop_size + simpl_residual = res.synth(simpl_harmonic, self.audio) + + sms_residual, sampling_rate = simpl.read_wav( + libsms_residual_synthesis_path + ) + + assert len(simpl_residual) == len(sms_residual) + + # import matplotlib.pyplot as plt + # plt.plot(simpl_residual) + # plt.plot(sms_residual) + # plt.show() + + for i in range(len(simpl_residual)): + assert_almost_equals(simpl_residual[i], sms_residual[i], + float_precision) -- cgit v1.2.3