From beeff313be6db058c178b99ead6759bd837037a9 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 21 Mar 2023 10:36:28 -0700 Subject: [PATCH 1/2] Separate core NAM code from other plugin dsp code and remove IPlug dependency for NAM code. --- {dsp => NAM}/activations.h | 0 NAM/dsp.cpp | 498 ++++++++++++++++++++++++++++++++++++ NAM/dsp.h | 299 ++++++++++++++++++++++ {dsp => NAM}/get_dsp.cpp | 0 {dsp => NAM}/lstm.cpp | 0 {dsp => NAM}/lstm.h | 0 {dsp => NAM}/numpy_util.cpp | 0 {dsp => NAM}/numpy_util.h | 0 {dsp => NAM}/util.cpp | 0 {dsp => NAM}/util.h | 0 {dsp => NAM}/version.h | 0 {dsp => NAM}/wavenet.cpp | 0 {dsp => NAM}/wavenet.h | 0 dsp/dsp.cpp | 488 ----------------------------------- dsp/dsp.h | 294 --------------------- 15 files changed, 797 insertions(+), 782 deletions(-) rename {dsp => NAM}/activations.h (100%) create mode 100644 NAM/dsp.cpp create mode 100644 NAM/dsp.h rename {dsp => NAM}/get_dsp.cpp (100%) rename {dsp => NAM}/lstm.cpp (100%) rename {dsp => NAM}/lstm.h (100%) rename {dsp => NAM}/numpy_util.cpp (100%) rename {dsp => NAM}/numpy_util.h (100%) rename {dsp => NAM}/util.cpp (100%) rename {dsp => NAM}/util.h (100%) rename {dsp => NAM}/version.h (100%) rename {dsp => NAM}/wavenet.cpp (100%) rename {dsp => NAM}/wavenet.h (100%) diff --git a/dsp/activations.h b/NAM/activations.h similarity index 100% rename from dsp/activations.h rename to NAM/activations.h diff --git a/NAM/dsp.cpp b/NAM/dsp.cpp new file mode 100644 index 0000000..930ee91 --- /dev/null +++ b/NAM/dsp.cpp @@ -0,0 +1,498 @@ +#include // std::max_element +#include +#include // pow, tanh, expf +#include +#include +#include +#include +#include + +#include "dsp.h" +#include "json.hpp" +#include "numpy_util.h" +#include "util.h" + +#define tanh_impl_ std::tanh +// #define tanh_impl_ fast_tanh_ + +constexpr auto _INPUT_BUFFER_SAFETY_FACTOR = 32; + +DSP::DSP() { this->_stale_params = true; } + +void DSP::process(double **inputs, double **outputs, + const int num_channels, const int num_frames, + const double input_gain, const double output_gain, + const std::unordered_map ¶ms) { + this->_get_params_(params); + this->_apply_input_level_(inputs, num_channels, num_frames, input_gain); + this->_ensure_core_dsp_output_ready_(); + this->_process_core_(); + this->_apply_output_level_(outputs, num_channels, num_frames, output_gain); +} + +void DSP::finalize_(const int num_frames) {} + +void DSP::_get_params_( + const std::unordered_map &input_params) { + this->_stale_params = false; + for (auto it = input_params.begin(); it != input_params.end(); ++it) { + const std::string key = util::lowercase(it->first); + const double value = it->second; + if (this->_params.find(key) == this->_params.end()) // Not contained + this->_stale_params = true; + else if (this->_params[key] != value) // Contained but new value + this->_stale_params = true; + this->_params[key] = value; + } +} + +void DSP::_apply_input_level_(double **inputs, const int num_channels, + const int num_frames, const double gain) { + // Must match exactly; we're going to use the size of _input_post_gain later + // for num_frames. + if (this->_input_post_gain.size() != num_frames) + this->_input_post_gain.resize(num_frames); + // MONO ONLY + const int channel = 0; + for (int i = 0; i < num_frames; i++) + this->_input_post_gain[i] = float(gain * inputs[channel][i]); +} + +void DSP::_ensure_core_dsp_output_ready_() { + if (this->_core_dsp_output.size() < this->_input_post_gain.size()) + this->_core_dsp_output.resize(this->_input_post_gain.size()); +} + +void DSP::_process_core_() { + // Default implementation is the null operation + for (int i = 0; i < this->_input_post_gain.size(); i++) + this->_core_dsp_output[i] = this->_input_post_gain[i]; +} + +void DSP::_apply_output_level_(double **outputs, const int num_channels, + const int num_frames, const double gain) { + for (int c = 0; c < num_channels; c++) + for (int s = 0; s < num_frames; s++) + outputs[c][s] = double(gain * this->_core_dsp_output[s]); +} + +// Buffer ===================================================================== + +Buffer::Buffer(const int receptive_field) : DSP() { + this->_set_receptive_field(receptive_field); +} + +void Buffer::_set_receptive_field(const int new_receptive_field) { + this->_set_receptive_field(new_receptive_field, + _INPUT_BUFFER_SAFETY_FACTOR * new_receptive_field); +}; + +void Buffer::_set_receptive_field(const int new_receptive_field, + const int input_buffer_size) { + this->_receptive_field = new_receptive_field; + this->_input_buffer.resize(input_buffer_size); + this->_reset_input_buffer(); +} + +void Buffer::_update_buffers_() { + const long int num_frames = this->_input_post_gain.size(); + // Make sure that the buffer is big enough for the receptive field and the + // frames needed! + { + const long minimum_input_buffer_size = + (long)this->_receptive_field + _INPUT_BUFFER_SAFETY_FACTOR * num_frames; + if (this->_input_buffer.size() < minimum_input_buffer_size) { + long new_buffer_size = 2; + while (new_buffer_size < minimum_input_buffer_size) + new_buffer_size *= 2; + this->_input_buffer.resize(new_buffer_size); + } + } + + // If we'd run off the end of the input buffer, then we need to move the data + // back to the start of the buffer and start again. + if (this->_input_buffer_offset + num_frames > this->_input_buffer.size()) + this->_rewind_buffers_(); + // Put the new samples into the input buffer + for (long i = this->_input_buffer_offset, j = 0; j < num_frames; i++, j++) + this->_input_buffer[i] = this->_input_post_gain[j]; + // And resize the output buffer: + this->_output_buffer.resize(num_frames); +} + +void Buffer::_rewind_buffers_() { + // Copy the input buffer back + // RF-1 samples because we've got at least one new one inbound. + for (long i = 0, j = this->_input_buffer_offset - this->_receptive_field; + i < this->_receptive_field; i++, j++) + this->_input_buffer[i] = this->_input_buffer[j]; + // And reset the offset. + // Even though we could be stingy about that one sample that we won't be using + // (because a new set is incoming) it's probably not worth the + // hyper-optimization and liable for bugs. And the code looks way tidier this + // way. + this->_input_buffer_offset = this->_receptive_field; +} + +void Buffer::_reset_input_buffer() { + this->_input_buffer_offset = this->_receptive_field; +} + +void Buffer::finalize_(const int num_frames) { + this->DSP::finalize_(num_frames); + this->_input_buffer_offset += num_frames; +} + +// Linear ===================================================================== + +Linear::Linear(const int receptive_field, const bool _bias, + const std::vector ¶ms) + : Buffer(receptive_field) { + if (params.size() != (receptive_field + (_bias ? 1 : 0))) + throw std::runtime_error("Params vector does not match expected size based " + "on architecture parameters"); + + this->_weight.resize(this->_receptive_field); + // Pass in in reverse order so that dot products work out of the box. + for (int i = 0; i < this->_receptive_field; i++) + this->_weight(i) = params[receptive_field - 1 - i]; + this->_bias = _bias ? params[receptive_field] : (float)0.0; +} + +void Linear::_process_core_() { + this->Buffer::_update_buffers_(); + + // Main computation! + for (long i = 0; i < this->_input_post_gain.size(); i++) { + const long offset = + this->_input_buffer_offset - this->_weight.size() + i + 1; + auto input = Eigen::Map(&this->_input_buffer[offset], + this->_receptive_field); + this->_core_dsp_output[i] = this->_bias + this->_weight.dot(input); + } +} + +// NN modules ================================================================= + +void relu_(Eigen::MatrixXf &x, const long i_start, const long i_end, + const long j_start, const long j_end) { + for (long j = j_start; j < j_end; j++) + for (long i = 0; i < x.rows(); i++) + x(i, j) = x(i, j) < (float)0.0 ? (float)0.0 : x(i, j); +} + +void relu_(Eigen::MatrixXf &x, const long j_start, const long j_end) { + relu_(x, 0, x.rows(), j_start, j_end); +} + +void relu_(Eigen::MatrixXf &x) { relu_(x, 0, x.rows(), 0, x.cols()); } + +void sigmoid_(Eigen::MatrixXf &x, const long i_start, const long i_end, + const long j_start, const long j_end) { + for (long j = j_start; j < j_end; j++) + for (long i = i_start; i < i_end; i++) + x(i, j) = 1.0 / (1.0 + expf(-x(i, j))); +} + +inline float fast_tanh_(const float x) { + const float ax = fabs(x); + const float x2 = x * x; + + return (x * + (2.45550750702956f + 2.45550750702956f * ax + + (0.893229853513558f + 0.821226666969744f * ax) * x2) / + (2.44506634652299f + + (2.44506634652299f + x2) * fabs(x + 0.814642734961073f * x * ax))); +} + +void tanh_(Eigen::MatrixXf &x, const long i_start, const long i_end, + const long j_start, const long j_end) { + for (long j = j_start; j < j_end; j++) + for (long i = i_start; i < i_end; i++) + x(i, j) = tanh_impl_(x(i, j)); +} + +void tanh_(Eigen::MatrixXf &x, const long j_start, const long j_end) { + tanh_(x, 0, x.rows(), j_start, j_end); +} + +void tanh_(Eigen::MatrixXf &x) { + + float *ptr = x.data(); + + long size = x.rows() * x.cols(); + + for (long pos = 0; pos < size; pos++) { + ptr[pos] = tanh_impl_(ptr[pos]); + } +} + +void Conv1D::set_params_(std::vector::iterator ¶ms) { + if (this->_weight.size() > 0) { + const long out_channels = this->_weight[0].rows(); + const long in_channels = this->_weight[0].cols(); + // Crazy ordering because that's how it gets flattened. + for (auto i = 0; i < out_channels; i++) + for (auto j = 0; j < in_channels; j++) + for (auto k = 0; k < this->_weight.size(); k++) + this->_weight[k](i, j) = *(params++); + } + for (int i = 0; i < this->_bias.size(); i++) + this->_bias(i) = *(params++); +} + +void Conv1D::set_size_(const int in_channels, const int out_channels, + const int kernel_size, const bool do_bias, + const int _dilation) { + this->_weight.resize(kernel_size); + for (int i = 0; i < this->_weight.size(); i++) + this->_weight[i].resize(out_channels, + in_channels); // y = Ax, input array (C,L) + if (do_bias) + this->_bias.resize(out_channels); + else + this->_bias.resize(0); + this->_dilation = _dilation; +} + +void Conv1D::set_size_and_params_(const int in_channels, const int out_channels, + const int kernel_size, const int _dilation, + const bool do_bias, + std::vector::iterator ¶ms) { + this->set_size_(in_channels, out_channels, kernel_size, do_bias, _dilation); + this->set_params_(params); +} + +void Conv1D::process_(const Eigen::MatrixXf &input, Eigen::MatrixXf &output, + const long i_start, const long ncols, + const long j_start) const { + // This is the clever part ;) + for (long k = 0; k < this->_weight.size(); k++) { + const long offset = this->_dilation * (k + 1 - this->_weight.size()); + if (k == 0) + output.middleCols(j_start, ncols) = + this->_weight[k] * input.middleCols(i_start + offset, ncols); + else + output.middleCols(j_start, ncols) += + this->_weight[k] * input.middleCols(i_start + offset, ncols); + } + if (this->_bias.size() > 0) + output.middleCols(j_start, ncols).colwise() += this->_bias; +} + +long Conv1D::get_num_params() const { + long num_params = this->_bias.size(); + for (long i = 0; i < this->_weight.size(); i++) + num_params += this->_weight[i].size(); + return num_params; +} + +Conv1x1::Conv1x1(const int in_channels, const int out_channels, + const bool _bias) { + this->_weight.resize(out_channels, in_channels); + this->_do_bias = _bias; + if (_bias) + this->_bias.resize(out_channels); +} + +void Conv1x1::set_params_(std::vector::iterator ¶ms) { + for (int i = 0; i < this->_weight.rows(); i++) + for (int j = 0; j < this->_weight.cols(); j++) + this->_weight(i, j) = *(params++); + if (this->_do_bias) + for (int i = 0; i < this->_bias.size(); i++) + this->_bias(i) = *(params++); +} + +Eigen::MatrixXf Conv1x1::process(const Eigen::MatrixXf &input) const { + if (this->_do_bias) + return (this->_weight * input).colwise() + this->_bias; + else + return this->_weight * input; +} + +// ConvNet ==================================================================== + +convnet::BatchNorm::BatchNorm(const int dim, + std::vector::iterator ¶ms) { + // Extract from param buffer + Eigen::VectorXf running_mean(dim); + Eigen::VectorXf running_var(dim); + Eigen::VectorXf _weight(dim); + Eigen::VectorXf _bias(dim); + for (int i = 0; i < dim; i++) + running_mean(i) = *(params++); + for (int i = 0; i < dim; i++) + running_var(i) = *(params++); + for (int i = 0; i < dim; i++) + _weight(i) = *(params++); + for (int i = 0; i < dim; i++) + _bias(i) = *(params++); + float eps = *(params++); + + // Convert to scale & loc + this->scale.resize(dim); + this->loc.resize(dim); + for (int i = 0; i < dim; i++) + this->scale(i) = _weight(i) / sqrt(eps + running_var(i)); + this->loc = _bias - this->scale.cwiseProduct(running_mean); +} + +void convnet::BatchNorm::process_(Eigen::MatrixXf &x, const long i_start, + const long i_end) const { + // todo using colwise? + // #speed but conv probably dominates + for (auto i = i_start; i < i_end; i++) { + x.col(i) = x.col(i).cwiseProduct(this->scale); + x.col(i) += this->loc; + } +} + +void convnet::ConvNetBlock::set_params_(const int in_channels, + const int out_channels, + const int _dilation, + const bool batchnorm, + const std::string activation, + std::vector::iterator ¶ms) { + this->_batchnorm = batchnorm; + // HACK 2 kernel + this->conv.set_size_and_params_(in_channels, out_channels, 2, _dilation, + !batchnorm, params); + if (this->_batchnorm) + this->batchnorm = BatchNorm(out_channels, params); + this->activation = activation; +} + +void convnet::ConvNetBlock::process_(const Eigen::MatrixXf &input, + Eigen::MatrixXf &output, + const long i_start, + const long i_end) const { + const long ncols = i_end - i_start; + this->conv.process_(input, output, i_start, ncols, i_start); + if (this->_batchnorm) + this->batchnorm.process_(output, i_start, i_end); + if (this->activation == "Tanh") + tanh_(output, i_start, i_end); + else if (this->activation == "ReLU") + relu_(output, i_start, i_end); + else + throw std::runtime_error("Unrecognized activation"); +} + +long convnet::ConvNetBlock::get_out_channels() const { + return this->conv.get_out_channels(); +} + +convnet::_Head::_Head(const int channels, + std::vector::iterator ¶ms) { + this->_weight.resize(channels); + for (int i = 0; i < channels; i++) + this->_weight[i] = *(params++); + this->_bias = *(params++); +} + +void convnet::_Head::process_(const Eigen::MatrixXf &input, + Eigen::VectorXf &output, const long i_start, + const long i_end) const { + const long length = i_end - i_start; + output.resize(length); + for (long i = 0, j = i_start; i < length; i++, j++) + output(i) = this->_bias + input.col(j).dot(this->_weight); +} + +convnet::ConvNet::ConvNet(const int channels, const std::vector &dilations, + const bool batchnorm, const std::string activation, + std::vector ¶ms) + : Buffer(*std::max_element(dilations.begin(), dilations.end())) { + this->_verify_params(channels, dilations, batchnorm, params.size()); + this->_blocks.resize(dilations.size()); + std::vector::iterator it = params.begin(); + for (int i = 0; i < dilations.size(); i++) + this->_blocks[i].set_params_(i == 0 ? 1 : channels, channels, dilations[i], + batchnorm, activation, it); + this->_block_vals.resize(this->_blocks.size() + 1); + this->_head = _Head(channels, it); + if (it != params.end()) + throw std::runtime_error( + "Didn't touch all the params when initializing wavenet"); + this->_reset_anti_pop_(); +} + +void convnet::ConvNet::_process_core_() { + this->_update_buffers_(); + // Main computation! + const long i_start = this->_input_buffer_offset; + const long num_frames = this->_input_post_gain.size(); + const long i_end = i_start + num_frames; + // TODO one unnecessary copy :/ #speed + for (auto i = i_start; i < i_end; i++) + this->_block_vals[0](0, i) = this->_input_buffer[i]; + for (auto i = 0; i < this->_blocks.size(); i++) + this->_blocks[i].process_(this->_block_vals[i], this->_block_vals[i + 1], + i_start, i_end); + // TODO clean up this allocation + this->_head.process_(this->_block_vals[this->_blocks.size()], + this->_head_output, i_start, i_end); + // Copy to required output array (TODO tighten this up) + for (int s = 0; s < num_frames; s++) + this->_core_dsp_output[s] = this->_head_output(s); + // Apply anti-pop + this->_anti_pop_(); +} + +void convnet::ConvNet::_verify_params(const int channels, + const std::vector &dilations, + const bool batchnorm, + const size_t actual_params) { + // TODO +} + +void convnet::ConvNet::_update_buffers_() { + this->Buffer::_update_buffers_(); + const long buffer_size = this->_input_buffer.size(); + this->_block_vals[0].resize(1, buffer_size); + for (long i = 1; i < this->_block_vals.size(); i++) + this->_block_vals[i].resize(this->_blocks[i - 1].get_out_channels(), + buffer_size); +} + +void convnet::ConvNet::_rewind_buffers_() { + // Need to rewind the block vals first because Buffer::rewind_buffers() + // resets the offset index + // The last _block_vals is the output of the last block and doesn't need to be + // rewound. + for (long k = 0; k < this->_block_vals.size() - 1; k++) { + // We actually don't need to pull back a lot...just as far as the first + // input sample would grab from dilation + const long _dilation = this->_blocks[k].conv.get_dilation(); + for (long i = this->_receptive_field - _dilation, + j = this->_input_buffer_offset - _dilation; + j < this->_input_buffer_offset; i++, j++) + for (long r = 0; r < this->_block_vals[k].rows(); r++) + this->_block_vals[k](r, i) = this->_block_vals[k](r, j); + } + // Now we can do the rest of the rewind + this->Buffer::_rewind_buffers_(); +} + +void convnet::ConvNet::_anti_pop_() { + if (this->_anti_pop_countdown >= this->_anti_pop_ramp) + return; + const float slope = 1.0f / float(this->_anti_pop_ramp); + for (int i = 0; i < this->_core_dsp_output.size(); i++) { + if (this->_anti_pop_countdown >= this->_anti_pop_ramp) + break; + const float gain = + std::max(slope * float(this->_anti_pop_countdown), float(0.0)); + this->_core_dsp_output[i] *= gain; + this->_anti_pop_countdown++; + } +} + +void convnet::ConvNet::_reset_anti_pop_() { + // You need the "real" receptive field, not the buffers. + long receptive_field = 1; + for (int i = 0; i < this->_blocks.size(); i++) + receptive_field += this->_blocks[i].conv.get_dilation(); + this->_anti_pop_countdown = -receptive_field; +} diff --git a/NAM/dsp.h b/NAM/dsp.h new file mode 100644 index 0000000..2eecabe --- /dev/null +++ b/NAM/dsp.h @@ -0,0 +1,299 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +enum EArchitectures { + kLinear = 0, + kConvNet, + kLSTM, + kCatLSTM, + kWaveNet, + kCatWaveNet, + kNumModels +}; + +// Class for providing params from the plugin to the DSP module +// For now, we'll work with doubles. Later, we'll add other types. +class DSPParam { +public: + const char *name; + const double val; +}; +// And the params shall be provided as a std::vector. + +class DSP { +public: + DSP(); + // process() does all of the processing requried to take `inputs` array and + // fill in the required values on `outputs`. + // To do this: + // 1. The parameters from the plugin (I/O levels and any other parametric + // inputs) are gotten. + // 2. The input level is applied + // 3. The core DSP algorithm is run (This is what should probably be + // overridden in subclasses). + // 4. The output level is applied and the result stored to `output`. + virtual void process(double **inputs, double **outputs, + const int num_channels, const int num_frames, + const double input_gain, const double output_gain, + const std::unordered_map ¶ms); + // Anything to take care of before next buffer comes in. + // For example: + // * Move the buffer index forward + // * Does NOT say that params aren't stale; that's the job of the routine + // that actually uses them, which varies depends on the particulars of the + // DSP subclass implementation. + virtual void finalize_(const int num_frames); + +protected: + // Parameters (aka "knobs") + std::unordered_map _params; + // If the params have changed since the last buffer was processed: + bool _stale_params; + // Where to store the samples after applying input gain + std::vector _input_post_gain; + // Location for the output of the core DSP algorithm. + std::vector _core_dsp_output; + + // Methods + + // Copy the parameters to the DSP module. + // If anything has changed, then set this->_stale_params to true. + // (TODO use "listener" approach) + void + _get_params_(const std::unordered_map &input_params); + + // Apply the input gain + // Result populates this->_input_post_gain + void _apply_input_level_(double **inputs, const int num_channels, + const int num_frames, const double gain); + + // i.e. ensure the size is correct. + void _ensure_core_dsp_output_ready_(); + + // The core of your DSP algorithm. + // Access the inputs in this->_input_post_gain + // Place the outputs in this->_core_dsp_output + virtual void _process_core_(); + + // Copy this->_core_dsp_output to output and apply the output volume + void _apply_output_level_(double **outputs, const int num_channels, + const int num_frames, const double gain); +}; + +// Class where an input buffer is kept so that long-time effects can be +// captured. (e.g. conv nets or impulse responses, where we need history that's +// longer than the sample buffer that's coming in.) +class Buffer : public DSP { +public: + Buffer(const int receptive_field); + void finalize_(const int num_frames); + +protected: + // Input buffer + const int _input_buffer_channels = 1; // Mono + int _receptive_field; + // First location where we add new samples from the input + long _input_buffer_offset; + std::vector _input_buffer; + std::vector _output_buffer; + + void _set_receptive_field(const int new_receptive_field, + const int input_buffer_size); + void _set_receptive_field(const int new_receptive_field); + void _reset_input_buffer(); + // Use this->_input_post_gain + virtual void _update_buffers_(); + virtual void _rewind_buffers_(); +}; + +// Basic linear model (an IR!) +class Linear : public Buffer { +public: + Linear(const int receptive_field, const bool _bias, + const std::vector ¶ms); + void _process_core_() override; + +protected: + Eigen::VectorXf _weight; + float _bias; +}; + +// NN modules ================================================================= + +// Activations + +// In-place ReLU on (N,M) array +void relu_(Eigen::MatrixXf &x, const long i_start, const long i_end, + const long j_start, const long j_end); +// Subset of the columns +void relu_(Eigen::MatrixXf &x, const long j_start, const long j_end); +void relu_(Eigen::MatrixXf &x); + +// In-place sigmoid +void sigmoid_(Eigen::MatrixXf &x, const long i_start, const long i_end, + const long j_start, const long j_end); +void sigmoid_(Eigen::MatrixXf &x); + +// In-place Tanh on (N,M) array +void tanh_(Eigen::MatrixXf &x, const long i_start, const long i_end, + const long j_start, const long j_end); +// Subset of the columns +void tanh_(Eigen::MatrixXf &x, const long i_start, const long i_end); + +void tanh_(Eigen::MatrixXf &x); + +class Conv1D { +public: + Conv1D() { this->_dilation = 1; }; + void set_params_(std::vector::iterator ¶ms); + void set_size_(const int in_channels, const int out_channels, + const int kernel_size, const bool do_bias, + const int _dilation); + void set_size_and_params_(const int in_channels, const int out_channels, + const int kernel_size, const int _dilation, + const bool do_bias, + std::vector::iterator ¶ms); + // Process from input to output + // Rightmost indices of input go from i_start to i_end, + // Indices on output for from j_start (to j_start + i_end - i_start) + void process_(const Eigen::MatrixXf &input, Eigen::MatrixXf &output, + const long i_start, const long i_end, const long j_start) const; + long get_in_channels() const { + return this->_weight.size() > 0 ? this->_weight[0].cols() : 0; + }; + long get_kernel_size() const { return this->_weight.size(); }; + long get_num_params() const; + long get_out_channels() const { + return this->_weight.size() > 0 ? this->_weight[0].rows() : 0; + }; + int get_dilation() const { return this->_dilation; }; + +private: + // Gonna wing this... + // conv[kernel](cout, cin) + std::vector _weight; + Eigen::VectorXf _bias; + int _dilation; +}; + +// Really just a linear layer +class Conv1x1 { +public: + Conv1x1(const int in_channels, const int out_channels, const bool _bias); + void set_params_(std::vector::iterator ¶ms); + // :param input: (N,Cin) or (Cin,) + // :return: (N,Cout) or (Cout,), respectively + Eigen::MatrixXf process(const Eigen::MatrixXf &input) const; + + long get_out_channels() const { return this->_weight.rows(); }; + +private: + Eigen::MatrixXf _weight; + Eigen::VectorXf _bias; + bool _do_bias; +}; + +// ConvNet ==================================================================== + +namespace convnet { +// Custom Conv that avoids re-computing on pieces of the input and trusts +// that the corresponding outputs are where they need to be. +// Beware: this is clever! + +// Batch normalization +// In prod mode, so really just an elementwise affine layer. +class BatchNorm { +public: + BatchNorm(){}; + BatchNorm(const int dim, std::vector::iterator ¶ms); + void process_(Eigen::MatrixXf &input, const long i_start, + const long i_end) const; + +private: + // TODO simplify to just ax+b + // y = (x-m)/sqrt(v+eps) * w + bias + // y = ax+b + // a = w / sqrt(v+eps) + // b = a * m + bias + Eigen::VectorXf scale; + Eigen::VectorXf loc; +}; + +class ConvNetBlock { +public: + ConvNetBlock() { this->_batchnorm = false; }; + void set_params_(const int in_channels, const int out_channels, + const int _dilation, const bool batchnorm, + const std::string activation, + std::vector::iterator ¶ms); + void process_(const Eigen::MatrixXf &input, Eigen::MatrixXf &output, + const long i_start, const long i_end) const; + long get_out_channels() const; + Conv1D conv; + +private: + BatchNorm batchnorm; + bool _batchnorm; + std::string activation; +}; + +class _Head { +public: + _Head() { this->_bias = (float)0.0; }; + _Head(const int channels, std::vector::iterator ¶ms); + void process_(const Eigen::MatrixXf &input, Eigen::VectorXf &output, + const long i_start, const long i_end) const; + +private: + Eigen::VectorXf _weight; + float _bias; +}; + +class ConvNet : public Buffer { +public: + ConvNet(const int channels, const std::vector &dilations, + const bool batchnorm, const std::string activation, + std::vector ¶ms); + +protected: + std::vector _blocks; + std::vector _block_vals; + Eigen::VectorXf _head_output; + _Head _head; + void _verify_params(const int channels, const std::vector &dilations, + const bool batchnorm, const size_t actual_params); + void _update_buffers_() override; + void _rewind_buffers_() override; + + void _process_core_() override; + + // The net starts with random parameters inside; we need to wait for a full + // receptive field to pass through before we can count on the output being + // ok. This implements a gentle "ramp-up" so that there's no "pop" at the + // start. + long _anti_pop_countdown; + const long _anti_pop_ramp = 100; + void _anti_pop_(); + void _reset_anti_pop_(); +}; +}; // namespace convnet + +// Utilities ================================================================== +// Implemented in get_dsp.cpp + +// Verify that the config that we are building our model from is supported by +// this plugin version. +void verify_config_version(const std::string version); + +// Takes the model file and uses it to instantiate an instance of DSP. +std::unique_ptr get_dsp(const std::filesystem::path model_file); +// Legacy loader for directory-type DSPs +std::unique_ptr get_dsp_legacy(const std::filesystem::path dirname); diff --git a/dsp/get_dsp.cpp b/NAM/get_dsp.cpp similarity index 100% rename from dsp/get_dsp.cpp rename to NAM/get_dsp.cpp diff --git a/dsp/lstm.cpp b/NAM/lstm.cpp similarity index 100% rename from dsp/lstm.cpp rename to NAM/lstm.cpp diff --git a/dsp/lstm.h b/NAM/lstm.h similarity index 100% rename from dsp/lstm.h rename to NAM/lstm.h diff --git a/dsp/numpy_util.cpp b/NAM/numpy_util.cpp similarity index 100% rename from dsp/numpy_util.cpp rename to NAM/numpy_util.cpp diff --git a/dsp/numpy_util.h b/NAM/numpy_util.h similarity index 100% rename from dsp/numpy_util.h rename to NAM/numpy_util.h diff --git a/dsp/util.cpp b/NAM/util.cpp similarity index 100% rename from dsp/util.cpp rename to NAM/util.cpp diff --git a/dsp/util.h b/NAM/util.h similarity index 100% rename from dsp/util.h rename to NAM/util.h diff --git a/dsp/version.h b/NAM/version.h similarity index 100% rename from dsp/version.h rename to NAM/version.h diff --git a/dsp/wavenet.cpp b/NAM/wavenet.cpp similarity index 100% rename from dsp/wavenet.cpp rename to NAM/wavenet.cpp diff --git a/dsp/wavenet.h b/NAM/wavenet.h similarity index 100% rename from dsp/wavenet.h rename to NAM/wavenet.h diff --git a/dsp/dsp.cpp b/dsp/dsp.cpp index c83736c..ac0f831 100644 --- a/dsp/dsp.cpp +++ b/dsp/dsp.cpp @@ -8,494 +8,6 @@ #include #include "dsp.h" -#include "json.hpp" -#include "numpy_util.h" -#include "util.h" - -#define tanh_impl_ std::tanh -// #define tanh_impl_ fast_tanh_ - -constexpr auto _INPUT_BUFFER_SAFETY_FACTOR = 32; - -DSP::DSP() { this->_stale_params = true; } - -void DSP::process(iplug::sample **inputs, iplug::sample **outputs, - const int num_channels, const int num_frames, - const double input_gain, const double output_gain, - const std::unordered_map ¶ms) { - this->_get_params_(params); - this->_apply_input_level_(inputs, num_channels, num_frames, input_gain); - this->_ensure_core_dsp_output_ready_(); - this->_process_core_(); - this->_apply_output_level_(outputs, num_channels, num_frames, output_gain); -} - -void DSP::finalize_(const int num_frames) {} - -void DSP::_get_params_( - const std::unordered_map &input_params) { - this->_stale_params = false; - for (auto it = input_params.begin(); it != input_params.end(); ++it) { - const std::string key = util::lowercase(it->first); - const double value = it->second; - if (this->_params.find(key) == this->_params.end()) // Not contained - this->_stale_params = true; - else if (this->_params[key] != value) // Contained but new value - this->_stale_params = true; - this->_params[key] = value; - } -} - -void DSP::_apply_input_level_(iplug::sample **inputs, const int num_channels, - const int num_frames, const double gain) { - // Must match exactly; we're going to use the size of _input_post_gain later - // for num_frames. - if (this->_input_post_gain.size() != num_frames) - this->_input_post_gain.resize(num_frames); - // MONO ONLY - const int channel = 0; - for (int i = 0; i < num_frames; i++) - this->_input_post_gain[i] = float(gain * inputs[channel][i]); -} - -void DSP::_ensure_core_dsp_output_ready_() { - if (this->_core_dsp_output.size() < this->_input_post_gain.size()) - this->_core_dsp_output.resize(this->_input_post_gain.size()); -} - -void DSP::_process_core_() { - // Default implementation is the null operation - for (int i = 0; i < this->_input_post_gain.size(); i++) - this->_core_dsp_output[i] = this->_input_post_gain[i]; -} - -void DSP::_apply_output_level_(iplug::sample **outputs, const int num_channels, - const int num_frames, const double gain) { - for (int c = 0; c < num_channels; c++) - for (int s = 0; s < num_frames; s++) - outputs[c][s] = double(gain * this->_core_dsp_output[s]); -} - -// Buffer ===================================================================== - -Buffer::Buffer(const int receptive_field) : DSP() { - this->_set_receptive_field(receptive_field); -} - -void Buffer::_set_receptive_field(const int new_receptive_field) { - this->_set_receptive_field(new_receptive_field, - _INPUT_BUFFER_SAFETY_FACTOR * new_receptive_field); -}; - -void Buffer::_set_receptive_field(const int new_receptive_field, - const int input_buffer_size) { - this->_receptive_field = new_receptive_field; - this->_input_buffer.resize(input_buffer_size); - this->_reset_input_buffer(); -} - -void Buffer::_update_buffers_() { - const long int num_frames = this->_input_post_gain.size(); - // Make sure that the buffer is big enough for the receptive field and the - // frames needed! - { - const long minimum_input_buffer_size = - (long)this->_receptive_field + _INPUT_BUFFER_SAFETY_FACTOR * num_frames; - if (this->_input_buffer.size() < minimum_input_buffer_size) { - long new_buffer_size = 2; - while (new_buffer_size < minimum_input_buffer_size) - new_buffer_size *= 2; - this->_input_buffer.resize(new_buffer_size); - } - } - - // If we'd run off the end of the input buffer, then we need to move the data - // back to the start of the buffer and start again. - if (this->_input_buffer_offset + num_frames > this->_input_buffer.size()) - this->_rewind_buffers_(); - // Put the new samples into the input buffer - for (long i = this->_input_buffer_offset, j = 0; j < num_frames; i++, j++) - this->_input_buffer[i] = this->_input_post_gain[j]; - // And resize the output buffer: - this->_output_buffer.resize(num_frames); -} - -void Buffer::_rewind_buffers_() { - // Copy the input buffer back - // RF-1 samples because we've got at least one new one inbound. - for (long i = 0, j = this->_input_buffer_offset - this->_receptive_field; - i < this->_receptive_field; i++, j++) - this->_input_buffer[i] = this->_input_buffer[j]; - // And reset the offset. - // Even though we could be stingy about that one sample that we won't be using - // (because a new set is incoming) it's probably not worth the - // hyper-optimization and liable for bugs. And the code looks way tidier this - // way. - this->_input_buffer_offset = this->_receptive_field; -} - -void Buffer::_reset_input_buffer() { - this->_input_buffer_offset = this->_receptive_field; -} - -void Buffer::finalize_(const int num_frames) { - this->DSP::finalize_(num_frames); - this->_input_buffer_offset += num_frames; -} - -// Linear ===================================================================== - -Linear::Linear(const int receptive_field, const bool _bias, - const std::vector ¶ms) - : Buffer(receptive_field) { - if (params.size() != (receptive_field + (_bias ? 1 : 0))) - throw std::runtime_error("Params vector does not match expected size based " - "on architecture parameters"); - - this->_weight.resize(this->_receptive_field); - // Pass in in reverse order so that dot products work out of the box. - for (int i = 0; i < this->_receptive_field; i++) - this->_weight(i) = params[receptive_field - 1 - i]; - this->_bias = _bias ? params[receptive_field] : (float)0.0; -} - -void Linear::_process_core_() { - this->Buffer::_update_buffers_(); - - // Main computation! - for (long i = 0; i < this->_input_post_gain.size(); i++) { - const long offset = - this->_input_buffer_offset - this->_weight.size() + i + 1; - auto input = Eigen::Map(&this->_input_buffer[offset], - this->_receptive_field); - this->_core_dsp_output[i] = this->_bias + this->_weight.dot(input); - } -} - -// NN modules ================================================================= - -void relu_(Eigen::MatrixXf &x, const long i_start, const long i_end, - const long j_start, const long j_end) { - for (long j = j_start; j < j_end; j++) - for (long i = 0; i < x.rows(); i++) - x(i, j) = x(i, j) < (float)0.0 ? (float)0.0 : x(i, j); -} - -void relu_(Eigen::MatrixXf &x, const long j_start, const long j_end) { - relu_(x, 0, x.rows(), j_start, j_end); -} - -void relu_(Eigen::MatrixXf &x) { relu_(x, 0, x.rows(), 0, x.cols()); } - -void sigmoid_(Eigen::MatrixXf &x, const long i_start, const long i_end, - const long j_start, const long j_end) { - for (long j = j_start; j < j_end; j++) - for (long i = i_start; i < i_end; i++) - x(i, j) = 1.0 / (1.0 + expf(-x(i, j))); -} - -inline float fast_tanh_(const float x) { - const float ax = fabs(x); - const float x2 = x * x; - - return (x * - (2.45550750702956f + 2.45550750702956f * ax + - (0.893229853513558f + 0.821226666969744f * ax) * x2) / - (2.44506634652299f + - (2.44506634652299f + x2) * fabs(x + 0.814642734961073f * x * ax))); -} - -void tanh_(Eigen::MatrixXf &x, const long i_start, const long i_end, - const long j_start, const long j_end) { - for (long j = j_start; j < j_end; j++) - for (long i = i_start; i < i_end; i++) - x(i, j) = tanh_impl_(x(i, j)); -} - -void tanh_(Eigen::MatrixXf &x, const long j_start, const long j_end) { - tanh_(x, 0, x.rows(), j_start, j_end); -} - -void tanh_(Eigen::MatrixXf &x) { - - float *ptr = x.data(); - - long size = x.rows() * x.cols(); - - for (long pos = 0; pos < size; pos++) { - ptr[pos] = tanh_impl_(ptr[pos]); - } -} - -void Conv1D::set_params_(std::vector::iterator ¶ms) { - if (this->_weight.size() > 0) { - const long out_channels = this->_weight[0].rows(); - const long in_channels = this->_weight[0].cols(); - // Crazy ordering because that's how it gets flattened. - for (auto i = 0; i < out_channels; i++) - for (auto j = 0; j < in_channels; j++) - for (auto k = 0; k < this->_weight.size(); k++) - this->_weight[k](i, j) = *(params++); - } - for (int i = 0; i < this->_bias.size(); i++) - this->_bias(i) = *(params++); -} - -void Conv1D::set_size_(const int in_channels, const int out_channels, - const int kernel_size, const bool do_bias, - const int _dilation) { - this->_weight.resize(kernel_size); - for (int i = 0; i < this->_weight.size(); i++) - this->_weight[i].resize(out_channels, - in_channels); // y = Ax, input array (C,L) - if (do_bias) - this->_bias.resize(out_channels); - else - this->_bias.resize(0); - this->_dilation = _dilation; -} - -void Conv1D::set_size_and_params_(const int in_channels, const int out_channels, - const int kernel_size, const int _dilation, - const bool do_bias, - std::vector::iterator ¶ms) { - this->set_size_(in_channels, out_channels, kernel_size, do_bias, _dilation); - this->set_params_(params); -} - -void Conv1D::process_(const Eigen::MatrixXf &input, Eigen::MatrixXf &output, - const long i_start, const long ncols, - const long j_start) const { - // This is the clever part ;) - for (long k = 0; k < this->_weight.size(); k++) { - const long offset = this->_dilation * (k + 1 - this->_weight.size()); - if (k == 0) - output.middleCols(j_start, ncols) = - this->_weight[k] * input.middleCols(i_start + offset, ncols); - else - output.middleCols(j_start, ncols) += - this->_weight[k] * input.middleCols(i_start + offset, ncols); - } - if (this->_bias.size() > 0) - output.middleCols(j_start, ncols).colwise() += this->_bias; -} - -long Conv1D::get_num_params() const { - long num_params = this->_bias.size(); - for (long i = 0; i < this->_weight.size(); i++) - num_params += this->_weight[i].size(); - return num_params; -} - -Conv1x1::Conv1x1(const int in_channels, const int out_channels, - const bool _bias) { - this->_weight.resize(out_channels, in_channels); - this->_do_bias = _bias; - if (_bias) - this->_bias.resize(out_channels); -} - -void Conv1x1::set_params_(std::vector::iterator ¶ms) { - for (int i = 0; i < this->_weight.rows(); i++) - for (int j = 0; j < this->_weight.cols(); j++) - this->_weight(i, j) = *(params++); - if (this->_do_bias) - for (int i = 0; i < this->_bias.size(); i++) - this->_bias(i) = *(params++); -} - -Eigen::MatrixXf Conv1x1::process(const Eigen::MatrixXf &input) const { - if (this->_do_bias) - return (this->_weight * input).colwise() + this->_bias; - else - return this->_weight * input; -} - -// ConvNet ==================================================================== - -convnet::BatchNorm::BatchNorm(const int dim, - std::vector::iterator ¶ms) { - // Extract from param buffer - Eigen::VectorXf running_mean(dim); - Eigen::VectorXf running_var(dim); - Eigen::VectorXf _weight(dim); - Eigen::VectorXf _bias(dim); - for (int i = 0; i < dim; i++) - running_mean(i) = *(params++); - for (int i = 0; i < dim; i++) - running_var(i) = *(params++); - for (int i = 0; i < dim; i++) - _weight(i) = *(params++); - for (int i = 0; i < dim; i++) - _bias(i) = *(params++); - float eps = *(params++); - - // Convert to scale & loc - this->scale.resize(dim); - this->loc.resize(dim); - for (int i = 0; i < dim; i++) - this->scale(i) = _weight(i) / sqrt(eps + running_var(i)); - this->loc = _bias - this->scale.cwiseProduct(running_mean); -} - -void convnet::BatchNorm::process_(Eigen::MatrixXf &x, const long i_start, - const long i_end) const { - // todo using colwise? - // #speed but conv probably dominates - for (auto i = i_start; i < i_end; i++) { - x.col(i) = x.col(i).cwiseProduct(this->scale); - x.col(i) += this->loc; - } -} - -void convnet::ConvNetBlock::set_params_(const int in_channels, - const int out_channels, - const int _dilation, - const bool batchnorm, - const std::string activation, - std::vector::iterator ¶ms) { - this->_batchnorm = batchnorm; - // HACK 2 kernel - this->conv.set_size_and_params_(in_channels, out_channels, 2, _dilation, - !batchnorm, params); - if (this->_batchnorm) - this->batchnorm = BatchNorm(out_channels, params); - this->activation = activation; -} - -void convnet::ConvNetBlock::process_(const Eigen::MatrixXf &input, - Eigen::MatrixXf &output, - const long i_start, - const long i_end) const { - const long ncols = i_end - i_start; - this->conv.process_(input, output, i_start, ncols, i_start); - if (this->_batchnorm) - this->batchnorm.process_(output, i_start, i_end); - if (this->activation == "Tanh") - tanh_(output, i_start, i_end); - else if (this->activation == "ReLU") - relu_(output, i_start, i_end); - else - throw std::runtime_error("Unrecognized activation"); -} - -long convnet::ConvNetBlock::get_out_channels() const { - return this->conv.get_out_channels(); -} - -convnet::_Head::_Head(const int channels, - std::vector::iterator ¶ms) { - this->_weight.resize(channels); - for (int i = 0; i < channels; i++) - this->_weight[i] = *(params++); - this->_bias = *(params++); -} - -void convnet::_Head::process_(const Eigen::MatrixXf &input, - Eigen::VectorXf &output, const long i_start, - const long i_end) const { - const long length = i_end - i_start; - output.resize(length); - for (long i = 0, j = i_start; i < length; i++, j++) - output(i) = this->_bias + input.col(j).dot(this->_weight); -} - -convnet::ConvNet::ConvNet(const int channels, const std::vector &dilations, - const bool batchnorm, const std::string activation, - std::vector ¶ms) - : Buffer(*std::max_element(dilations.begin(), dilations.end())) { - this->_verify_params(channels, dilations, batchnorm, params.size()); - this->_blocks.resize(dilations.size()); - std::vector::iterator it = params.begin(); - for (int i = 0; i < dilations.size(); i++) - this->_blocks[i].set_params_(i == 0 ? 1 : channels, channels, dilations[i], - batchnorm, activation, it); - this->_block_vals.resize(this->_blocks.size() + 1); - this->_head = _Head(channels, it); - if (it != params.end()) - throw std::runtime_error( - "Didn't touch all the params when initializing wavenet"); - this->_reset_anti_pop_(); -} - -void convnet::ConvNet::_process_core_() { - this->_update_buffers_(); - // Main computation! - const long i_start = this->_input_buffer_offset; - const long num_frames = this->_input_post_gain.size(); - const long i_end = i_start + num_frames; - // TODO one unnecessary copy :/ #speed - for (auto i = i_start; i < i_end; i++) - this->_block_vals[0](0, i) = this->_input_buffer[i]; - for (auto i = 0; i < this->_blocks.size(); i++) - this->_blocks[i].process_(this->_block_vals[i], this->_block_vals[i + 1], - i_start, i_end); - // TODO clean up this allocation - this->_head.process_(this->_block_vals[this->_blocks.size()], - this->_head_output, i_start, i_end); - // Copy to required output array (TODO tighten this up) - for (int s = 0; s < num_frames; s++) - this->_core_dsp_output[s] = this->_head_output(s); - // Apply anti-pop - this->_anti_pop_(); -} - -void convnet::ConvNet::_verify_params(const int channels, - const std::vector &dilations, - const bool batchnorm, - const size_t actual_params) { - // TODO -} - -void convnet::ConvNet::_update_buffers_() { - this->Buffer::_update_buffers_(); - const long buffer_size = this->_input_buffer.size(); - this->_block_vals[0].resize(1, buffer_size); - for (long i = 1; i < this->_block_vals.size(); i++) - this->_block_vals[i].resize(this->_blocks[i - 1].get_out_channels(), - buffer_size); -} - -void convnet::ConvNet::_rewind_buffers_() { - // Need to rewind the block vals first because Buffer::rewind_buffers() - // resets the offset index - // The last _block_vals is the output of the last block and doesn't need to be - // rewound. - for (long k = 0; k < this->_block_vals.size() - 1; k++) { - // We actually don't need to pull back a lot...just as far as the first - // input sample would grab from dilation - const long _dilation = this->_blocks[k].conv.get_dilation(); - for (long i = this->_receptive_field - _dilation, - j = this->_input_buffer_offset - _dilation; - j < this->_input_buffer_offset; i++, j++) - for (long r = 0; r < this->_block_vals[k].rows(); r++) - this->_block_vals[k](r, i) = this->_block_vals[k](r, j); - } - // Now we can do the rest of the rewind - this->Buffer::_rewind_buffers_(); -} - -void convnet::ConvNet::_anti_pop_() { - if (this->_anti_pop_countdown >= this->_anti_pop_ramp) - return; - const float slope = 1.0f / float(this->_anti_pop_ramp); - for (int i = 0; i < this->_core_dsp_output.size(); i++) { - if (this->_anti_pop_countdown >= this->_anti_pop_ramp) - break; - const float gain = - std::max(slope * float(this->_anti_pop_countdown), float(0.0)); - this->_core_dsp_output[i] *= gain; - this->_anti_pop_countdown++; - } -} - -void convnet::ConvNet::_reset_anti_pop_() { - // You need the "real" receptive field, not the buffers. - long receptive_field = 1; - for (int i = 0; i < this->_blocks.size(); i++) - receptive_field += this->_blocks[i].conv.get_dilation(); - this->_anti_pop_countdown = -receptive_field; -} // ============================================================================ // Implementation of Version 2 interface diff --git a/dsp/dsp.h b/dsp/dsp.h index 359b572..666a883 100644 --- a/dsp/dsp.h +++ b/dsp/dsp.h @@ -1,7 +1,5 @@ #pragma once -#if IPLUG_DSP - #include #include #include @@ -10,296 +8,6 @@ #include #include "IPlugConstants.h" -#include - -enum EArchitectures { - kLinear = 0, - kConvNet, - kLSTM, - kCatLSTM, - kWaveNet, - kCatWaveNet, - kNumModels -}; - -// Class for providing params from the plugin to the DSP module -// For now, we'll work with doubles. Later, we'll add other types. -class DSPParam { -public: - const char *name; - const double val; -}; -// And the params shall be provided as a std::vector. - -class DSP { -public: - DSP(); - // process() does all of the processing requried to take `inputs` array and - // fill in the required values on `outputs`. - // To do this: - // 1. The parameters from the plugin (I/O levels and any other parametric - // inputs) are gotten. - // 2. The input level is applied - // 3. The core DSP algorithm is run (This is what should probably be - // overridden in subclasses). - // 4. The output level is applied and the result stored to `output`. - virtual void process(iplug::sample **inputs, iplug::sample **outputs, - const int num_channels, const int num_frames, - const double input_gain, const double output_gain, - const std::unordered_map ¶ms); - // Anything to take care of before next buffer comes in. - // For example: - // * Move the buffer index forward - // * Does NOT say that params aren't stale; that's the job of the routine - // that actually uses them, which varies depends on the particulars of the - // DSP subclass implementation. - virtual void finalize_(const int num_frames); - -protected: - // Parameters (aka "knobs") - std::unordered_map _params; - // If the params have changed since the last buffer was processed: - bool _stale_params; - // Where to store the samples after applying input gain - std::vector _input_post_gain; - // Location for the output of the core DSP algorithm. - std::vector _core_dsp_output; - - // Methods - - // Copy the parameters to the DSP module. - // If anything has changed, then set this->_stale_params to true. - // (TODO use "listener" approach) - void - _get_params_(const std::unordered_map &input_params); - - // Apply the input gain - // Result populates this->_input_post_gain - void _apply_input_level_(iplug::sample **inputs, const int num_channels, - const int num_frames, const double gain); - - // i.e. ensure the size is correct. - void _ensure_core_dsp_output_ready_(); - - // The core of your DSP algorithm. - // Access the inputs in this->_input_post_gain - // Place the outputs in this->_core_dsp_output - virtual void _process_core_(); - - // Copy this->_core_dsp_output to output and apply the output volume - void _apply_output_level_(iplug::sample **outputs, const int num_channels, - const int num_frames, const double gain); -}; - -// Class where an input buffer is kept so that long-time effects can be -// captured. (e.g. conv nets or impulse responses, where we need history that's -// longer than the sample buffer that's coming in.) -class Buffer : public DSP { -public: - Buffer(const int receptive_field); - void finalize_(const int num_frames); - -protected: - // Input buffer - const int _input_buffer_channels = 1; // Mono - int _receptive_field; - // First location where we add new samples from the input - long _input_buffer_offset; - std::vector _input_buffer; - std::vector _output_buffer; - - void _set_receptive_field(const int new_receptive_field, - const int input_buffer_size); - void _set_receptive_field(const int new_receptive_field); - void _reset_input_buffer(); - // Use this->_input_post_gain - virtual void _update_buffers_(); - virtual void _rewind_buffers_(); -}; - -// Basic linear model (an IR!) -class Linear : public Buffer { -public: - Linear(const int receptive_field, const bool _bias, - const std::vector ¶ms); - void _process_core_() override; - -protected: - Eigen::VectorXf _weight; - float _bias; -}; - -// NN modules ================================================================= - -// Activations - -// In-place ReLU on (N,M) array -void relu_(Eigen::MatrixXf &x, const long i_start, const long i_end, - const long j_start, const long j_end); -// Subset of the columns -void relu_(Eigen::MatrixXf &x, const long j_start, const long j_end); -void relu_(Eigen::MatrixXf &x); - -// In-place sigmoid -void sigmoid_(Eigen::MatrixXf &x, const long i_start, const long i_end, - const long j_start, const long j_end); -void sigmoid_(Eigen::MatrixXf &x); - -// In-place Tanh on (N,M) array -void tanh_(Eigen::MatrixXf &x, const long i_start, const long i_end, - const long j_start, const long j_end); -// Subset of the columns -void tanh_(Eigen::MatrixXf &x, const long i_start, const long i_end); - -void tanh_(Eigen::MatrixXf &x); - -class Conv1D { -public: - Conv1D() { this->_dilation = 1; }; - void set_params_(std::vector::iterator ¶ms); - void set_size_(const int in_channels, const int out_channels, - const int kernel_size, const bool do_bias, - const int _dilation); - void set_size_and_params_(const int in_channels, const int out_channels, - const int kernel_size, const int _dilation, - const bool do_bias, - std::vector::iterator ¶ms); - // Process from input to output - // Rightmost indices of input go from i_start to i_end, - // Indices on output for from j_start (to j_start + i_end - i_start) - void process_(const Eigen::MatrixXf &input, Eigen::MatrixXf &output, - const long i_start, const long i_end, const long j_start) const; - long get_in_channels() const { - return this->_weight.size() > 0 ? this->_weight[0].cols() : 0; - }; - long get_kernel_size() const { return this->_weight.size(); }; - long get_num_params() const; - long get_out_channels() const { - return this->_weight.size() > 0 ? this->_weight[0].rows() : 0; - }; - int get_dilation() const { return this->_dilation; }; - -private: - // Gonna wing this... - // conv[kernel](cout, cin) - std::vector _weight; - Eigen::VectorXf _bias; - int _dilation; -}; - -// Really just a linear layer -class Conv1x1 { -public: - Conv1x1(const int in_channels, const int out_channels, const bool _bias); - void set_params_(std::vector::iterator ¶ms); - // :param input: (N,Cin) or (Cin,) - // :return: (N,Cout) or (Cout,), respectively - Eigen::MatrixXf process(const Eigen::MatrixXf &input) const; - - long get_out_channels() const { return this->_weight.rows(); }; - -private: - Eigen::MatrixXf _weight; - Eigen::VectorXf _bias; - bool _do_bias; -}; - -// ConvNet ==================================================================== - -namespace convnet { -// Custom Conv that avoids re-computing on pieces of the input and trusts -// that the corresponding outputs are where they need to be. -// Beware: this is clever! - -// Batch normalization -// In prod mode, so really just an elementwise affine layer. -class BatchNorm { -public: - BatchNorm(){}; - BatchNorm(const int dim, std::vector::iterator ¶ms); - void process_(Eigen::MatrixXf &input, const long i_start, - const long i_end) const; - -private: - // TODO simplify to just ax+b - // y = (x-m)/sqrt(v+eps) * w + bias - // y = ax+b - // a = w / sqrt(v+eps) - // b = a * m + bias - Eigen::VectorXf scale; - Eigen::VectorXf loc; -}; - -class ConvNetBlock { -public: - ConvNetBlock() { this->_batchnorm = false; }; - void set_params_(const int in_channels, const int out_channels, - const int _dilation, const bool batchnorm, - const std::string activation, - std::vector::iterator ¶ms); - void process_(const Eigen::MatrixXf &input, Eigen::MatrixXf &output, - const long i_start, const long i_end) const; - long get_out_channels() const; - Conv1D conv; - -private: - BatchNorm batchnorm; - bool _batchnorm; - std::string activation; -}; - -class _Head { -public: - _Head() { this->_bias = (float)0.0; }; - _Head(const int channels, std::vector::iterator ¶ms); - void process_(const Eigen::MatrixXf &input, Eigen::VectorXf &output, - const long i_start, const long i_end) const; - -private: - Eigen::VectorXf _weight; - float _bias; -}; - -class ConvNet : public Buffer { -public: - ConvNet(const int channels, const std::vector &dilations, - const bool batchnorm, const std::string activation, - std::vector ¶ms); - -protected: - std::vector _blocks; - std::vector _block_vals; - Eigen::VectorXf _head_output; - _Head _head; - void _verify_params(const int channels, const std::vector &dilations, - const bool batchnorm, const size_t actual_params); - void _update_buffers_() override; - void _rewind_buffers_() override; - - void _process_core_() override; - - // The net starts with random parameters inside; we need to wait for a full - // receptive field to pass through before we can count on the output being - // ok. This implements a gentle "ramp-up" so that there's no "pop" at the - // start. - long _anti_pop_countdown; - const long _anti_pop_ramp = 100; - void _anti_pop_(); - void _reset_anti_pop_(); -}; -}; // namespace convnet - -// Utilities ================================================================== -// Implemented in get_dsp.cpp - -// Verify that the config that we are building our model from is supported by -// this plugin version. -void verify_config_version(const std::string version); - -// Takes the model file and uses it to instantiate an instance of DSP. -std::unique_ptr get_dsp(const std::filesystem::path model_file); -// Legacy loader for directory-type DSPs -std::unique_ptr get_dsp_legacy(const std::filesystem::path dirname); // Version 2 DSP abstraction ================================================== @@ -396,5 +104,3 @@ class History : public DSP { void _RewindHistory(); }; }; // namespace dsp - -#endif // IPLUG_DSP From 95b2ac6234d91b960f10b1da4575431f306ad1b2 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 24 Mar 2023 12:29:03 -0700 Subject: [PATCH 2/2] Move cncp.cpp/.h to NAM subdir --- {dsp => NAM}/cnpy.cpp | 0 {dsp => NAM}/cnpy.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {dsp => NAM}/cnpy.cpp (100%) rename {dsp => NAM}/cnpy.h (100%) diff --git a/dsp/cnpy.cpp b/NAM/cnpy.cpp similarity index 100% rename from dsp/cnpy.cpp rename to NAM/cnpy.cpp diff --git a/dsp/cnpy.h b/NAM/cnpy.h similarity index 100% rename from dsp/cnpy.h rename to NAM/cnpy.h