From e43a95e4e2371e53dee63e93fb81c2b3e16af7bd Mon Sep 17 00:00:00 2001 From: Steven Atkinson Date: Sat, 19 Oct 2024 16:15:38 -0700 Subject: [PATCH 1/2] Add support for LeakyReLU activation --- NAM/activations.cpp | 6 ++-- NAM/activations.h | 19 ++++++++++ NAM/convnet.h | 6 ++-- NAM/get_dsp.h | 17 ++++----- NAM/wavenet.h | 2 +- tools/run_tests.cpp | 7 ++++ tools/test/test_activations.cpp | 61 +++++++++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 tools/test/test_activations.cpp diff --git a/NAM/activations.cpp b/NAM/activations.cpp index 92f1d8d..828cea8 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -4,12 +4,14 @@ nam::activations::ActivationTanh _TANH = nam::activations::ActivationTanh(); nam::activations::ActivationFastTanh _FAST_TANH = nam::activations::ActivationFastTanh(); nam::activations::ActivationHardTanh _HARD_TANH = nam::activations::ActivationHardTanh(); nam::activations::ActivationReLU _RELU = nam::activations::ActivationReLU(); +nam::activations::ActivationLeakyReLU _LEAKY_RELU = nam::activations::ActivationLeakyReLU(); nam::activations::ActivationSigmoid _SIGMOID = nam::activations::ActivationSigmoid(); bool nam::activations::Activation::using_fast_tanh = false; -std::unordered_map nam::activations::Activation::_activations = - {{"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, {"ReLU", &_RELU}, {"Sigmoid", &_SIGMOID}}; +std::unordered_map nam::activations::Activation::_activations = { + {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, + {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}}; nam::activations::Activation* tanh_bak = nullptr; diff --git a/NAM/activations.h b/NAM/activations.h index e9afc33..fe47203 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -39,6 +39,13 @@ inline float fast_sigmoid(const float x) return 0.5f * (fast_tanh(x * 0.5f) + 1.0f); } +// Assumes PyTorch default of 0.01 for negative slope. This may change to be +// configurable in the future. +inline float leaky_relu(float x) +{ + const float negative_slope = 0.01; + return x > 0.0f ? x : negative_slope * x; +} class Activation { @@ -110,6 +117,18 @@ class ActivationReLU : public Activation } }; +class ActivationLeakyReLU : public Activation +{ +public: + void apply(float* data, long size) override + { + for (long pos = 0; pos < size; pos++) + { + data[pos] = leaky_relu(data[pos]); + } + } +}; + class ActivationSigmoid : public Activation { public: diff --git a/NAM/convnet.h b/NAM/convnet.h index 458cf67..b1fa142 100644 --- a/NAM/convnet.h +++ b/NAM/convnet.h @@ -22,7 +22,7 @@ namespace convnet class BatchNorm { public: - BatchNorm() {}; + BatchNorm(){}; BatchNorm(const int dim, std::vector::iterator& weights); void process_(Eigen::MatrixXf& input, const long i_start, const long i_end) const; @@ -39,7 +39,7 @@ class BatchNorm class ConvNetBlock { public: - ConvNetBlock() {}; + ConvNetBlock(){}; void set_weights_(const int in_channels, const int out_channels, const int _dilation, const bool batchnorm, const std::string activation, std::vector::iterator& weights); void process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long i_end) const; @@ -55,7 +55,7 @@ class ConvNetBlock class _Head { public: - _Head() {}; + _Head(){}; _Head(const int channels, std::vector::iterator& weights); void process_(const Eigen::MatrixXf& input, Eigen::VectorXf& output, const long i_start, const long i_end) const; diff --git a/NAM/get_dsp.h b/NAM/get_dsp.h index d414ca5..1d2c8d5 100644 --- a/NAM/get_dsp.h +++ b/NAM/get_dsp.h @@ -1,12 +1,13 @@ #include -namespace nam { - // Get NAM from a .nam file at the provided location - std::unique_ptr get_dsp(const std::filesystem::path config_filename); +namespace nam +{ +// Get NAM from a .nam file at the provided location +std::unique_ptr get_dsp(const std::filesystem::path config_filename); - // Get NAM from a provided configuration struct - std::unique_ptr get_dsp(dspData& conf); +// Get NAM from a provided configuration struct +std::unique_ptr get_dsp(dspData& conf); - // Get NAM from a provided .nam file path and store its configuration in the provided conf - std::unique_ptr get_dsp(const std::filesystem::path config_filename, dspData& returnedConfig); -}; // namespace nam +// Get NAM from a provided .nam file path and store its configuration in the provided conf +std::unique_ptr get_dsp(const std::filesystem::path config_filename, dspData& returnedConfig); +}; // namespace nam diff --git a/NAM/wavenet.h b/NAM/wavenet.h index ff64b32..a2ad2fc 100644 --- a/NAM/wavenet.h +++ b/NAM/wavenet.h @@ -29,7 +29,7 @@ class _Layer , _input_mixin(condition_size, gated ? 2 * channels : channels, false) , _1x1(channels, channels, true) , _activation(activations::Activation::get_activation(activation)) - , _gated(gated) {}; + , _gated(gated){}; void set_weights_(std::vector::iterator& weights); // :param `input`: from previous layer // :param `output`: to next layer diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 243dcce..6e65ee8 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -1,6 +1,8 @@ // Entry point for tests +// See the GitHub Action for a demo how to build and run tests. #include +#include "test/test_activations.cpp" #include "test/test_dsp.cpp" #include "test/test_get_dsp.cpp" @@ -8,6 +10,11 @@ int main() { std::cout << "Running tests..." << std::endl; // TODO Automatically loop, catch exceptions, log results + + test_activations::TestLeakyReLU::test_core_function(); + test_activations::TestLeakyReLU::test_get_by_init(); + test_activations::TestLeakyReLU::test_get_by_str(); + test_dsp::test_construct(); test_dsp::test_get_input_level(); test_dsp::test_get_output_level(); diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp new file mode 100644 index 0000000..ab722ad --- /dev/null +++ b/tools/test/test_activations.cpp @@ -0,0 +1,61 @@ +// Tests for activation functions +// +// Things you want ot test for: +// 1. That the core elementwise funciton is snapshot-correct. +// 2. The class that wraps the core function for an array of data +// 3. .cpp: that you have the singleton defined, and that it's in the unordered map to get by string + +#include +#include +#include + +#include "NAM/activations.h" + +namespace test_activations +{ + class TestLeakyReLU { + public: + static void test_core_function() { + auto TestCase = [] (float input, float expectedOutput) { + float actualOutput = nam::activations::leaky_relu(input); + assert(actualOutput == expectedOutput); + }; + // A few snapshot tests + TestCase(0.0f, 0.0f); + TestCase(1.0f, 1.0f); + TestCase(-1.0f, -0.01f); + }; + + static void test_get_by_init() { + auto a = nam::activations::ActivationLeakyReLU(); + _test_class(&a); + } + + // Get the singleton and test it + static void test_get_by_str() { + const std::string name="LeakyReLU"; + auto a = nam::activations::Activation::get_activation(name); + _test_class(a); + } + + private: + // Put the class through its paces + static void _test_class(nam::activations::Activation *a) { + std::vector inputs, expectedOutputs; + + inputs.push_back(0.0f); + expectedOutputs.push_back(0.0f); + + inputs.push_back(1.0f); + expectedOutputs.push_back(1.0f); + + inputs.push_back(-1.0f); + expectedOutputs.push_back(-0.01f); + + a->apply(inputs.data(), (long) inputs.size()); + for (auto itActual=inputs.begin(), itExpected=expectedOutputs.begin(); itActual != inputs.end(); ++itActual, ++itExpected) { + assert(*itActual == *itExpected); + } + }; + }; +}; // namespace test_activation From 1ff63cb069b20d1a110262a7dd7142729888a953 Mon Sep 17 00:00:00 2001 From: Steven Atkinson Date: Sat, 19 Oct 2024 16:21:20 -0700 Subject: [PATCH 2/2] Formatting --- tools/run_tests.cpp | 2 +- tools/test/test_activations.cpp | 97 ++++++++++++++++++--------------- 2 files changed, 53 insertions(+), 46 deletions(-) diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 6e65ee8..443a729 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -10,7 +10,7 @@ int main() { std::cout << "Running tests..." << std::endl; // TODO Automatically loop, catch exceptions, log results - + test_activations::TestLeakyReLU::test_core_function(); test_activations::TestLeakyReLU::test_get_by_init(); test_activations::TestLeakyReLU::test_get_by_str(); diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp index ab722ad..e83e9f6 100644 --- a/tools/test/test_activations.cpp +++ b/tools/test/test_activations.cpp @@ -13,49 +13,56 @@ namespace test_activations { - class TestLeakyReLU { - public: - static void test_core_function() { - auto TestCase = [] (float input, float expectedOutput) { - float actualOutput = nam::activations::leaky_relu(input); - assert(actualOutput == expectedOutput); - }; - // A few snapshot tests - TestCase(0.0f, 0.0f); - TestCase(1.0f, 1.0f); - TestCase(-1.0f, -0.01f); - }; - - static void test_get_by_init() { - auto a = nam::activations::ActivationLeakyReLU(); - _test_class(&a); - } - - // Get the singleton and test it - static void test_get_by_str() { - const std::string name="LeakyReLU"; - auto a = nam::activations::Activation::get_activation(name); - _test_class(a); - } - - private: - // Put the class through its paces - static void _test_class(nam::activations::Activation *a) { - std::vector inputs, expectedOutputs; - - inputs.push_back(0.0f); - expectedOutputs.push_back(0.0f); - - inputs.push_back(1.0f); - expectedOutputs.push_back(1.0f); - - inputs.push_back(-1.0f); - expectedOutputs.push_back(-0.01f); - - a->apply(inputs.data(), (long) inputs.size()); - for (auto itActual=inputs.begin(), itExpected=expectedOutputs.begin(); itActual != inputs.end(); ++itActual, ++itExpected) { - assert(*itActual == *itExpected); - } - }; +class TestLeakyReLU +{ +public: + static void test_core_function() + { + auto TestCase = [](float input, float expectedOutput) { + float actualOutput = nam::activations::leaky_relu(input); + assert(actualOutput == expectedOutput); }; -}; // namespace test_activation + // A few snapshot tests + TestCase(0.0f, 0.0f); + TestCase(1.0f, 1.0f); + TestCase(-1.0f, -0.01f); + }; + + static void test_get_by_init() + { + auto a = nam::activations::ActivationLeakyReLU(); + _test_class(&a); + } + + // Get the singleton and test it + static void test_get_by_str() + { + const std::string name = "LeakyReLU"; + auto a = nam::activations::Activation::get_activation(name); + _test_class(a); + } + +private: + // Put the class through its paces + static void _test_class(nam::activations::Activation* a) + { + std::vector inputs, expectedOutputs; + + inputs.push_back(0.0f); + expectedOutputs.push_back(0.0f); + + inputs.push_back(1.0f); + expectedOutputs.push_back(1.0f); + + inputs.push_back(-1.0f); + expectedOutputs.push_back(-0.01f); + + a->apply(inputs.data(), (long)inputs.size()); + for (auto itActual = inputs.begin(), itExpected = expectedOutputs.begin(); itActual != inputs.end(); + ++itActual, ++itExpected) + { + assert(*itActual == *itExpected); + } + }; +}; +}; // namespace test_activations