diff --git a/NAM/activations.cpp b/NAM/activations.cpp index 92f1d8d..828cea8 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -4,12 +4,14 @@ nam::activations::ActivationTanh _TANH = nam::activations::ActivationTanh(); nam::activations::ActivationFastTanh _FAST_TANH = nam::activations::ActivationFastTanh(); nam::activations::ActivationHardTanh _HARD_TANH = nam::activations::ActivationHardTanh(); nam::activations::ActivationReLU _RELU = nam::activations::ActivationReLU(); +nam::activations::ActivationLeakyReLU _LEAKY_RELU = nam::activations::ActivationLeakyReLU(); nam::activations::ActivationSigmoid _SIGMOID = nam::activations::ActivationSigmoid(); bool nam::activations::Activation::using_fast_tanh = false; -std::unordered_map nam::activations::Activation::_activations = - {{"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, {"ReLU", &_RELU}, {"Sigmoid", &_SIGMOID}}; +std::unordered_map nam::activations::Activation::_activations = { + {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, + {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}}; nam::activations::Activation* tanh_bak = nullptr; diff --git a/NAM/activations.h b/NAM/activations.h index e9afc33..fe47203 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -39,6 +39,13 @@ inline float fast_sigmoid(const float x) return 0.5f * (fast_tanh(x * 0.5f) + 1.0f); } +// Assumes PyTorch default of 0.01 for negative slope. This may change to be +// configurable in the future. +inline float leaky_relu(float x) +{ + const float negative_slope = 0.01; + return x > 0.0f ? x : negative_slope * x; +} class Activation { @@ -110,6 +117,18 @@ class ActivationReLU : public Activation } }; +class ActivationLeakyReLU : public Activation +{ +public: + void apply(float* data, long size) override + { + for (long pos = 0; pos < size; pos++) + { + data[pos] = leaky_relu(data[pos]); + } + } +}; + class ActivationSigmoid : public Activation { public: diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 243dcce..443a729 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -1,6 +1,8 @@ // Entry point for tests +// See the GitHub Action for a demo how to build and run tests. #include +#include "test/test_activations.cpp" #include "test/test_dsp.cpp" #include "test/test_get_dsp.cpp" @@ -8,6 +10,11 @@ int main() { std::cout << "Running tests..." << std::endl; // TODO Automatically loop, catch exceptions, log results + + test_activations::TestLeakyReLU::test_core_function(); + test_activations::TestLeakyReLU::test_get_by_init(); + test_activations::TestLeakyReLU::test_get_by_str(); + test_dsp::test_construct(); test_dsp::test_get_input_level(); test_dsp::test_get_output_level(); diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp new file mode 100644 index 0000000..e83e9f6 --- /dev/null +++ b/tools/test/test_activations.cpp @@ -0,0 +1,68 @@ +// Tests for activation functions +// +// Things you want ot test for: +// 1. That the core elementwise funciton is snapshot-correct. +// 2. The class that wraps the core function for an array of data +// 3. .cpp: that you have the singleton defined, and that it's in the unordered map to get by string + +#include +#include +#include + +#include "NAM/activations.h" + +namespace test_activations +{ +class TestLeakyReLU +{ +public: + static void test_core_function() + { + auto TestCase = [](float input, float expectedOutput) { + float actualOutput = nam::activations::leaky_relu(input); + assert(actualOutput == expectedOutput); + }; + // A few snapshot tests + TestCase(0.0f, 0.0f); + TestCase(1.0f, 1.0f); + TestCase(-1.0f, -0.01f); + }; + + static void test_get_by_init() + { + auto a = nam::activations::ActivationLeakyReLU(); + _test_class(&a); + } + + // Get the singleton and test it + static void test_get_by_str() + { + const std::string name = "LeakyReLU"; + auto a = nam::activations::Activation::get_activation(name); + _test_class(a); + } + +private: + // Put the class through its paces + static void _test_class(nam::activations::Activation* a) + { + std::vector inputs, expectedOutputs; + + inputs.push_back(0.0f); + expectedOutputs.push_back(0.0f); + + inputs.push_back(1.0f); + expectedOutputs.push_back(1.0f); + + inputs.push_back(-1.0f); + expectedOutputs.push_back(-0.01f); + + a->apply(inputs.data(), (long)inputs.size()); + for (auto itActual = inputs.begin(), itExpected = expectedOutputs.begin(); itActual != inputs.end(); + ++itActual, ++itExpected) + { + assert(*itActual == *itExpected); + } + }; +}; +}; // namespace test_activations