[FEATURE] Add support for LeakyReLU activation

NAM/activations.cpp

@@ -4,12 +4,14 @@ nam::activations::ActivationTanh _TANH = nam::activations::ActivationTanh();
 nam::activations::ActivationFastTanh _FAST_TANH = nam::activations::ActivationFastTanh();
 nam::activations::ActivationHardTanh _HARD_TANH = nam::activations::ActivationHardTanh();
 nam::activations::ActivationReLU _RELU = nam::activations::ActivationReLU();
+nam::activations::ActivationLeakyReLU _LEAKY_RELU = nam::activations::ActivationLeakyReLU();
 nam::activations::ActivationSigmoid _SIGMOID = nam::activations::ActivationSigmoid();
 
 bool nam::activations::Activation::using_fast_tanh = false;
 
-std::unordered_map<std::string, nam::activations::Activation*> nam::activations::Activation::_activations =
-  {{"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, {"ReLU", &_RELU}, {"Sigmoid", &_SIGMOID}};
+std::unordered_map<std::string, nam::activations::Activation*> nam::activations::Activation::_activations = {
+  {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH},   {"Fasttanh", &_FAST_TANH},
+  {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}};
 
 nam::activations::Activation* tanh_bak = nullptr;
 

NAM/activations.h

@@ -39,6 +39,13 @@ inline float fast_sigmoid(const float x)
   return 0.5f * (fast_tanh(x * 0.5f) + 1.0f);
 }
 
+// Assumes PyTorch default of 0.01 for negative slope. This may change to be
+// configurable in the future.
+inline float leaky_relu(float x)
+{
+  const float negative_slope = 0.01;
+  return x > 0.0f ? x : negative_slope * x;
+}
 
 class Activation
 {
@@ -110,6 +117,18 @@ class ActivationReLU : public Activation
   }
 };
 
+class ActivationLeakyReLU : public Activation
+{
+public:
+  void apply(float* data, long size) override
+  {
+    for (long pos = 0; pos < size; pos++)
+    {
+      data[pos] = leaky_relu(data[pos]);
+    }
+  }
+};
+
 class ActivationSigmoid : public Activation
 {
 public:

tools/run_tests.cpp

@@ -1,13 +1,20 @@
 // Entry point for tests
+// See the GitHub Action for a demo how to build and run tests.
 
 #include <iostream>
+#include "test/test_activations.cpp"
 #include "test/test_dsp.cpp"
 #include "test/test_get_dsp.cpp"
 
 int main()
 {
   std::cout << "Running tests..." << std::endl;
   // TODO Automatically loop, catch exceptions, log results
+
+  test_activations::TestLeakyReLU::test_core_function();
+  test_activations::TestLeakyReLU::test_get_by_init();
+  test_activations::TestLeakyReLU::test_get_by_str();
+
   test_dsp::test_construct();
   test_dsp::test_get_input_level();
   test_dsp::test_get_output_level();

tools/test/test_activations.cpp

@@ -0,0 +1,68 @@
+// Tests for activation functions
+//
+// Things you want ot test for:
+// 1. That the core elementwise funciton is snapshot-correct.
+// 2. The class that wraps the core function for an array of data
+// 3. .cpp: that you have the singleton defined, and that it's in the unordered map to get by string
+
+#include <cassert>
+#include <string>
+#include <vector>
+
+#include "NAM/activations.h"
+
+namespace test_activations
+{
+class TestLeakyReLU
+{
+public:
+  static void test_core_function()
+  {
+    auto TestCase = [](float input, float expectedOutput) {
+      float actualOutput = nam::activations::leaky_relu(input);
+      assert(actualOutput == expectedOutput);
+    };
+    // A few snapshot tests
+    TestCase(0.0f, 0.0f);
+    TestCase(1.0f, 1.0f);
+    TestCase(-1.0f, -0.01f);
+  };
+
+  static void test_get_by_init()
+  {
+    auto a = nam::activations::ActivationLeakyReLU();
+    _test_class(&a);
+  }
+
+  // Get the singleton and test it
+  static void test_get_by_str()
+  {
+    const std::string name = "LeakyReLU";
+    auto a = nam::activations::Activation::get_activation(name);
+    _test_class(a);
+  }
+
+private:
+  // Put the class through its paces
+  static void _test_class(nam::activations::Activation* a)
+  {
+    std::vector<float> inputs, expectedOutputs;
+
+    inputs.push_back(0.0f);
+    expectedOutputs.push_back(0.0f);
+
+    inputs.push_back(1.0f);
+    expectedOutputs.push_back(1.0f);
+
+    inputs.push_back(-1.0f);
+    expectedOutputs.push_back(-0.01f);
+
+    a->apply(inputs.data(), (long)inputs.size());
+    for (auto itActual = inputs.begin(), itExpected = expectedOutputs.begin(); itActual != inputs.end();
+         ++itActual, ++itExpected)
+    {
+      assert(*itActual == *itExpected);
+    }
+  };
+};
+}; // namespace test_activations