From e43a95e4e2371e53dee63e93fb81c2b3e16af7bd Mon Sep 17 00:00:00 2001
From: Steven Atkinson <steven@atkinson.mn>
Date: Sat, 19 Oct 2024 16:15:38 -0700
Subject: [PATCH 1/2] Add support for LeakyReLU activation

---
 NAM/activations.cpp             |  6 ++--
 NAM/activations.h               | 19 ++++++++++
 NAM/convnet.h                   |  6 ++--
 NAM/get_dsp.h                   | 17 ++++-----
 NAM/wavenet.h                   |  2 +-
 tools/run_tests.cpp             |  7 ++++
 tools/test/test_activations.cpp | 61 +++++++++++++++++++++++++++++++++
 7 files changed, 104 insertions(+), 14 deletions(-)
 create mode 100644 tools/test/test_activations.cpp

diff --git a/NAM/activations.cpp b/NAM/activations.cpp
index 92f1d8d..828cea8 100644
--- a/NAM/activations.cpp
+++ b/NAM/activations.cpp
@@ -4,12 +4,14 @@ nam::activations::ActivationTanh _TANH = nam::activations::ActivationTanh();
 nam::activations::ActivationFastTanh _FAST_TANH = nam::activations::ActivationFastTanh();
 nam::activations::ActivationHardTanh _HARD_TANH = nam::activations::ActivationHardTanh();
 nam::activations::ActivationReLU _RELU = nam::activations::ActivationReLU();
+nam::activations::ActivationLeakyReLU _LEAKY_RELU = nam::activations::ActivationLeakyReLU();
 nam::activations::ActivationSigmoid _SIGMOID = nam::activations::ActivationSigmoid();
 
 bool nam::activations::Activation::using_fast_tanh = false;
 
-std::unordered_map<std::string, nam::activations::Activation*> nam::activations::Activation::_activations =
-  {{"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, {"ReLU", &_RELU}, {"Sigmoid", &_SIGMOID}};
+std::unordered_map<std::string, nam::activations::Activation*> nam::activations::Activation::_activations = {
+  {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH},   {"Fasttanh", &_FAST_TANH},
+  {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}};
 
 nam::activations::Activation* tanh_bak = nullptr;
 
diff --git a/NAM/activations.h b/NAM/activations.h
index e9afc33..fe47203 100644
--- a/NAM/activations.h
+++ b/NAM/activations.h
@@ -39,6 +39,13 @@ inline float fast_sigmoid(const float x)
   return 0.5f * (fast_tanh(x * 0.5f) + 1.0f);
 }
 
+// Assumes PyTorch default of 0.01 for negative slope. This may change to be
+// configurable in the future.
+inline float leaky_relu(float x)
+{
+  const float negative_slope = 0.01;
+  return x > 0.0f ? x : negative_slope * x;
+}
 
 class Activation
 {
@@ -110,6 +117,18 @@ class ActivationReLU : public Activation
   }
 };
 
+class ActivationLeakyReLU : public Activation
+{
+public:
+  void apply(float* data, long size) override
+  {
+    for (long pos = 0; pos < size; pos++)
+    {
+      data[pos] = leaky_relu(data[pos]);
+    }
+  }
+};
+
 class ActivationSigmoid : public Activation
 {
 public:
diff --git a/NAM/convnet.h b/NAM/convnet.h
index 458cf67..b1fa142 100644
--- a/NAM/convnet.h
+++ b/NAM/convnet.h
@@ -22,7 +22,7 @@ namespace convnet
 class BatchNorm
 {
 public:
-  BatchNorm() {};
+  BatchNorm(){};
   BatchNorm(const int dim, std::vector<float>::iterator& weights);
   void process_(Eigen::MatrixXf& input, const long i_start, const long i_end) const;
 
@@ -39,7 +39,7 @@ class BatchNorm
 class ConvNetBlock
 {
 public:
-  ConvNetBlock() {};
+  ConvNetBlock(){};
   void set_weights_(const int in_channels, const int out_channels, const int _dilation, const bool batchnorm,
                     const std::string activation, std::vector<float>::iterator& weights);
   void process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long i_end) const;
@@ -55,7 +55,7 @@ class ConvNetBlock
 class _Head
 {
 public:
-  _Head() {};
+  _Head(){};
   _Head(const int channels, std::vector<float>::iterator& weights);
   void process_(const Eigen::MatrixXf& input, Eigen::VectorXf& output, const long i_start, const long i_end) const;
 
diff --git a/NAM/get_dsp.h b/NAM/get_dsp.h
index d414ca5..1d2c8d5 100644
--- a/NAM/get_dsp.h
+++ b/NAM/get_dsp.h
@@ -1,12 +1,13 @@
 #include <fstream>
 
-namespace nam {
-    // Get NAM from a .nam file at the provided location
-    std::unique_ptr<DSP> get_dsp(const std::filesystem::path config_filename);
+namespace nam
+{
+// Get NAM from a .nam file at the provided location
+std::unique_ptr<DSP> get_dsp(const std::filesystem::path config_filename);
 
-    // Get NAM from a provided configuration struct
-    std::unique_ptr<DSP> get_dsp(dspData& conf);
+// Get NAM from a provided configuration struct
+std::unique_ptr<DSP> get_dsp(dspData& conf);
 
-    // Get NAM from a provided .nam file path and store its configuration in the provided conf
-    std::unique_ptr<DSP> get_dsp(const std::filesystem::path config_filename, dspData& returnedConfig);
-};  // namespace nam
+// Get NAM from a provided .nam file path and store its configuration in the provided conf
+std::unique_ptr<DSP> get_dsp(const std::filesystem::path config_filename, dspData& returnedConfig);
+}; // namespace nam
diff --git a/NAM/wavenet.h b/NAM/wavenet.h
index ff64b32..a2ad2fc 100644
--- a/NAM/wavenet.h
+++ b/NAM/wavenet.h
@@ -29,7 +29,7 @@ class _Layer
   , _input_mixin(condition_size, gated ? 2 * channels : channels, false)
   , _1x1(channels, channels, true)
   , _activation(activations::Activation::get_activation(activation))
-  , _gated(gated) {};
+  , _gated(gated){};
   void set_weights_(std::vector<float>::iterator& weights);
   // :param `input`: from previous layer
   // :param `output`: to next layer
diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp
index 243dcce..6e65ee8 100644
--- a/tools/run_tests.cpp
+++ b/tools/run_tests.cpp
@@ -1,6 +1,8 @@
 // Entry point for tests
+// See the GitHub Action for a demo how to build and run tests.
 
 #include <iostream>
+#include "test/test_activations.cpp"
 #include "test/test_dsp.cpp"
 #include "test/test_get_dsp.cpp"
 
@@ -8,6 +10,11 @@ int main()
 {
   std::cout << "Running tests..." << std::endl;
   // TODO Automatically loop, catch exceptions, log results
+  
+  test_activations::TestLeakyReLU::test_core_function();
+  test_activations::TestLeakyReLU::test_get_by_init();
+  test_activations::TestLeakyReLU::test_get_by_str();
+
   test_dsp::test_construct();
   test_dsp::test_get_input_level();
   test_dsp::test_get_output_level();
diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp
new file mode 100644
index 0000000..ab722ad
--- /dev/null
+++ b/tools/test/test_activations.cpp
@@ -0,0 +1,61 @@
+// Tests for activation functions
+//
+// Things you want ot test for:
+// 1. That the core elementwise funciton is snapshot-correct.
+// 2. The class that wraps the core function for an array of data
+// 3. .cpp: that you have the singleton defined, and that it's in the unordered map to get by string
+
+#include <cassert>
+#include <string>
+#include <vector>
+
+#include "NAM/activations.h"
+
+namespace test_activations
+{
+    class TestLeakyReLU {
+        public:
+        static void test_core_function() {
+            auto TestCase = [] (float input, float expectedOutput) {
+                float actualOutput = nam::activations::leaky_relu(input);
+                assert(actualOutput == expectedOutput);
+            };
+            // A few snapshot tests
+            TestCase(0.0f, 0.0f);
+            TestCase(1.0f, 1.0f);
+            TestCase(-1.0f, -0.01f);
+        };
+
+        static void test_get_by_init() {
+            auto a = nam::activations::ActivationLeakyReLU();
+            _test_class(&a);
+        }
+
+        // Get the singleton and test it
+        static void test_get_by_str() {
+            const std::string name="LeakyReLU";
+            auto a = nam::activations::Activation::get_activation(name);
+            _test_class(a);
+        }
+
+        private:
+        // Put the class through its paces
+        static void _test_class(nam::activations::Activation *a) {
+            std::vector<float> inputs, expectedOutputs;
+            
+            inputs.push_back(0.0f);
+            expectedOutputs.push_back(0.0f);
+
+            inputs.push_back(1.0f);
+            expectedOutputs.push_back(1.0f);
+
+            inputs.push_back(-1.0f);
+            expectedOutputs.push_back(-0.01f);
+
+            a->apply(inputs.data(), (long) inputs.size());
+            for (auto itActual=inputs.begin(), itExpected=expectedOutputs.begin(); itActual != inputs.end(); ++itActual, ++itExpected) {
+                assert(*itActual == *itExpected);
+            }
+        };
+    };
+}; // namespace test_activation

From 1ff63cb069b20d1a110262a7dd7142729888a953 Mon Sep 17 00:00:00 2001
From: Steven Atkinson <steven@atkinson.mn>
Date: Sat, 19 Oct 2024 16:21:20 -0700
Subject: [PATCH 2/2] Formatting

---
 tools/run_tests.cpp             |  2 +-
 tools/test/test_activations.cpp | 97 ++++++++++++++++++---------------
 2 files changed, 53 insertions(+), 46 deletions(-)

diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp
index 6e65ee8..443a729 100644
--- a/tools/run_tests.cpp
+++ b/tools/run_tests.cpp
@@ -10,7 +10,7 @@ int main()
 {
   std::cout << "Running tests..." << std::endl;
   // TODO Automatically loop, catch exceptions, log results
-  
+
   test_activations::TestLeakyReLU::test_core_function();
   test_activations::TestLeakyReLU::test_get_by_init();
   test_activations::TestLeakyReLU::test_get_by_str();
diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp
index ab722ad..e83e9f6 100644
--- a/tools/test/test_activations.cpp
+++ b/tools/test/test_activations.cpp
@@ -13,49 +13,56 @@
 
 namespace test_activations
 {
-    class TestLeakyReLU {
-        public:
-        static void test_core_function() {
-            auto TestCase = [] (float input, float expectedOutput) {
-                float actualOutput = nam::activations::leaky_relu(input);
-                assert(actualOutput == expectedOutput);
-            };
-            // A few snapshot tests
-            TestCase(0.0f, 0.0f);
-            TestCase(1.0f, 1.0f);
-            TestCase(-1.0f, -0.01f);
-        };
-
-        static void test_get_by_init() {
-            auto a = nam::activations::ActivationLeakyReLU();
-            _test_class(&a);
-        }
-
-        // Get the singleton and test it
-        static void test_get_by_str() {
-            const std::string name="LeakyReLU";
-            auto a = nam::activations::Activation::get_activation(name);
-            _test_class(a);
-        }
-
-        private:
-        // Put the class through its paces
-        static void _test_class(nam::activations::Activation *a) {
-            std::vector<float> inputs, expectedOutputs;
-            
-            inputs.push_back(0.0f);
-            expectedOutputs.push_back(0.0f);
-
-            inputs.push_back(1.0f);
-            expectedOutputs.push_back(1.0f);
-
-            inputs.push_back(-1.0f);
-            expectedOutputs.push_back(-0.01f);
-
-            a->apply(inputs.data(), (long) inputs.size());
-            for (auto itActual=inputs.begin(), itExpected=expectedOutputs.begin(); itActual != inputs.end(); ++itActual, ++itExpected) {
-                assert(*itActual == *itExpected);
-            }
-        };
+class TestLeakyReLU
+{
+public:
+  static void test_core_function()
+  {
+    auto TestCase = [](float input, float expectedOutput) {
+      float actualOutput = nam::activations::leaky_relu(input);
+      assert(actualOutput == expectedOutput);
     };
-}; // namespace test_activation
+    // A few snapshot tests
+    TestCase(0.0f, 0.0f);
+    TestCase(1.0f, 1.0f);
+    TestCase(-1.0f, -0.01f);
+  };
+
+  static void test_get_by_init()
+  {
+    auto a = nam::activations::ActivationLeakyReLU();
+    _test_class(&a);
+  }
+
+  // Get the singleton and test it
+  static void test_get_by_str()
+  {
+    const std::string name = "LeakyReLU";
+    auto a = nam::activations::Activation::get_activation(name);
+    _test_class(a);
+  }
+
+private:
+  // Put the class through its paces
+  static void _test_class(nam::activations::Activation* a)
+  {
+    std::vector<float> inputs, expectedOutputs;
+
+    inputs.push_back(0.0f);
+    expectedOutputs.push_back(0.0f);
+
+    inputs.push_back(1.0f);
+    expectedOutputs.push_back(1.0f);
+
+    inputs.push_back(-1.0f);
+    expectedOutputs.push_back(-0.01f);
+
+    a->apply(inputs.data(), (long)inputs.size());
+    for (auto itActual = inputs.begin(), itExpected = expectedOutputs.begin(); itActual != inputs.end();
+         ++itActual, ++itExpected)
+    {
+      assert(*itActual == *itExpected);
+    }
+  };
+};
+}; // namespace test_activations
