From 14cb71e0b54f93c34796bda77372f5b2d8496f80 Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Mon, 3 Apr 2023 13:34:12 -0700
Subject: [PATCH 1/9] Begin to refactor activation function handling

---
 NAM/activations.cpp |  10 ++++
 NAM/activations.h   | 130 ++++++++++++++++++++++++++++++++++++++++++--
 NAM/dsp.h           |   2 +
 NAM/lstm.cpp        |   1 -
 NAM/wavenet.cpp     |  35 +++++++-----
 NAM/wavenet.h       |   6 +-
 6 files changed, 161 insertions(+), 23 deletions(-)
 create mode 100644 NAM/activations.cpp

diff --git a/NAM/activations.cpp b/NAM/activations.cpp
new file mode 100644
index 0000000..746cf2e
--- /dev/null
+++ b/NAM/activations.cpp
@@ -0,0 +1,10 @@
+#pragma once
+
+#include "activations.h"
+
+std::unordered_map<std::string, activations::Activation*> activations::Activation::_activations = {
+  {"Tanh", new activations::ActivationHardTanh()},
+  {"Hardtanh", new activations::ActivationHardTanh()},
+  {"Fasttanh", new activations::ActivationFastTanh()},
+  {"ReLU", new activations::ActivationReLU()},
+  {"Sigmoid", new activations::ActivationSigmoid()}};
diff --git a/NAM/activations.h b/NAM/activations.h
index 52b1334..efa80a3 100644
--- a/NAM/activations.h
+++ b/NAM/activations.h
@@ -1,15 +1,137 @@
 #pragma once
 
+#include <string>
 #include <cmath> // expf
+#include <unordered_map>
+#include <Eigen/Dense>
 
 namespace activations
 {
-float relu(float x)
+inline float relu(float x)
 {
-  return x > 0.0f ? x : 0.0f;
+return x > 0.0f ? x : 0.0f;
 };
-float sigmoid(float x)
+
+inline float sigmoid(float x)
 {
-  return 1.0f / (1.0f + expf(-x));
+return 1.0f / (1.0f + expf(-x));
 };
+
+inline float hard_tanh(float x)
+{
+const float t = x < -1 ? -1 : x;
+return t > 1 ? 1 : t;
+}
+
+inline float fast_tanh(const float x)
+{
+const float ax = fabsf(x);
+const float x2 = x * x;
+
+return (x * (2.45550750702956f + 2.45550750702956f * ax + (0.893229853513558f + 0.821226666969744f * ax) * x2)
+        / (2.44506634652299f + (2.44506634652299f + x2) * fabsf(x + 0.814642734961073f * x * ax)));
+}
+
+class Activation
+{
+public:
+Activation(){};
+  virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.block(0, 0, matrix.rows(), matrix.cols())); }
+  virtual void apply(Eigen::Block<Eigen::MatrixXf> block) {}
+  static Activation* get_activation(const std::string name)
+  { 
+    if (_activations.find(name) == _activations.end())
+      return nullptr;
+
+    return _activations[name];
+  }
+
+  protected:
+    static std::unordered_map<std::string, Activation *> _activations;
+};
+
+class ActivationTanh : public Activation
+{
+  public:
+    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    {
+      float* ptr = block.data();
+
+      long size = block.rows() * block.cols();
+
+      for (long pos = 0; pos < size; pos++)
+      {
+        ptr[pos] = std::tanh(ptr[pos]);
+      }
+    }
+};
+
+class ActivationHardTanh : public Activation
+{
+  public:
+    ActivationHardTanh(){};
+    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    {
+      float* ptr = block.data();
+
+      long size = block.rows() * block.cols();
+
+      for (long pos = 0; pos < size; pos++)
+      {
+        ptr[pos] = hard_tanh(ptr[pos]);
+      }
+    }
+};
+
+class ActivationFastTanh : public Activation
+{
+  public:
+    ActivationFastTanh(){};
+    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    {
+      float* ptr = block.data();
+
+      long size = block.rows() * block.cols();
+
+      for (long pos = 0; pos < size; pos++)
+      {
+        ptr[pos] = fast_tanh(ptr[pos]);
+      }
+    }
+};
+
+class ActivationReLU : public Activation
+{
+  public:
+    ActivationReLU(){};
+    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    {
+      float* ptr = block.data();
+
+      long size = block.rows() * block.cols();
+
+      for (long pos = 0; pos < size; pos++)
+      {
+        ptr[pos] = relu(ptr[pos]);
+      }
+    }
+};
+
+class ActivationSigmoid : public Activation
+{
+  public:
+    ActivationSigmoid(){};
+    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    {
+      float* ptr = block.data();
+
+      long size = block.rows() * block.cols();
+
+      for (long pos = 0; pos < size; pos++)
+      {
+        ptr[pos] = sigmoid(ptr[pos]);
+      }
+    }
+};
+
 }; // namespace activations
\ No newline at end of file
diff --git a/NAM/dsp.h b/NAM/dsp.h
index c1f5bff..8123360 100644
--- a/NAM/dsp.h
+++ b/NAM/dsp.h
@@ -9,6 +9,8 @@
 
 #include <Eigen/Dense>
 
+#include "activations.h"
+
 enum EArchitectures
 {
   kLinear = 0,
diff --git a/NAM/lstm.cpp b/NAM/lstm.cpp
index bffd18d..39e3acf 100644
--- a/NAM/lstm.cpp
+++ b/NAM/lstm.cpp
@@ -2,7 +2,6 @@
 #include <string>
 #include <vector>
 
-#include "activations.h"
 #include "lstm.h"
 
 lstm::LSTMCell::LSTMCell(const int input_size, const int hidden_size, std::vector<float>::iterator& params)
diff --git a/NAM/wavenet.cpp b/NAM/wavenet.cpp
index 2dbf491..13f7e31 100644
--- a/NAM/wavenet.cpp
+++ b/NAM/wavenet.cpp
@@ -29,14 +29,18 @@ void wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::Matrix
   this->_conv.process_(input, this->_z, i_start, ncols, 0);
   // Mix-in condition
   this->_z += this->_input_mixin.process(condition);
-  if (this->_activation == "Hardtanh")
-    hard_tanh_(this->_z);
-  else if (this->_activation == "Tanh")
-    tanh_(this->_z);
-  else if (this->_activation == "ReLU")
-    relu_(this->_z, 0, channels, 0, this->_z.cols());
-  else
-    throw std::runtime_error("Unrecognized activation.");
+  
+  this->_activation->apply(this->_z);
+
+  //if (this->_activation == "Hardtanh")
+  //  hard_tanh_(this->_z);
+  //else if (this->_activation == "Tanh")
+  //  tanh_(this->_z);
+  //else if (this->_activation == "ReLU")
+  //  relu_(this->_z, 0, channels, 0, this->_z.cols());
+  //else
+  //  throw std::runtime_error("Unrecognized activation.");
+
   if (this->_gated)
   {
     sigmoid_(this->_z, channels, 2 * channels, 0, this->_z.cols());
@@ -172,7 +176,7 @@ void wavenet::_LayerArray::_rewind_buffers_()
 
 wavenet::_Head::_Head(const int input_size, const int num_layers, const int channels, const std::string activation)
 : _channels(channels)
-, _activation(activation)
+, _activation(activations::Activation::get_activation(activation))
 , _head(num_layers > 0 ? channels : input_size, 1, true)
 {
   assert(num_layers > 0);
@@ -220,12 +224,13 @@ void wavenet::_Head::set_num_frames_(const long num_frames)
 
 void wavenet::_Head::_apply_activation_(Eigen::MatrixXf& x)
 {
-  if (this->_activation == "Tanh")
-    tanh_(x);
-  else if (this->_activation == "ReLU")
-    relu_(x);
-  else
-    throw std::runtime_error("Unrecognized activation.");
+  this->_activation->apply(x);
+  //if (this->_activation == "Tanh")
+  //  tanh_(x);
+  //else if (this->_activation == "ReLU")
+  //  relu_(x);
+  //else
+  //  throw std::runtime_error("Unrecognized activation.");
 }
 
 // WaveNet ====================================================================
diff --git a/NAM/wavenet.h b/NAM/wavenet.h
index fc1daab..ee28e8f 100644
--- a/NAM/wavenet.h
+++ b/NAM/wavenet.h
@@ -23,7 +23,7 @@ class _Layer
 public:
   _Layer(const int condition_size, const int channels, const int kernel_size, const int dilation,
          const std::string activation, const bool gated)
-  : _activation(activation)
+  : _activation(activations::Activation::get_activation(activation))
   , _gated(gated)
   , _conv(channels, gated ? 2 * channels : channels, kernel_size, true, dilation)
   , _input_mixin(condition_size, gated ? 2 * channels : channels, false)
@@ -48,7 +48,7 @@ class _Layer
   // The internal state
   Eigen::MatrixXf _z;
 
-  const std::string _activation;
+  activations::Activation *_activation;
   const bool _gated;
 };
 
@@ -152,7 +152,7 @@ class _Head
   int _channels;
   std::vector<Conv1x1> _layers;
   Conv1x1 _head;
-  std::string _activation;
+  activations::Activation *_activation;
 
   // Stores the outputs of the convs *except* the last one, which goes in
   // The array `outputs` provided to .process_()

From 2aa7020fc1192837654b6a9c873e2b13159bcf52 Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Mon, 3 Apr 2023 13:35:00 -0700
Subject: [PATCH 2/9] Remove #pragma

---
 NAM/activations.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/NAM/activations.cpp b/NAM/activations.cpp
index 746cf2e..0c5cb59 100644
--- a/NAM/activations.cpp
+++ b/NAM/activations.cpp
@@ -1,5 +1,3 @@
-#pragma once
-
 #include "activations.h"
 
 std::unordered_map<std::string, activations::Activation*> activations::Activation::_activations = {

From 069679d450e10fd2f91af542dda629abf94b59d6 Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Mon, 3 Apr 2023 13:57:52 -0700
Subject: [PATCH 3/9] Fixed apply overrides

---
 NAM/activations.cpp |  2 +-
 NAM/activations.h   | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/NAM/activations.cpp b/NAM/activations.cpp
index 0c5cb59..3c64763 100644
--- a/NAM/activations.cpp
+++ b/NAM/activations.cpp
@@ -1,7 +1,7 @@
 #include "activations.h"
 
 std::unordered_map<std::string, activations::Activation*> activations::Activation::_activations = {
-  {"Tanh", new activations::ActivationHardTanh()},
+  {"Tanh", new activations::ActivationTanh()},
   {"Hardtanh", new activations::ActivationHardTanh()},
   {"Fasttanh", new activations::ActivationFastTanh()},
   {"ReLU", new activations::ActivationReLU()},
diff --git a/NAM/activations.h b/NAM/activations.h
index efa80a3..39c9f8f 100644
--- a/NAM/activations.h
+++ b/NAM/activations.h
@@ -53,7 +53,7 @@ Activation(){};
 class ActivationTanh : public Activation
 {
   public:
-    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    void apply(Eigen::Block<Eigen::MatrixXf> block) override
     {
       float* ptr = block.data();
 
@@ -70,7 +70,7 @@ class ActivationHardTanh : public Activation
 {
   public:
     ActivationHardTanh(){};
-    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    void apply(Eigen::Block<Eigen::MatrixXf> block) override
     {
       float* ptr = block.data();
 
@@ -87,7 +87,7 @@ class ActivationFastTanh : public Activation
 {
   public:
     ActivationFastTanh(){};
-    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    void apply(Eigen::Block<Eigen::MatrixXf> block) override
     {
       float* ptr = block.data();
 
@@ -104,7 +104,7 @@ class ActivationReLU : public Activation
 {
   public:
     ActivationReLU(){};
-    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    void apply(Eigen::Block<Eigen::MatrixXf> block) override
     {
       float* ptr = block.data();
 
@@ -121,7 +121,7 @@ class ActivationSigmoid : public Activation
 {
   public:
     ActivationSigmoid(){};
-    void Apply(Eigen::Block<Eigen::MatrixXf> block)
+    void apply(Eigen::Block<Eigen::MatrixXf> block) override
     {
       float* ptr = block.data();
 

From a3f3ce7028c0fa02fb040b381a2fc21bb3e0088f Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Mon, 3 Apr 2023 14:29:33 -0700
Subject: [PATCH 4/9] More switching to new activation implementaion

---
 NAM/activations.h | 14 ++++----
 NAM/convnet.cpp   | 10 ++----
 NAM/convnet.h     |  2 +-
 NAM/dsp.cpp       | 81 -----------------------------------------------
 NAM/dsp.h         | 21 ------------
 NAM/wavenet.cpp   | 15 ---------
 6 files changed, 11 insertions(+), 132 deletions(-)

diff --git a/NAM/activations.h b/NAM/activations.h
index 39c9f8f..7639094 100644
--- a/NAM/activations.h
+++ b/NAM/activations.h
@@ -36,8 +36,8 @@ class Activation
 {
 public:
 Activation(){};
-  virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.block(0, 0, matrix.rows(), matrix.cols())); }
-  virtual void apply(Eigen::Block<Eigen::MatrixXf> block) {}
+  virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.middleCols(0, matrix.cols())); }
+  virtual void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) {}
   static Activation* get_activation(const std::string name)
   { 
     if (_activations.find(name) == _activations.end())
@@ -53,7 +53,7 @@ Activation(){};
 class ActivationTanh : public Activation
 {
   public:
-    void apply(Eigen::Block<Eigen::MatrixXf> block) override
+    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
     {
       float* ptr = block.data();
 
@@ -70,7 +70,7 @@ class ActivationHardTanh : public Activation
 {
   public:
     ActivationHardTanh(){};
-    void apply(Eigen::Block<Eigen::MatrixXf> block) override
+    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
     {
       float* ptr = block.data();
 
@@ -87,7 +87,7 @@ class ActivationFastTanh : public Activation
 {
   public:
     ActivationFastTanh(){};
-    void apply(Eigen::Block<Eigen::MatrixXf> block) override
+    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
     {
       float* ptr = block.data();
 
@@ -104,7 +104,7 @@ class ActivationReLU : public Activation
 {
   public:
     ActivationReLU(){};
-    void apply(Eigen::Block<Eigen::MatrixXf> block) override
+    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
     {
       float* ptr = block.data();
 
@@ -121,7 +121,7 @@ class ActivationSigmoid : public Activation
 {
   public:
     ActivationSigmoid(){};
-    void apply(Eigen::Block<Eigen::MatrixXf> block) override
+    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
     {
       float* ptr = block.data();
 
diff --git a/NAM/convnet.cpp b/NAM/convnet.cpp
index 8c1aea2..3018473 100644
--- a/NAM/convnet.cpp
+++ b/NAM/convnet.cpp
@@ -57,7 +57,7 @@ void convnet::ConvNetBlock::set_params_(const int in_channels, const int out_cha
   this->conv.set_size_and_params_(in_channels, out_channels, 2, _dilation, !batchnorm, params);
   if (this->_batchnorm)
     this->batchnorm = BatchNorm(out_channels, params);
-  this->activation = activation;
+  this->activation = activations::Activation::get_activation(activation);
 }
 
 void convnet::ConvNetBlock::process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start,
@@ -67,12 +67,8 @@ void convnet::ConvNetBlock::process_(const Eigen::MatrixXf& input, Eigen::Matrix
   this->conv.process_(input, output, i_start, ncols, i_start);
   if (this->_batchnorm)
     this->batchnorm.process_(output, i_start, i_end);
-  if (this->activation == "Tanh")
-    tanh_(output, i_start, i_end);
-  else if (this->activation == "ReLU")
-    relu_(output, i_start, i_end);
-  else
-    throw std::runtime_error("Unrecognized activation");
+
+  this->activation->apply(output.middleCols(i_start, ncols));
 }
 
 long convnet::ConvNetBlock::get_out_channels() const
diff --git a/NAM/convnet.h b/NAM/convnet.h
index c4dcec8..b77e581 100644
--- a/NAM/convnet.h
+++ b/NAM/convnet.h
@@ -47,7 +47,7 @@ class ConvNetBlock
 private:
   BatchNorm batchnorm;
   bool _batchnorm;
-  std::string activation;
+  activations::Activation *activation;
 };
 
 class _Head
diff --git a/NAM/dsp.cpp b/NAM/dsp.cpp
index 0f51f6b..e28eade 100644
--- a/NAM/dsp.cpp
+++ b/NAM/dsp.cpp
@@ -206,23 +206,6 @@ void Linear::_process_core_()
 
 // NN modules =================================================================
 
-void relu_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end)
-{
-  for (long j = j_start; j < j_end; j++)
-    for (long i = 0; i < x.rows(); i++)
-      x(i, j) = x(i, j) < (float)0.0 ? (float)0.0 : x(i, j);
-}
-
-void relu_(Eigen::MatrixXf& x, const long j_start, const long j_end)
-{
-  relu_(x, 0, x.rows(), j_start, j_end);
-}
-
-void relu_(Eigen::MatrixXf& x)
-{
-  relu_(x, 0, x.rows(), 0, x.cols());
-}
-
 void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end)
 {
   for (long j = j_start; j < j_end; j++)
@@ -230,70 +213,6 @@ void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const lo
       x(i, j) = 1.0 / (1.0 + expf(-x(i, j)));
 }
 
-inline float fast_tanh_(const float x)
-{
-  const float ax = fabsf(x);
-  const float x2 = x * x;
-
-  return (x * (2.45550750702956f + 2.45550750702956f * ax + (0.893229853513558f + 0.821226666969744f * ax) * x2)
-          / (2.44506634652299f + (2.44506634652299f + x2) * fabsf(x + 0.814642734961073f * x * ax)));
-}
-
-inline float hard_tanh_(const float x)
-{
-  const float t = x < -1 ? -1 : x;
-  return t > 1 ? 1 : t;
-}
-
-void tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end)
-{
-  for (long j = j_start; j < j_end; j++)
-    for (long i = i_start; i < i_end; i++)
-      x(i, j) = tanh_impl_(x(i, j));
-}
-
-void tanh_(Eigen::MatrixXf& x, const long j_start, const long j_end)
-{
-  tanh_(x, 0, x.rows(), j_start, j_end);
-}
-
-void tanh_(Eigen::MatrixXf& x)
-{
-
-  float* ptr = x.data();
-
-  long size = x.rows() * x.cols();
-
-  for (long pos = 0; pos < size; pos++)
-  {
-    ptr[pos] = tanh_impl_(ptr[pos]);
-  }
-}
-
-void hard_tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end)
-{
-  for (long j = j_start; j < j_end; j++)
-    for (long i = i_start; i < i_end; i++)
-      x(i, j) = hard_tanh_(x(i, j));
-}
-
-void hard_tanh_(Eigen::MatrixXf& x, const long j_start, const long j_end)
-{
-  hard_tanh_(x, 0, x.rows(), j_start, j_end);
-}
-
-void hard_tanh_(Eigen::MatrixXf& x)
-{
-  float* ptr = x.data();
-
-  long size = x.rows() * x.cols();
-
-  for (long pos = 0; pos < size; pos++)
-  {
-    ptr[pos] = hard_tanh_(ptr[pos]);
-  }
-}
-
 void Conv1D::set_params_(std::vector<float>::iterator& params)
 {
   if (this->_weight.size() > 0)
diff --git a/NAM/dsp.h b/NAM/dsp.h
index 8123360..1464114 100644
--- a/NAM/dsp.h
+++ b/NAM/dsp.h
@@ -142,29 +142,8 @@ class Linear : public Buffer
 
 // Activations
 
-// In-place ReLU on (N,M) array
-void relu_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end);
-// Subset of the columns
-void relu_(Eigen::MatrixXf& x, const long j_start, const long j_end);
-void relu_(Eigen::MatrixXf& x);
-
 // In-place sigmoid
 void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end);
-void sigmoid_(Eigen::MatrixXf& x);
-
-// In-place Tanh on (N,M) array
-void tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end);
-// Subset of the columns
-void tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end);
-
-void tanh_(Eigen::MatrixXf& x);
-
-// In-place Hardtanh on (N,M) array
-void hard_tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end);
-// Subset of the columns
-void hard_tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end);
-
-void hard_tanh_(Eigen::MatrixXf& x);
 
 class Conv1D
 {
diff --git a/NAM/wavenet.cpp b/NAM/wavenet.cpp
index 13f7e31..b2d6821 100644
--- a/NAM/wavenet.cpp
+++ b/NAM/wavenet.cpp
@@ -32,15 +32,6 @@ void wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::Matrix
   
   this->_activation->apply(this->_z);
 
-  //if (this->_activation == "Hardtanh")
-  //  hard_tanh_(this->_z);
-  //else if (this->_activation == "Tanh")
-  //  tanh_(this->_z);
-  //else if (this->_activation == "ReLU")
-  //  relu_(this->_z, 0, channels, 0, this->_z.cols());
-  //else
-  //  throw std::runtime_error("Unrecognized activation.");
-
   if (this->_gated)
   {
     sigmoid_(this->_z, channels, 2 * channels, 0, this->_z.cols());
@@ -225,12 +216,6 @@ void wavenet::_Head::set_num_frames_(const long num_frames)
 void wavenet::_Head::_apply_activation_(Eigen::MatrixXf& x)
 {
   this->_activation->apply(x);
-  //if (this->_activation == "Tanh")
-  //  tanh_(x);
-  //else if (this->_activation == "ReLU")
-  //  relu_(x);
-  //else
-  //  throw std::runtime_error("Unrecognized activation.");
 }
 
 // WaveNet ====================================================================

From 1cf8a4ff199b520dccf15487e2e632eec86cb3aa Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Mon, 3 Apr 2023 14:47:31 -0700
Subject: [PATCH 5/9] Move static function implementations into .cpp

---
 NAM/activations.cpp | 27 +++++++++++++++++++++++++++
 NAM/activations.h   |  9 +++------
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/NAM/activations.cpp b/NAM/activations.cpp
index 3c64763..3366be5 100644
--- a/NAM/activations.cpp
+++ b/NAM/activations.cpp
@@ -6,3 +6,30 @@ std::unordered_map<std::string, activations::Activation*> activations::Activatio
   {"Fasttanh", new activations::ActivationFastTanh()},
   {"ReLU", new activations::ActivationReLU()},
   {"Sigmoid", new activations::ActivationSigmoid()}};
+
+activations::Activation* tanh_bak = nullptr;
+
+activations::Activation* activations::Activation::get_activation(const std::string name)
+{
+  if (_activations.find(name) == _activations.end())
+    return nullptr;
+
+  return _activations[name];
+}
+
+void activations::Activation::enable_fast_tanh()
+{
+  if (_activations["Tanh"] != _activations["Fasttanh"])
+  {
+    tanh_bak = _activations["Tanh"];
+    _activations["Tanh"] = _activations["Fasttanh"];
+  }
+}
+
+void activations::Activation::disable_fast_tanh()
+{
+  if (_activations["Tanh"] == _activations["Fasttanh"])
+  {
+    _activations["Tanh"] = tanh_bak;
+  }
+}
diff --git a/NAM/activations.h b/NAM/activations.h
index 7639094..05aca9c 100644
--- a/NAM/activations.h
+++ b/NAM/activations.h
@@ -38,13 +38,10 @@ class Activation
 Activation(){};
   virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.middleCols(0, matrix.cols())); }
   virtual void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) {}
-  static Activation* get_activation(const std::string name)
-  { 
-    if (_activations.find(name) == _activations.end())
-      return nullptr;
 
-    return _activations[name];
-  }
+  static Activation* get_activation(const std::string name);
+  static void enable_fast_tanh();
+  static void disable_fast_tanh();
 
   protected:
     static std::unordered_map<std::string, Activation *> _activations;

From 74b0a4977917b67e54abb000223296e9e666595f Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Mon, 3 Apr 2023 16:32:15 -0700
Subject: [PATCH 6/9] Simplify Activation apply

---
 NAM/activations.h | 55 ++++++++++++++++++++---------------------------
 1 file changed, 23 insertions(+), 32 deletions(-)

diff --git a/NAM/activations.h b/NAM/activations.h
index 05aca9c..4b65b1f 100644
--- a/NAM/activations.h
+++ b/NAM/activations.h
@@ -36,8 +36,19 @@ class Activation
 {
 public:
 Activation(){};
-  virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.middleCols(0, matrix.cols())); }
-  virtual void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) {}
+  virtual void apply(Eigen::MatrixXf& matrix)
+  {
+    apply(matrix.middleCols(0, matrix.cols()).data(), matrix.rows() * matrix.cols());
+  }
+  virtual void apply(Eigen::Block<Eigen::MatrixXf> block)
+  {
+    apply(block.data(), block.rows() * block.cols());
+  }
+  virtual void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block)
+  {
+    apply(block.data(), block.rows() * block.cols());
+  }
+  virtual void apply(float* data, long size) {}
 
   static Activation* get_activation(const std::string name);
   static void enable_fast_tanh();
@@ -50,15 +61,11 @@ Activation(){};
 class ActivationTanh : public Activation
 {
   public:
-    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
+    void apply(float *data, long size) override
     {
-      float* ptr = block.data();
-
-      long size = block.rows() * block.cols();
-
       for (long pos = 0; pos < size; pos++)
       {
-        ptr[pos] = std::tanh(ptr[pos]);
+        data[pos] = std::tanh(data[pos]);
       }
     }
 };
@@ -67,15 +74,11 @@ class ActivationHardTanh : public Activation
 {
   public:
     ActivationHardTanh(){};
-    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
+    void apply(float* data, long size) override
     {
-      float* ptr = block.data();
-
-      long size = block.rows() * block.cols();
-
       for (long pos = 0; pos < size; pos++)
       {
-        ptr[pos] = hard_tanh(ptr[pos]);
+        data[pos] = hard_tanh(data[pos]);
       }
     }
 };
@@ -84,15 +87,11 @@ class ActivationFastTanh : public Activation
 {
   public:
     ActivationFastTanh(){};
-    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
+    void apply(float* data, long size) override
     {
-      float* ptr = block.data();
-
-      long size = block.rows() * block.cols();
-
       for (long pos = 0; pos < size; pos++)
       {
-        ptr[pos] = fast_tanh(ptr[pos]);
+        data[pos] = fast_tanh(data[pos]);
       }
     }
 };
@@ -101,15 +100,11 @@ class ActivationReLU : public Activation
 {
   public:
     ActivationReLU(){};
-    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
+    void apply(float* data, long size) override
     {
-      float* ptr = block.data();
-
-      long size = block.rows() * block.cols();
-
       for (long pos = 0; pos < size; pos++)
       {
-        ptr[pos] = relu(ptr[pos]);
+        data[pos] = relu(data[pos]);
       }
     }
 };
@@ -118,15 +113,11 @@ class ActivationSigmoid : public Activation
 {
   public:
     ActivationSigmoid(){};
-    void apply(Eigen::Block<Eigen::MatrixXf, -1, -1, true> block) override
+    void apply(float* data, long size) override
     {
-      float* ptr = block.data();
-
-      long size = block.rows() * block.cols();
-
       for (long pos = 0; pos < size; pos++)
       {
-        ptr[pos] = sigmoid(ptr[pos]);
+        data[pos] = sigmoid(data[pos]);
       }
     }
 };

From 26a725db8e8ab3e1ec1f505efc95b3ef8a5dfe0a Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Mon, 3 Apr 2023 16:36:34 -0700
Subject: [PATCH 7/9] Use new activation code for WaveNet Sigmoid

---
 NAM/dsp.cpp     | 7 -------
 NAM/dsp.h       | 5 -----
 NAM/wavenet.cpp | 3 ++-
 3 files changed, 2 insertions(+), 13 deletions(-)

diff --git a/NAM/dsp.cpp b/NAM/dsp.cpp
index e28eade..3e97665 100644
--- a/NAM/dsp.cpp
+++ b/NAM/dsp.cpp
@@ -206,13 +206,6 @@ void Linear::_process_core_()
 
 // NN modules =================================================================
 
-void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end)
-{
-  for (long j = j_start; j < j_end; j++)
-    for (long i = i_start; i < i_end; i++)
-      x(i, j) = 1.0 / (1.0 + expf(-x(i, j)));
-}
-
 void Conv1D::set_params_(std::vector<float>::iterator& params)
 {
   if (this->_weight.size() > 0)
diff --git a/NAM/dsp.h b/NAM/dsp.h
index 1464114..f15989a 100644
--- a/NAM/dsp.h
+++ b/NAM/dsp.h
@@ -140,11 +140,6 @@ class Linear : public Buffer
 
 // NN modules =================================================================
 
-// Activations
-
-// In-place sigmoid
-void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end);
-
 class Conv1D
 {
 public:
diff --git a/NAM/wavenet.cpp b/NAM/wavenet.cpp
index b2d6821..231489b 100644
--- a/NAM/wavenet.cpp
+++ b/NAM/wavenet.cpp
@@ -34,7 +34,8 @@ void wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::Matrix
 
   if (this->_gated)
   {
-    sigmoid_(this->_z, channels, 2 * channels, 0, this->_z.cols());
+    activations::Activation::get_activation("Sigmoid")->apply(this->_z.block(channels, 2 * channels, 0, this->_z.cols()));
+
     this->_z.topRows(channels).array() *= this->_z.bottomRows(channels).array();
     // this->_z.topRows(channels) = this->_z.topRows(channels).cwiseProduct(
     //   this->_z.bottomRows(channels)

From b41d172910024ee7eb179731b50615424af182d6 Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Mon, 3 Apr 2023 17:01:52 -0700
Subject: [PATCH 8/9] Fixed WaveNet sigmoid

---
 NAM/wavenet.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/NAM/wavenet.cpp b/NAM/wavenet.cpp
index 231489b..b844e1d 100644
--- a/NAM/wavenet.cpp
+++ b/NAM/wavenet.cpp
@@ -34,7 +34,7 @@ void wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::Matrix
 
   if (this->_gated)
   {
-    activations::Activation::get_activation("Sigmoid")->apply(this->_z.block(channels, 2 * channels, 0, this->_z.cols()));
+    activations::Activation::get_activation("Sigmoid")->apply(this->_z.block(channels, 0, channels, this->_z.cols()));
 
     this->_z.topRows(channels).array() *= this->_z.bottomRows(channels).array();
     // this->_z.topRows(channels) = this->_z.topRows(channels).cwiseProduct(

From 482b69aa547b72aedf0ac2569218111becb3d831 Mon Sep 17 00:00:00 2001
From: Mike Oliphant <oliphant@nostatic.org>
Date: Tue, 4 Apr 2023 07:50:06 -0700
Subject: [PATCH 9/9] No need for middleCols() in full matrix apply anymore

---
 NAM/activations.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/NAM/activations.h b/NAM/activations.h
index 4b65b1f..fabcc05 100644
--- a/NAM/activations.h
+++ b/NAM/activations.h
@@ -38,7 +38,7 @@ class Activation
 Activation(){};
   virtual void apply(Eigen::MatrixXf& matrix)
   {
-    apply(matrix.middleCols(0, matrix.cols()).data(), matrix.rows() * matrix.cols());
+    apply(matrix.data(), matrix.rows() * matrix.cols());
   }
   virtual void apply(Eigen::Block<Eigen::MatrixXf> block)
   {
