Add fast tanh and fast sigmoid to LSTM

NAM/activations.cpp

@@ -6,6 +6,8 @@ activations::ActivationHardTanh _HARD_TANH = activations::ActivationHardTanh();
 activations::ActivationReLU _RELU = activations::ActivationReLU();
 activations::ActivationSigmoid _SIGMOID = activations::ActivationSigmoid();
 
+bool activations::Activation::using_fast_tanh = false;
+
 std::unordered_map<std::string, activations::Activation*> activations::Activation::_activations =
   {{"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, {"ReLU", &_RELU}, {"Sigmoid", &_SIGMOID}};
 
@@ -21,6 +23,8 @@ activations::Activation* activations::Activation::get_activation(const std::stri
 
 void activations::Activation::enable_fast_tanh()
 {
+  activations::Activation::using_fast_tanh = true;
+
   if (_activations["Tanh"] != _activations["Fasttanh"])
   {
     tanh_bak = _activations["Tanh"];
@@ -30,6 +34,8 @@ void activations::Activation::enable_fast_tanh()
 
 void activations::Activation::disable_fast_tanh()
 {
+  activations::Activation::using_fast_tanh = false;
+
   if (_activations["Tanh"] == _activations["Fasttanh"])
   {
     _activations["Tanh"] = tanh_bak;

NAM/activations.h

@@ -32,6 +32,12 @@ inline float fast_tanh(const float x)
           / (2.44506634652299f + (2.44506634652299f + x2) * fabsf(x + 0.814642734961073f * x * ax)));
 }
 
+inline float fast_sigmoid(const float x)
+{
+  return 0.5f * (fast_tanh(x * 0.5f) + 1.0f);
+}
+
+
 class Activation
 {
 public:
@@ -48,6 +54,7 @@ class Activation
   static Activation* get_activation(const std::string name);
   static void enable_fast_tanh();
   static void disable_fast_tanh();
+  static bool using_fast_tanh;
 
 protected:
   static std::unordered_map<std::string, Activation*> _activations;

NAM/lstm.cpp

@@ -39,12 +39,28 @@ void lstm::LSTMCell::process_(const Eigen::VectorXf& x)
   const long f_offset = hidden_size;
   const long g_offset = 2 * hidden_size;
   const long o_offset = 3 * hidden_size;
-  for (auto i = 0; i < hidden_size; i++)
-    this->_c[i] = activations::sigmoid(this->_ifgo[i + f_offset]) * this->_c[i]
-                  + activations::sigmoid(this->_ifgo[i + i_offset]) * tanhf(this->_ifgo[i + g_offset]);
   const long h_offset = input_size;
-  for (int i = 0; i < hidden_size; i++)
-    this->_xh[i + h_offset] = activations::sigmoid(this->_ifgo[i + o_offset]) * tanhf(this->_c[i]);
+
+  if (activations::Activation::using_fast_tanh)
+  {
+    for (auto i = 0; i < hidden_size; i++)
+      this->_c[i] =
+        activations::fast_sigmoid(this->_ifgo[i + f_offset]) * this->_c[i]
+        + activations::fast_sigmoid(this->_ifgo[i + i_offset]) * activations::fast_tanh(this->_ifgo[i + g_offset]);
+
+    for (int i = 0; i < hidden_size; i++)
+      this->_xh[i + h_offset] =
+        activations::fast_sigmoid(this->_ifgo[i + o_offset]) * activations::fast_tanh(this->_c[i]);
+  }
+  else
+  {
+    for (auto i = 0; i < hidden_size; i++)
+      this->_c[i] = activations::sigmoid(this->_ifgo[i + f_offset]) * this->_c[i]
+                    + activations::sigmoid(this->_ifgo[i + i_offset]) * tanhf(this->_ifgo[i + g_offset]);
+
+    for (int i = 0; i < hidden_size; i++)
+      this->_xh[i + h_offset] = activations::sigmoid(this->_ifgo[i + o_offset]) * tanhf(this->_c[i]);
+  }
 }
 
 lstm::LSTM::LSTM(const int num_layers, const int input_size, const int hidden_size, std::vector<float>& params,