From 14cb71e0b54f93c34796bda77372f5b2d8496f80 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 3 Apr 2023 13:34:12 -0700 Subject: [PATCH 1/9] Begin to refactor activation function handling --- NAM/activations.cpp | 10 ++++ NAM/activations.h | 130 ++++++++++++++++++++++++++++++++++++++++++-- NAM/dsp.h | 2 + NAM/lstm.cpp | 1 - NAM/wavenet.cpp | 35 +++++++----- NAM/wavenet.h | 6 +- 6 files changed, 161 insertions(+), 23 deletions(-) create mode 100644 NAM/activations.cpp diff --git a/NAM/activations.cpp b/NAM/activations.cpp new file mode 100644 index 0000000..746cf2e --- /dev/null +++ b/NAM/activations.cpp @@ -0,0 +1,10 @@ +#pragma once + +#include "activations.h" + +std::unordered_map activations::Activation::_activations = { + {"Tanh", new activations::ActivationHardTanh()}, + {"Hardtanh", new activations::ActivationHardTanh()}, + {"Fasttanh", new activations::ActivationFastTanh()}, + {"ReLU", new activations::ActivationReLU()}, + {"Sigmoid", new activations::ActivationSigmoid()}}; diff --git a/NAM/activations.h b/NAM/activations.h index 52b1334..efa80a3 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -1,15 +1,137 @@ #pragma once +#include #include // expf +#include +#include namespace activations { -float relu(float x) +inline float relu(float x) { - return x > 0.0f ? x : 0.0f; +return x > 0.0f ? x : 0.0f; }; -float sigmoid(float x) + +inline float sigmoid(float x) { - return 1.0f / (1.0f + expf(-x)); +return 1.0f / (1.0f + expf(-x)); }; + +inline float hard_tanh(float x) +{ +const float t = x < -1 ? -1 : x; +return t > 1 ? 1 : t; +} + +inline float fast_tanh(const float x) +{ +const float ax = fabsf(x); +const float x2 = x * x; + +return (x * (2.45550750702956f + 2.45550750702956f * ax + (0.893229853513558f + 0.821226666969744f * ax) * x2) + / (2.44506634652299f + (2.44506634652299f + x2) * fabsf(x + 0.814642734961073f * x * ax))); +} + +class Activation +{ +public: +Activation(){}; + virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.block(0, 0, matrix.rows(), matrix.cols())); } + virtual void apply(Eigen::Block block) {} + static Activation* get_activation(const std::string name) + { + if (_activations.find(name) == _activations.end()) + return nullptr; + + return _activations[name]; + } + + protected: + static std::unordered_map _activations; +}; + +class ActivationTanh : public Activation +{ + public: + void Apply(Eigen::Block block) + { + float* ptr = block.data(); + + long size = block.rows() * block.cols(); + + for (long pos = 0; pos < size; pos++) + { + ptr[pos] = std::tanh(ptr[pos]); + } + } +}; + +class ActivationHardTanh : public Activation +{ + public: + ActivationHardTanh(){}; + void Apply(Eigen::Block block) + { + float* ptr = block.data(); + + long size = block.rows() * block.cols(); + + for (long pos = 0; pos < size; pos++) + { + ptr[pos] = hard_tanh(ptr[pos]); + } + } +}; + +class ActivationFastTanh : public Activation +{ + public: + ActivationFastTanh(){}; + void Apply(Eigen::Block block) + { + float* ptr = block.data(); + + long size = block.rows() * block.cols(); + + for (long pos = 0; pos < size; pos++) + { + ptr[pos] = fast_tanh(ptr[pos]); + } + } +}; + +class ActivationReLU : public Activation +{ + public: + ActivationReLU(){}; + void Apply(Eigen::Block block) + { + float* ptr = block.data(); + + long size = block.rows() * block.cols(); + + for (long pos = 0; pos < size; pos++) + { + ptr[pos] = relu(ptr[pos]); + } + } +}; + +class ActivationSigmoid : public Activation +{ + public: + ActivationSigmoid(){}; + void Apply(Eigen::Block block) + { + float* ptr = block.data(); + + long size = block.rows() * block.cols(); + + for (long pos = 0; pos < size; pos++) + { + ptr[pos] = sigmoid(ptr[pos]); + } + } +}; + }; // namespace activations \ No newline at end of file diff --git a/NAM/dsp.h b/NAM/dsp.h index c1f5bff..8123360 100644 --- a/NAM/dsp.h +++ b/NAM/dsp.h @@ -9,6 +9,8 @@ #include +#include "activations.h" + enum EArchitectures { kLinear = 0, diff --git a/NAM/lstm.cpp b/NAM/lstm.cpp index bffd18d..39e3acf 100644 --- a/NAM/lstm.cpp +++ b/NAM/lstm.cpp @@ -2,7 +2,6 @@ #include #include -#include "activations.h" #include "lstm.h" lstm::LSTMCell::LSTMCell(const int input_size, const int hidden_size, std::vector::iterator& params) diff --git a/NAM/wavenet.cpp b/NAM/wavenet.cpp index 2dbf491..13f7e31 100644 --- a/NAM/wavenet.cpp +++ b/NAM/wavenet.cpp @@ -29,14 +29,18 @@ void wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::Matrix this->_conv.process_(input, this->_z, i_start, ncols, 0); // Mix-in condition this->_z += this->_input_mixin.process(condition); - if (this->_activation == "Hardtanh") - hard_tanh_(this->_z); - else if (this->_activation == "Tanh") - tanh_(this->_z); - else if (this->_activation == "ReLU") - relu_(this->_z, 0, channels, 0, this->_z.cols()); - else - throw std::runtime_error("Unrecognized activation."); + + this->_activation->apply(this->_z); + + //if (this->_activation == "Hardtanh") + // hard_tanh_(this->_z); + //else if (this->_activation == "Tanh") + // tanh_(this->_z); + //else if (this->_activation == "ReLU") + // relu_(this->_z, 0, channels, 0, this->_z.cols()); + //else + // throw std::runtime_error("Unrecognized activation."); + if (this->_gated) { sigmoid_(this->_z, channels, 2 * channels, 0, this->_z.cols()); @@ -172,7 +176,7 @@ void wavenet::_LayerArray::_rewind_buffers_() wavenet::_Head::_Head(const int input_size, const int num_layers, const int channels, const std::string activation) : _channels(channels) -, _activation(activation) +, _activation(activations::Activation::get_activation(activation)) , _head(num_layers > 0 ? channels : input_size, 1, true) { assert(num_layers > 0); @@ -220,12 +224,13 @@ void wavenet::_Head::set_num_frames_(const long num_frames) void wavenet::_Head::_apply_activation_(Eigen::MatrixXf& x) { - if (this->_activation == "Tanh") - tanh_(x); - else if (this->_activation == "ReLU") - relu_(x); - else - throw std::runtime_error("Unrecognized activation."); + this->_activation->apply(x); + //if (this->_activation == "Tanh") + // tanh_(x); + //else if (this->_activation == "ReLU") + // relu_(x); + //else + // throw std::runtime_error("Unrecognized activation."); } // WaveNet ==================================================================== diff --git a/NAM/wavenet.h b/NAM/wavenet.h index fc1daab..ee28e8f 100644 --- a/NAM/wavenet.h +++ b/NAM/wavenet.h @@ -23,7 +23,7 @@ class _Layer public: _Layer(const int condition_size, const int channels, const int kernel_size, const int dilation, const std::string activation, const bool gated) - : _activation(activation) + : _activation(activations::Activation::get_activation(activation)) , _gated(gated) , _conv(channels, gated ? 2 * channels : channels, kernel_size, true, dilation) , _input_mixin(condition_size, gated ? 2 * channels : channels, false) @@ -48,7 +48,7 @@ class _Layer // The internal state Eigen::MatrixXf _z; - const std::string _activation; + activations::Activation *_activation; const bool _gated; }; @@ -152,7 +152,7 @@ class _Head int _channels; std::vector _layers; Conv1x1 _head; - std::string _activation; + activations::Activation *_activation; // Stores the outputs of the convs *except* the last one, which goes in // The array `outputs` provided to .process_() From 2aa7020fc1192837654b6a9c873e2b13159bcf52 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 3 Apr 2023 13:35:00 -0700 Subject: [PATCH 2/9] Remove #pragma --- NAM/activations.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index 746cf2e..0c5cb59 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -1,5 +1,3 @@ -#pragma once - #include "activations.h" std::unordered_map activations::Activation::_activations = { From 069679d450e10fd2f91af542dda629abf94b59d6 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 3 Apr 2023 13:57:52 -0700 Subject: [PATCH 3/9] Fixed apply overrides --- NAM/activations.cpp | 2 +- NAM/activations.h | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index 0c5cb59..3c64763 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -1,7 +1,7 @@ #include "activations.h" std::unordered_map activations::Activation::_activations = { - {"Tanh", new activations::ActivationHardTanh()}, + {"Tanh", new activations::ActivationTanh()}, {"Hardtanh", new activations::ActivationHardTanh()}, {"Fasttanh", new activations::ActivationFastTanh()}, {"ReLU", new activations::ActivationReLU()}, diff --git a/NAM/activations.h b/NAM/activations.h index efa80a3..39c9f8f 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -53,7 +53,7 @@ Activation(){}; class ActivationTanh : public Activation { public: - void Apply(Eigen::Block block) + void apply(Eigen::Block block) override { float* ptr = block.data(); @@ -70,7 +70,7 @@ class ActivationHardTanh : public Activation { public: ActivationHardTanh(){}; - void Apply(Eigen::Block block) + void apply(Eigen::Block block) override { float* ptr = block.data(); @@ -87,7 +87,7 @@ class ActivationFastTanh : public Activation { public: ActivationFastTanh(){}; - void Apply(Eigen::Block block) + void apply(Eigen::Block block) override { float* ptr = block.data(); @@ -104,7 +104,7 @@ class ActivationReLU : public Activation { public: ActivationReLU(){}; - void Apply(Eigen::Block block) + void apply(Eigen::Block block) override { float* ptr = block.data(); @@ -121,7 +121,7 @@ class ActivationSigmoid : public Activation { public: ActivationSigmoid(){}; - void Apply(Eigen::Block block) + void apply(Eigen::Block block) override { float* ptr = block.data(); From a3f3ce7028c0fa02fb040b381a2fc21bb3e0088f Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 3 Apr 2023 14:29:33 -0700 Subject: [PATCH 4/9] More switching to new activation implementaion --- NAM/activations.h | 14 ++++---- NAM/convnet.cpp | 10 ++---- NAM/convnet.h | 2 +- NAM/dsp.cpp | 81 ----------------------------------------------- NAM/dsp.h | 21 ------------ NAM/wavenet.cpp | 15 --------- 6 files changed, 11 insertions(+), 132 deletions(-) diff --git a/NAM/activations.h b/NAM/activations.h index 39c9f8f..7639094 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -36,8 +36,8 @@ class Activation { public: Activation(){}; - virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.block(0, 0, matrix.rows(), matrix.cols())); } - virtual void apply(Eigen::Block block) {} + virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.middleCols(0, matrix.cols())); } + virtual void apply(Eigen::Block block) {} static Activation* get_activation(const std::string name) { if (_activations.find(name) == _activations.end()) @@ -53,7 +53,7 @@ Activation(){}; class ActivationTanh : public Activation { public: - void apply(Eigen::Block block) override + void apply(Eigen::Block block) override { float* ptr = block.data(); @@ -70,7 +70,7 @@ class ActivationHardTanh : public Activation { public: ActivationHardTanh(){}; - void apply(Eigen::Block block) override + void apply(Eigen::Block block) override { float* ptr = block.data(); @@ -87,7 +87,7 @@ class ActivationFastTanh : public Activation { public: ActivationFastTanh(){}; - void apply(Eigen::Block block) override + void apply(Eigen::Block block) override { float* ptr = block.data(); @@ -104,7 +104,7 @@ class ActivationReLU : public Activation { public: ActivationReLU(){}; - void apply(Eigen::Block block) override + void apply(Eigen::Block block) override { float* ptr = block.data(); @@ -121,7 +121,7 @@ class ActivationSigmoid : public Activation { public: ActivationSigmoid(){}; - void apply(Eigen::Block block) override + void apply(Eigen::Block block) override { float* ptr = block.data(); diff --git a/NAM/convnet.cpp b/NAM/convnet.cpp index 8c1aea2..3018473 100644 --- a/NAM/convnet.cpp +++ b/NAM/convnet.cpp @@ -57,7 +57,7 @@ void convnet::ConvNetBlock::set_params_(const int in_channels, const int out_cha this->conv.set_size_and_params_(in_channels, out_channels, 2, _dilation, !batchnorm, params); if (this->_batchnorm) this->batchnorm = BatchNorm(out_channels, params); - this->activation = activation; + this->activation = activations::Activation::get_activation(activation); } void convnet::ConvNetBlock::process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, @@ -67,12 +67,8 @@ void convnet::ConvNetBlock::process_(const Eigen::MatrixXf& input, Eigen::Matrix this->conv.process_(input, output, i_start, ncols, i_start); if (this->_batchnorm) this->batchnorm.process_(output, i_start, i_end); - if (this->activation == "Tanh") - tanh_(output, i_start, i_end); - else if (this->activation == "ReLU") - relu_(output, i_start, i_end); - else - throw std::runtime_error("Unrecognized activation"); + + this->activation->apply(output.middleCols(i_start, ncols)); } long convnet::ConvNetBlock::get_out_channels() const diff --git a/NAM/convnet.h b/NAM/convnet.h index c4dcec8..b77e581 100644 --- a/NAM/convnet.h +++ b/NAM/convnet.h @@ -47,7 +47,7 @@ class ConvNetBlock private: BatchNorm batchnorm; bool _batchnorm; - std::string activation; + activations::Activation *activation; }; class _Head diff --git a/NAM/dsp.cpp b/NAM/dsp.cpp index 0f51f6b..e28eade 100644 --- a/NAM/dsp.cpp +++ b/NAM/dsp.cpp @@ -206,23 +206,6 @@ void Linear::_process_core_() // NN modules ================================================================= -void relu_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end) -{ - for (long j = j_start; j < j_end; j++) - for (long i = 0; i < x.rows(); i++) - x(i, j) = x(i, j) < (float)0.0 ? (float)0.0 : x(i, j); -} - -void relu_(Eigen::MatrixXf& x, const long j_start, const long j_end) -{ - relu_(x, 0, x.rows(), j_start, j_end); -} - -void relu_(Eigen::MatrixXf& x) -{ - relu_(x, 0, x.rows(), 0, x.cols()); -} - void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end) { for (long j = j_start; j < j_end; j++) @@ -230,70 +213,6 @@ void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const lo x(i, j) = 1.0 / (1.0 + expf(-x(i, j))); } -inline float fast_tanh_(const float x) -{ - const float ax = fabsf(x); - const float x2 = x * x; - - return (x * (2.45550750702956f + 2.45550750702956f * ax + (0.893229853513558f + 0.821226666969744f * ax) * x2) - / (2.44506634652299f + (2.44506634652299f + x2) * fabsf(x + 0.814642734961073f * x * ax))); -} - -inline float hard_tanh_(const float x) -{ - const float t = x < -1 ? -1 : x; - return t > 1 ? 1 : t; -} - -void tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end) -{ - for (long j = j_start; j < j_end; j++) - for (long i = i_start; i < i_end; i++) - x(i, j) = tanh_impl_(x(i, j)); -} - -void tanh_(Eigen::MatrixXf& x, const long j_start, const long j_end) -{ - tanh_(x, 0, x.rows(), j_start, j_end); -} - -void tanh_(Eigen::MatrixXf& x) -{ - - float* ptr = x.data(); - - long size = x.rows() * x.cols(); - - for (long pos = 0; pos < size; pos++) - { - ptr[pos] = tanh_impl_(ptr[pos]); - } -} - -void hard_tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end) -{ - for (long j = j_start; j < j_end; j++) - for (long i = i_start; i < i_end; i++) - x(i, j) = hard_tanh_(x(i, j)); -} - -void hard_tanh_(Eigen::MatrixXf& x, const long j_start, const long j_end) -{ - hard_tanh_(x, 0, x.rows(), j_start, j_end); -} - -void hard_tanh_(Eigen::MatrixXf& x) -{ - float* ptr = x.data(); - - long size = x.rows() * x.cols(); - - for (long pos = 0; pos < size; pos++) - { - ptr[pos] = hard_tanh_(ptr[pos]); - } -} - void Conv1D::set_params_(std::vector::iterator& params) { if (this->_weight.size() > 0) diff --git a/NAM/dsp.h b/NAM/dsp.h index 8123360..1464114 100644 --- a/NAM/dsp.h +++ b/NAM/dsp.h @@ -142,29 +142,8 @@ class Linear : public Buffer // Activations -// In-place ReLU on (N,M) array -void relu_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end); -// Subset of the columns -void relu_(Eigen::MatrixXf& x, const long j_start, const long j_end); -void relu_(Eigen::MatrixXf& x); - // In-place sigmoid void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end); -void sigmoid_(Eigen::MatrixXf& x); - -// In-place Tanh on (N,M) array -void tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end); -// Subset of the columns -void tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end); - -void tanh_(Eigen::MatrixXf& x); - -// In-place Hardtanh on (N,M) array -void hard_tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end); -// Subset of the columns -void hard_tanh_(Eigen::MatrixXf& x, const long i_start, const long i_end); - -void hard_tanh_(Eigen::MatrixXf& x); class Conv1D { diff --git a/NAM/wavenet.cpp b/NAM/wavenet.cpp index 13f7e31..b2d6821 100644 --- a/NAM/wavenet.cpp +++ b/NAM/wavenet.cpp @@ -32,15 +32,6 @@ void wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::Matrix this->_activation->apply(this->_z); - //if (this->_activation == "Hardtanh") - // hard_tanh_(this->_z); - //else if (this->_activation == "Tanh") - // tanh_(this->_z); - //else if (this->_activation == "ReLU") - // relu_(this->_z, 0, channels, 0, this->_z.cols()); - //else - // throw std::runtime_error("Unrecognized activation."); - if (this->_gated) { sigmoid_(this->_z, channels, 2 * channels, 0, this->_z.cols()); @@ -225,12 +216,6 @@ void wavenet::_Head::set_num_frames_(const long num_frames) void wavenet::_Head::_apply_activation_(Eigen::MatrixXf& x) { this->_activation->apply(x); - //if (this->_activation == "Tanh") - // tanh_(x); - //else if (this->_activation == "ReLU") - // relu_(x); - //else - // throw std::runtime_error("Unrecognized activation."); } // WaveNet ==================================================================== From 1cf8a4ff199b520dccf15487e2e632eec86cb3aa Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 3 Apr 2023 14:47:31 -0700 Subject: [PATCH 5/9] Move static function implementations into .cpp --- NAM/activations.cpp | 27 +++++++++++++++++++++++++++ NAM/activations.h | 9 +++------ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index 3c64763..3366be5 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -6,3 +6,30 @@ std::unordered_map activations::Activatio {"Fasttanh", new activations::ActivationFastTanh()}, {"ReLU", new activations::ActivationReLU()}, {"Sigmoid", new activations::ActivationSigmoid()}}; + +activations::Activation* tanh_bak = nullptr; + +activations::Activation* activations::Activation::get_activation(const std::string name) +{ + if (_activations.find(name) == _activations.end()) + return nullptr; + + return _activations[name]; +} + +void activations::Activation::enable_fast_tanh() +{ + if (_activations["Tanh"] != _activations["Fasttanh"]) + { + tanh_bak = _activations["Tanh"]; + _activations["Tanh"] = _activations["Fasttanh"]; + } +} + +void activations::Activation::disable_fast_tanh() +{ + if (_activations["Tanh"] == _activations["Fasttanh"]) + { + _activations["Tanh"] = tanh_bak; + } +} diff --git a/NAM/activations.h b/NAM/activations.h index 7639094..05aca9c 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -38,13 +38,10 @@ class Activation Activation(){}; virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.middleCols(0, matrix.cols())); } virtual void apply(Eigen::Block block) {} - static Activation* get_activation(const std::string name) - { - if (_activations.find(name) == _activations.end()) - return nullptr; - return _activations[name]; - } + static Activation* get_activation(const std::string name); + static void enable_fast_tanh(); + static void disable_fast_tanh(); protected: static std::unordered_map _activations; From 74b0a4977917b67e54abb000223296e9e666595f Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 3 Apr 2023 16:32:15 -0700 Subject: [PATCH 6/9] Simplify Activation apply --- NAM/activations.h | 55 ++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/NAM/activations.h b/NAM/activations.h index 05aca9c..4b65b1f 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -36,8 +36,19 @@ class Activation { public: Activation(){}; - virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.middleCols(0, matrix.cols())); } - virtual void apply(Eigen::Block block) {} + virtual void apply(Eigen::MatrixXf& matrix) + { + apply(matrix.middleCols(0, matrix.cols()).data(), matrix.rows() * matrix.cols()); + } + virtual void apply(Eigen::Block block) + { + apply(block.data(), block.rows() * block.cols()); + } + virtual void apply(Eigen::Block block) + { + apply(block.data(), block.rows() * block.cols()); + } + virtual void apply(float* data, long size) {} static Activation* get_activation(const std::string name); static void enable_fast_tanh(); @@ -50,15 +61,11 @@ Activation(){}; class ActivationTanh : public Activation { public: - void apply(Eigen::Block block) override + void apply(float *data, long size) override { - float* ptr = block.data(); - - long size = block.rows() * block.cols(); - for (long pos = 0; pos < size; pos++) { - ptr[pos] = std::tanh(ptr[pos]); + data[pos] = std::tanh(data[pos]); } } }; @@ -67,15 +74,11 @@ class ActivationHardTanh : public Activation { public: ActivationHardTanh(){}; - void apply(Eigen::Block block) override + void apply(float* data, long size) override { - float* ptr = block.data(); - - long size = block.rows() * block.cols(); - for (long pos = 0; pos < size; pos++) { - ptr[pos] = hard_tanh(ptr[pos]); + data[pos] = hard_tanh(data[pos]); } } }; @@ -84,15 +87,11 @@ class ActivationFastTanh : public Activation { public: ActivationFastTanh(){}; - void apply(Eigen::Block block) override + void apply(float* data, long size) override { - float* ptr = block.data(); - - long size = block.rows() * block.cols(); - for (long pos = 0; pos < size; pos++) { - ptr[pos] = fast_tanh(ptr[pos]); + data[pos] = fast_tanh(data[pos]); } } }; @@ -101,15 +100,11 @@ class ActivationReLU : public Activation { public: ActivationReLU(){}; - void apply(Eigen::Block block) override + void apply(float* data, long size) override { - float* ptr = block.data(); - - long size = block.rows() * block.cols(); - for (long pos = 0; pos < size; pos++) { - ptr[pos] = relu(ptr[pos]); + data[pos] = relu(data[pos]); } } }; @@ -118,15 +113,11 @@ class ActivationSigmoid : public Activation { public: ActivationSigmoid(){}; - void apply(Eigen::Block block) override + void apply(float* data, long size) override { - float* ptr = block.data(); - - long size = block.rows() * block.cols(); - for (long pos = 0; pos < size; pos++) { - ptr[pos] = sigmoid(ptr[pos]); + data[pos] = sigmoid(data[pos]); } } }; From 26a725db8e8ab3e1ec1f505efc95b3ef8a5dfe0a Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 3 Apr 2023 16:36:34 -0700 Subject: [PATCH 7/9] Use new activation code for WaveNet Sigmoid --- NAM/dsp.cpp | 7 ------- NAM/dsp.h | 5 ----- NAM/wavenet.cpp | 3 ++- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/NAM/dsp.cpp b/NAM/dsp.cpp index e28eade..3e97665 100644 --- a/NAM/dsp.cpp +++ b/NAM/dsp.cpp @@ -206,13 +206,6 @@ void Linear::_process_core_() // NN modules ================================================================= -void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end) -{ - for (long j = j_start; j < j_end; j++) - for (long i = i_start; i < i_end; i++) - x(i, j) = 1.0 / (1.0 + expf(-x(i, j))); -} - void Conv1D::set_params_(std::vector::iterator& params) { if (this->_weight.size() > 0) diff --git a/NAM/dsp.h b/NAM/dsp.h index 1464114..f15989a 100644 --- a/NAM/dsp.h +++ b/NAM/dsp.h @@ -140,11 +140,6 @@ class Linear : public Buffer // NN modules ================================================================= -// Activations - -// In-place sigmoid -void sigmoid_(Eigen::MatrixXf& x, const long i_start, const long i_end, const long j_start, const long j_end); - class Conv1D { public: diff --git a/NAM/wavenet.cpp b/NAM/wavenet.cpp index b2d6821..231489b 100644 --- a/NAM/wavenet.cpp +++ b/NAM/wavenet.cpp @@ -34,7 +34,8 @@ void wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::Matrix if (this->_gated) { - sigmoid_(this->_z, channels, 2 * channels, 0, this->_z.cols()); + activations::Activation::get_activation("Sigmoid")->apply(this->_z.block(channels, 2 * channels, 0, this->_z.cols())); + this->_z.topRows(channels).array() *= this->_z.bottomRows(channels).array(); // this->_z.topRows(channels) = this->_z.topRows(channels).cwiseProduct( // this->_z.bottomRows(channels) From b41d172910024ee7eb179731b50615424af182d6 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 3 Apr 2023 17:01:52 -0700 Subject: [PATCH 8/9] Fixed WaveNet sigmoid --- NAM/wavenet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NAM/wavenet.cpp b/NAM/wavenet.cpp index 231489b..b844e1d 100644 --- a/NAM/wavenet.cpp +++ b/NAM/wavenet.cpp @@ -34,7 +34,7 @@ void wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::Matrix if (this->_gated) { - activations::Activation::get_activation("Sigmoid")->apply(this->_z.block(channels, 2 * channels, 0, this->_z.cols())); + activations::Activation::get_activation("Sigmoid")->apply(this->_z.block(channels, 0, channels, this->_z.cols())); this->_z.topRows(channels).array() *= this->_z.bottomRows(channels).array(); // this->_z.topRows(channels) = this->_z.topRows(channels).cwiseProduct( From 482b69aa547b72aedf0ac2569218111becb3d831 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 4 Apr 2023 07:50:06 -0700 Subject: [PATCH 9/9] No need for middleCols() in full matrix apply anymore --- NAM/activations.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NAM/activations.h b/NAM/activations.h index 4b65b1f..fabcc05 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -38,7 +38,7 @@ class Activation Activation(){}; virtual void apply(Eigen::MatrixXf& matrix) { - apply(matrix.middleCols(0, matrix.cols()).data(), matrix.rows() * matrix.cols()); + apply(matrix.data(), matrix.rows() * matrix.cols()); } virtual void apply(Eigen::Block block) {