From 694584a41141b331159532b0dfeaed4d1566040b Mon Sep 17 00:00:00 2001 From: Dobiasd Date: Thu, 26 Aug 2021 11:21:52 +0200 Subject: [PATCH 1/2] Add support for negative_slope and threshold in ReLU layers --- include/fdeep/import_model.hpp | 12 ++++++++++-- include/fdeep/layers/relu_layer.hpp | 18 +++++++++++++++--- keras_export/convert_model.py | 9 --------- keras_export/generate_test_models.py | 2 ++ 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/include/fdeep/import_model.hpp b/include/fdeep/import_model.hpp index 7e227a21..6458124c 100644 --- a/include/fdeep/import_model.hpp +++ b/include/fdeep/import_model.hpp @@ -802,20 +802,28 @@ inline activation_layer_ptr create_relu_layer( const std::string& name) { float_type max_value = std::numeric_limits::max(); + float_type negative_slope = static_cast(0); + float_type threshold = static_cast(0); if (json_obj_has_member(data, "config") && json_obj_has_member(data["config"], "max_value") && !data["config"]["max_value"].is_null()) { max_value = data["config"]["max_value"]; + negative_slope = data["config"]["negative_slope"]; + threshold = data["config"]["threshold"]; } - return std::make_shared(name, max_value); + return std::make_shared(name, max_value, negative_slope, threshold); } inline activation_layer_ptr create_relu6_layer( const get_param_f&, const nlohmann::json&, const std::string& name) { - return std::make_shared(name, static_cast(6)); + return std::make_shared(name, + static_cast(6), + static_cast(0), + static_cast(0) + ); } inline activation_layer_ptr create_selu_layer( diff --git a/include/fdeep/layers/relu_layer.hpp b/include/fdeep/layers/relu_layer.hpp index 6f15aa6a..d8bc473b 100644 --- a/include/fdeep/layers/relu_layer.hpp +++ b/include/fdeep/layers/relu_layer.hpp @@ -17,8 +17,14 @@ namespace fdeep { namespace internal class relu_layer : public activation_layer { public: - explicit relu_layer(const std::string& name, const float_type max_value) - : activation_layer(name), max_value_(max_value) + explicit relu_layer(const std::string& name, + const float_type max_value, + const float_type negative_slope, + const float_type threshold) + : activation_layer(name), + max_value_(max_value), + negative_slope_(negative_slope), + threshold_(threshold) { } protected: @@ -26,11 +32,17 @@ class relu_layer : public activation_layer { auto activation_function = [&](float_type x) -> float_type { - return std::min(std::max(x, 0), max_value_); + if (x >= max_value_) + return max_value_; + if (threshold_ <= x && x < max_value_) + return x; + return negative_slope_ * (x - threshold_); }; return transform_tensor(activation_function, in_vol); } float_type max_value_; + float_type negative_slope_; + float_type threshold_; }; } } // namespace fdeep, namespace internal diff --git a/keras_export/convert_model.py b/keras_export/convert_model.py index 9cd9e82a..23c85fd6 100755 --- a/keras_export/convert_model.py +++ b/keras_export/convert_model.py @@ -352,14 +352,6 @@ def show_prelu_layer(layer): return result -def show_relu_layer(layer): - """Serialize relu layer to dict""" - assert layer.negative_slope == 0 - assert layer.threshold == 0 - assert layer.max_value == None - return {} - - def show_embedding_layer(layer): """Serialize Embedding layer to dict""" weights = layer.get_weights() @@ -507,7 +499,6 @@ def get_layer_functions_dict(): 'BatchNormalization': show_batch_normalization_layer, 'Dense': show_dense_layer, 'PReLU': show_prelu_layer, - 'ReLU': show_relu_layer, 'Embedding': show_embedding_layer, 'LSTM': show_lstm_layer, 'GRU': show_gru_layer, diff --git a/keras_export/generate_test_models.py b/keras_export/generate_test_models.py index 9ceda911..bdc52cc4 100644 --- a/keras_export/generate_test_models.py +++ b/keras_export/generate_test_models.py @@ -368,6 +368,8 @@ def get_test_model_exhaustive(): Activation('gelu')(inputs[25]), Activation('softsign')(inputs[25]), LeakyReLU()(inputs[25]), + ReLU()(inputs[25]), + ReLU(max_value=0.4, negative_slope=1.1, threshold=0.3)(inputs[25]), ELU()(inputs[25]), PReLU()(inputs[24]), PReLU()(inputs[25]), From 0b4f53452fe16b4b1cf607bdfd51c8835d0d23dc Mon Sep 17 00:00:00 2001 From: Dobiasd Date: Thu, 26 Aug 2021 11:31:52 +0200 Subject: [PATCH 2/2] Bump version number to 0.15.12 --- CMakeLists.txt | 2 +- INSTALL.md | 2 +- test/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e8f9e97..3ec26910 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ list(APPEND CMAKE_MODULE_PATH "${FDEEP_TOP_DIR}/cmake") include(cmake/hunter.cmake) # default off -project(frugally-deep VERSION 0.15.10) +project(frugally-deep VERSION 0.15.12) message(STATUS "===( ${PROJECT_NAME} ${PROJECT_VERSION} )===") diff --git a/INSTALL.md b/INSTALL.md index e096be09..a54540f6 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -63,7 +63,7 @@ Just add a *conanfile.txt* with frugally-deep as a requirement and chose the gen ``` [requires] -frugally-deep/v0.15.10-p0@dobiasd/stable +frugally-deep/v0.15.12-p0@dobiasd/stable [generators] cmake diff --git a/test/Dockerfile b/test/Dockerfile index d9aef42e..a146f5dd 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -29,7 +29,7 @@ WORKDIR / RUN git clone -b 'v0.2.15-p0' --single-branch --depth 1 https://github.com/Dobiasd/FunctionalPlus && cd FunctionalPlus && mkdir -p build && cd build && cmake .. && make && make install RUN git clone -b '3.3.9' --single-branch --depth 1 https://gitlab.com/libeigen/eigen.git && cd eigen && mkdir -p build && cd build && cmake .. && make && make install && ln -s /usr/local/include/eigen3/Eigen /usr/local/include/Eigen RUN git clone -b 'v3.9.1' --single-branch --depth 1 https://github.com/nlohmann/json && cd json && mkdir -p build && cd build && cmake -DBUILD_TESTING=OFF .. && make && make install -RUN git clone -b 'v0.15.10-p0' --single-branch --depth 1 https://github.com/Dobiasd/frugally-deep && cd frugally-deep && mkdir -p build && cd build && cmake .. && make && make install +RUN git clone -b 'v0.15.12-p0' --single-branch --depth 1 https://github.com/Dobiasd/frugally-deep && cd frugally-deep && mkdir -p build && cd build && cmake .. && make && make install # To have downloaded the Keras models already RUN pip install numpy --upgrade