Merge pull request #97 from hunyadi/embedding_layer

New layer type "Embedding"

include/fdeep/import_model.hpp

@@ -44,6 +44,7 @@
 #include "fdeep/layers/input_layer.hpp"
 #include "fdeep/layers/layer.hpp"
 #include "fdeep/layers/leaky_relu_layer.hpp"
+#include "fdeep/layers/embedding_layer.hpp"
 #include "fdeep/layers/lstm_layer.hpp"
 #include "fdeep/layers/gru_layer.hpp"
 #include "fdeep/layers/prelu_layer.hpp"
@@ -853,6 +854,18 @@ inline nodes create_nodes(const nlohmann::json& data)
     return fplus::transform(create_node, inbound_nodes_data);
 }
 
+inline layer_ptr create_embedding_layer(const get_param_f &get_param,
+                                        const get_global_param_f &,
+                                        const nlohmann::json &data,
+                                        const std::string &name)
+{
+    const std::size_t input_dim = data["config"]["input_dim"];
+    const std::size_t output_dim = data["config"]["output_dim"];
+    const float_vec weights = decode_floats(get_param(name, "weights"));
+
+    return std::make_shared<embedding_layer>(name, input_dim, output_dim, weights);
+}
+
 inline layer_ptr create_lstm_layer(const get_param_f &get_param,
                                    const get_global_param_f &,
                                    const nlohmann::json &data,
@@ -1004,6 +1017,7 @@ inline layer_ptr create_layer(const get_param_f& get_param,
             {"Cropping2D", create_cropping_2d_layer},
             {"Activation", create_activation_layer},
             {"Reshape", create_reshape_layer},
+            {"Embedding", create_embedding_layer},
             {"LSTM", create_lstm_layer},
             {"GRU", create_gru_layer},
             {"Bidirectional", create_bidirectional_layer},

include/fdeep/layers/embedding_layer.hpp

@@ -0,0 +1,69 @@
+// Copyright 2016, Tobias Hermann.
+// https://github.com/Dobiasd/frugally-deep
+// Distributed under the MIT License.
+// (See accompanying LICENSE file or at
+//  https://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include "fdeep/layers/layer.hpp"
+
+#include <string>
+#include <functional>
+
+namespace fdeep
+{
+namespace internal
+{
+
+class embedding_layer : public layer
+{
+  public:
+    explicit embedding_layer(const std::string& name,
+                             std::size_t input_dim,
+                             std::size_t output_dim,
+                             const float_vec& weights)
+        : layer(name)
+        , input_dim_(input_dim)
+        , output_dim_(output_dim)
+        , weights_(weights)
+    {}
+
+  protected:
+    tensor5s apply_impl(const tensor5s &inputs) const override final
+    {
+        const auto input_shapes = fplus::transform(fplus_c_mem_fn_t(tensor5, shape, shape5), inputs);
+
+        // ensure that tensor5 shape is (1, 1, 1, 1, seq_len)
+        assertion(inputs.front().shape().size_dim_5_ == 1
+                  && inputs.front().shape().size_dim_4_ == 1
+                  && inputs.front().shape().height_ == 1
+                  && inputs.front().shape().width_ == 1,
+                  "size_dim_5, size_dim_4, height and width dimension must be 1, but shape is '" + show_shape5s(input_shapes) + "'");
+
+        tensor5s results;
+        for (auto&& input : inputs)
+        {
+            const std::size_t sequence_len = input.shape().depth_;
+            float_vec output_vec(sequence_len * output_dim_);
+            auto&& it = output_vec.begin();
+
+            for (std::size_t i = 0; i < sequence_len; ++i)
+            {
+                std::size_t index = static_cast<std::size_t>(input.get(0, 0, 0, 0, i));
+                assertion(index < input_dim_, "vocabulary item indices must all be strictly less than the value of input_dim");
+                it = std::copy_n(weights_.cbegin() + static_cast<float_vec::const_iterator::difference_type>(index * output_dim_), output_dim_, it);
+            }
+
+            results.push_back(tensor5(shape5(1, 1, 1, sequence_len, output_dim_), std::move(output_vec)));
+        }
+        return results;
+    }
+
+    const std::size_t input_dim_;
+    const std::size_t output_dim_;
+    const float_vec weights_;
+};
+
+} // namespace internal
+} // namespace fdeep

keras_export/convert_model.py

@@ -104,9 +104,12 @@ def replace_none_with(value, shape):
     return tuple(list(map(lambda x: x if x is not None else value, shape)))
 
 
-def gen_test_data(model):
+def gen_test_data(model, random_fn=None):
     """Generate data for model verification test."""
 
+    if not random_fn:
+        random_fn = np.random.normal
+
     def set_shape_idx_0_to_1_if_none(shape):
         """Change first element in tuple to 1."""
         if shape[0] is not None:
@@ -122,7 +125,7 @@ def generate_input_data(layer):
             shape = layer.batch_input_shape
         except AttributeError:
             shape = layer.input_shape
-        return np.random.normal(
+        return random_fn(
             size=replace_none_with(42, set_shape_idx_0_to_1_if_none(shape))).astype(np.float32)
 
     data_in = list(map(generate_input_data, get_model_input_layers(model)))
@@ -306,6 +309,15 @@ def show_prelu_layer(layer):
     }
     return result
 
+def show_embedding_layer(layer):
+    """Serialize Embedding layer to dict"""
+    weights = layer.get_weights()
+    assert len(weights) == 1
+    result = {
+        'weights': encode_floats(weights[0])
+    }
+    return result
+
 def show_lstm_layer(layer):
     """Serialize LSTM layer to dict"""
     weights = layer.get_weights()
@@ -359,6 +371,7 @@ def get_layer_functions_dict():
         'BatchNormalization': show_batch_normalization_layer,
         'Dense': show_dense_layer,
         'PReLU': show_prelu_layer,
+        'Embedding': show_embedding_layer,
         'LSTM': show_lstm_layer,
         'GRU': show_gru_layer,
         'Bidirectional': show_bidirectional_layer,
@@ -579,7 +592,16 @@ def convert(in_path, out_path):
     model.compile(loss='mse', optimizer='sgd')
 
     model = convert_sequential_to_model(model)
-    test_data = gen_test_data(model)
+
+    random_fn = None
+
+    # use random integers as input for models with embedding layers
+    for layer in model.layers:
+        if isinstance(layer, keras.layers.Embedding):
+            random_fn = lambda size: np.random.randint(0, layer.input_dim, size)
+            break
+
+    test_data = gen_test_data(model, random_fn)
 
     json_output = {}
     json_output['architecture'] = json.loads(model.to_json())

keras_export/generate_test_models.py

@@ -19,7 +19,9 @@
 from keras.layers import SeparableConv2D, DepthwiseConv2D
 from keras.layers import LeakyReLU, ELU, PReLU
 from keras.layers import BatchNormalization, Concatenate
-from keras.layers import LSTM, GRU, Bidirectional, TimeDistributed
+from keras.layers import Embedding
+from keras.layers import LSTM, GRU
+from keras.layers import Bidirectional, TimeDistributed
 from keras import backend as K
 
 __author__ = "Tobias Hermann"
@@ -61,6 +63,18 @@ def generate_input_data(data_size, input_shapes):
             for input_shape in input_shapes]
 
 
+def generate_integer_random_data(data_size, low, high, shape):
+    """Random data for training."""
+    return np.random.randint(
+        low=low, high=high, size=get_shape_for_random_data(data_size, replace_none_with(42, shape)))
+
+
+def generate_integer_input_data(data_size, low, high, input_shapes):
+    """Random input data for training."""
+    return [generate_integer_random_data(data_size, low, high, input_shape)
+            for input_shape in input_shapes]
+
+
 def generate_output_data(data_size, outputs):
     """Random output data for training."""
     return [generate_random_data(data_size, output.shape[1:])
@@ -152,6 +166,40 @@ def get_test_model_small():
     model.fit(data_in, data_out, epochs=10)
     return model
 
+def get_test_model_embedding():
+    """Returns a minimalistic test model for the embedding layer."""
+
+    input_dim = 1023  # maximum integer value in input data
+    input_shapes = [
+        (100,),  # must be single-element tuple (for sequence length)
+        (1000,)
+    ]
+    output_dims = [8,3]  # embedding dimension
+
+    inputs = [Input(shape=s) for s in input_shapes]
+
+    outputs = []
+    for k in range(0, len(input_shapes)):
+        embedding = Embedding(input_dim=input_dim, output_dim=output_dims[k])(inputs[k])
+        lstm = LSTM(
+            units=4,
+            recurrent_activation='sigmoid',
+            return_sequences=False
+        )(embedding)
+
+        outputs.append(lstm)
+
+    model = Model(inputs=inputs, outputs=outputs, name='test_model_embedding')
+    model.compile(loss='mse', optimizer='nadam')
+
+    # fit to dummy data
+    training_data_size = 1
+    data_in = generate_integer_input_data(training_data_size, 0, input_dim, input_shapes)
+    initial_data_out = model.predict(data_in)
+    data_out = generate_output_data(training_data_size, initial_data_out)
+    model.fit(data_in, data_out, epochs=10)
+    return model
+
 def get_test_model_recurrent():
     """Returns a minimalistic test model for recurrent layers."""
     input_shapes = [
@@ -562,6 +610,7 @@ def main():
 
         get_model_functions = {
             'small': get_test_model_small,
+            'embedding': get_test_model_embedding,
             'recurrent': get_test_model_recurrent,
             'variable': get_test_model_variable,
             'sequential': get_test_model_sequential,

test/CMakeLists.txt

@@ -8,6 +8,10 @@ add_custom_command ( OUTPUT test_model_small.h5
                      COMMAND bash -c "python3 ${FDEEP_TOP_DIR}/keras_export/generate_test_models.py small test_model_small.h5"
                      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/)
 
+add_custom_command ( OUTPUT test_model_embedding.h5
+                     COMMAND bash -c "python3 ${FDEEP_TOP_DIR}/keras_export/generate_test_models.py embedding test_model_embedding.h5"
+                     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/)
+
 add_custom_command ( OUTPUT test_model_recurrent.h5
                      COMMAND bash -c "python3 ${FDEEP_TOP_DIR}/keras_export/generate_test_models.py recurrent test_model_recurrent.h5"
                      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/)
@@ -35,6 +39,11 @@ add_custom_command ( OUTPUT test_model_small.json
                      COMMAND bash -c "python3 ${FDEEP_TOP_DIR}/keras_export/convert_model.py test_model_small.h5 test_model_small.json"
                      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/)
 
+add_custom_command ( OUTPUT test_model_embedding.json
+                     DEPENDS test_model_embedding.h5
+                     COMMAND bash -c "python3 ${FDEEP_TOP_DIR}/keras_export/convert_model.py test_model_embedding.h5 test_model_embedding.json"
+                     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/)
+
 add_custom_command ( OUTPUT test_model_recurrent.json
                      DEPENDS test_model_recurrent.h5
                      COMMAND bash -c "python3 ${FDEEP_TOP_DIR}/keras_export/convert_model.py test_model_recurrent.h5 test_model_recurrent.json"
@@ -74,6 +83,7 @@ macro(_add_test _NAME _DEPENDS)
 endmacro()
 
 _add_test(test_model_small_test test_model_small.json)
+_add_test(test_model_embedding_test test_model_embedding.json)
 _add_test(test_model_recurrent_test test_model_recurrent.json)
 _add_test(test_model_variable_test test_model_variable.json)
 _add_test(test_model_sequential_test test_model_sequential.json)
@@ -86,6 +96,7 @@ _add_test(readme_example_main readme_example_model.json)
 if(FDEEP_BUILD_FULL_TEST)
   add_custom_target(unittest
     COMMAND test_model_small_test
+    COMMAND test_model_embedding_test
     COMMAND test_model_recurrent_test
     COMMAND test_model_variable_test
     COMMAND test_model_sequential_test
@@ -99,6 +110,7 @@ if(FDEEP_BUILD_FULL_TEST)
 else()
   add_custom_target(unittest
     COMMAND test_model_small_test
+    COMMAND test_model_embedding_test
     COMMAND test_model_recurrent_test
     COMMAND test_model_variable_test
     COMMAND test_model_sequential_test

test/test_model_embedding_test.cpp

@@ -0,0 +1,23 @@
+// Copyright 2016, Tobias Hermann.
+// https://github.com/Dobiasd/frugally-deep
+// Distributed under the MIT License.
+// (See accompanying LICENSE file or at
+//  https://opensource.org/licenses/MIT)
+
+#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+#include "doctest.h"
+#include <fdeep/fdeep.hpp>
+
+#define FDEEP_FLOAT_TYPE double
+
+TEST_CASE("test_model_embedding_test, load_model")
+{
+    const auto model = fdeep::load_model("../test_model_embedding.json",
+        true, fdeep::cout_logger, static_cast<fdeep::float_type>(0.00001));
+    const auto multi_inputs = fplus::generate<std::vector<fdeep::tensor5s>>(
+        [&]() -> fdeep::tensor5s {return model.generate_dummy_inputs();},
+        10);
+
+    model.predict_multi(multi_inputs, false);
+    model.predict_multi(multi_inputs, true);
+}