Skip to content

Commit 0474498

Browse files
DobiasdDobiasd
authored
Merge pull request #216 from Dobiasd/tf21
Update TensorFlow to version 2.1
2 parents cdbcfc4 + 37eaf67 commit 0474498

14 files changed

+463
-82
lines changed

‎.travis.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ install:
3333
- sudo apt-get install libblas-dev liblapack-dev libatlas-base-dev gfortran
3434
# python libs
3535
- sudo pip3 install --upgrade pip
36-
- sudo pip3 install numpy scipy h5py "tensorflow==2.0.0"
36+
- sudo pip3 install numpy scipy h5py "tensorflow==2.1.0"
3737
- echo "Version numbers of TensorFlow and Keras:"
3838
- python3 -c "import tensorflow as tf; import tensorflow.keras; print(tf.__version__); print(tensorflow.keras.__version__)"
3939
# FunctionalPlus

‎CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ list(APPEND CMAKE_MODULE_PATH "${FDEEP_TOP_DIR}/cmake")
66

77
include(cmake/hunter.cmake) # default off
88

9-
project(frugally-deep VERSION 0.12.1)
9+
project(frugally-deep VERSION 0.13.0)
1010

1111
message(STATUS "===( ${PROJECT_NAME} ${PROJECT_VERSION} )===")
1212

‎INSTALL.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ Just add a *conanfile.txt* with frugally-deep as a requirement and chose the gen
6363

6464
```
6565
[requires]
66-
frugally-deep/v0.12.1-p0@dobiasd/stable
66+
frugally-deep/v0.13.0-p0@dobiasd/stable
6767
6868
[generators]
6969
cmake

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ Requirements and Installation
173173

174174
- A **C++14**-compatible compiler: Compilers from these versions on are fine: GCC 4.9, Clang 3.7 (libc++ 3.7) and Visual C++ 2015.
175175
- Python 3.5 or higher.
176-
- TensorFlow 2.0.0
176+
- TensorFlow 2.1.0
177177

178178
Guides for different ways to install frugally-deep can be found in [`INSTALL.md`](INSTALL.md).
179179

‎include/fdeep/layers/bidirectional_layer.hpp

Lines changed: 41 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -56,17 +56,23 @@ class bidirectional_layer : public layer
5656
forward_state_h_(stateful ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>()),
5757
forward_state_c_(stateful && wrapped_layer_type_has_state_c(wrapped_layer_type) ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>()),
5858
backward_state_h_(stateful ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>()),
59-
backward_state_c_(stateful && wrapped_layer_type_has_state_c(wrapped_layer_type) ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>())
59+
backward_state_c_(stateful && wrapped_layer_type_has_state_c(wrapped_layer_type) ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>()),
60+
use_avail_input_state_for_stateful_(true)
61+
6062
{
6163
}
6264

6365
void reset_states() override
6466
{
67+
// TF 2.1 Bug: reset_states() does nothing in TF 2.1.
68+
// the implementation below is how TF 2.1 should behave.
69+
// to match TF 2.1, just comment out the code below.
6570
if (is_stateful()) {
6671
forward_state_h_ = tensor(tensor_shape(n_units_), static_cast<float_type>(0));
6772
forward_state_c_ = tensor(tensor_shape(n_units_), static_cast<float_type>(0));
6873
backward_state_h_ = tensor(tensor_shape(n_units_), static_cast<float_type>(0));
6974
backward_state_c_ = tensor(tensor_shape(n_units_), static_cast<float_type>(0));
75+
use_avail_input_state_for_stateful_ = true;
7076
}
7177
}
7278

@@ -110,29 +116,26 @@ class bidirectional_layer : public layer
110116
assertion(inputs.size() == 1 || inputs.size() == 5,
111117
"Invalid number of input tensors.");
112118

113-
tensor forward_state_h = inputs.size() == 5
114-
? inputs[1]
115-
: is_stateful()
116-
? forward_state_h_.unsafe_get_just()
117-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
118-
119-
tensor forward_state_c = inputs.size() == 5
120-
? inputs[2]
121-
: is_stateful()
122-
? forward_state_c_.unsafe_get_just()
123-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
124-
125-
tensor backward_state_h = inputs.size() == 5
126-
? inputs[3]
127-
: is_stateful()
128-
? backward_state_h_.unsafe_get_just()
129-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
130-
131-
tensor backward_state_c = inputs.size() == 5
132-
? inputs[4]
133-
: is_stateful()
134-
? backward_state_c_.unsafe_get_just()
135-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
119+
bool initial_state_provided = inputs.size() == 5;
120+
bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_;
121+
bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state;
122+
// bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state;
123+
124+
tensor forward_state_h = use_input_initial_state ? inputs[1] :
125+
use_last_state_for_initial_state ? forward_state_h_.unsafe_get_just() :
126+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
127+
128+
tensor forward_state_c = use_input_initial_state ? inputs[2] :
129+
use_last_state_for_initial_state ? forward_state_c_.unsafe_get_just() :
130+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
131+
132+
tensor backward_state_h = use_input_initial_state ? inputs[3] :
133+
use_last_state_for_initial_state ? backward_state_h_.unsafe_get_just() :
134+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
135+
136+
tensor backward_state_c = use_input_initial_state ? inputs[4] :
137+
use_last_state_for_initial_state ? backward_state_c_.unsafe_get_just() :
138+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
136139

137140
result_forward = lstm_impl(input, forward_state_h, forward_state_c,
138141
n_units_, use_bias_, return_sequences_, stateful_,
@@ -147,24 +150,26 @@ class bidirectional_layer : public layer
147150
forward_state_c_ = forward_state_c;
148151
backward_state_h_ = backward_state_h;
149152
backward_state_c_ = backward_state_c;
153+
use_avail_input_state_for_stateful_ = false;
150154
}
151155
}
152156
else if (wrapped_layer_type_ == "GRU" || wrapped_layer_type_ == "CuDNNGRU")
153157
{
154158
assertion(inputs.size() == 1 || inputs.size() == 3,
155159
"Invalid number of input tensors.");
156160

157-
tensor forward_state_h = inputs.size() == 3
158-
? inputs[1]
159-
: is_stateful()
160-
? forward_state_h_.unsafe_get_just()
161-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
161+
bool initial_state_provided = inputs.size() == 3;
162+
bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_;
163+
bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state;
164+
// bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state;
165+
166+
tensor forward_state_h = use_input_initial_state ? inputs[1] :
167+
use_last_state_for_initial_state ? forward_state_h_.unsafe_get_just() :
168+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
162169

163-
tensor backward_state_h = inputs.size() == 3
164-
? inputs[2]
165-
: is_stateful()
166-
? backward_state_h_.unsafe_get_just()
167-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
170+
tensor backward_state_h = use_input_initial_state ? inputs[2] :
171+
use_last_state_for_initial_state ? backward_state_h_.unsafe_get_just() :
172+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
168173

169174
result_forward = gru_impl(input, forward_state_h, n_units_, use_bias_, reset_after_, return_sequences_, false,
170175
forward_weights_, forward_recurrent_weights_,
@@ -175,6 +180,7 @@ class bidirectional_layer : public layer
175180
if (is_stateful()) {
176181
forward_state_h_ = forward_state_h;
177182
backward_state_h_ = backward_state_h;
183+
use_avail_input_state_for_stateful_ = false;
178184
}
179185
}
180186
else
@@ -223,6 +229,7 @@ class bidirectional_layer : public layer
223229
mutable fplus::maybe<tensor> forward_state_c_;
224230
mutable fplus::maybe<tensor> backward_state_h_;
225231
mutable fplus::maybe<tensor> backward_state_c_;
232+
mutable bool use_avail_input_state_for_stateful_;
226233
};
227234

228235
} // namespace internal

‎include/fdeep/layers/gru_layer.hpp

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ class gru_layer : public layer
4444
weights_(weights),
4545
recurrent_weights_(recurrent_weights),
4646
bias_(bias),
47-
state_h_(stateful ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>())
47+
state_h_(stateful ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>()),
48+
use_avail_input_state_for_stateful_(true)
4849

4950
{
5051
}
@@ -53,6 +54,7 @@ class gru_layer : public layer
5354
{
5455
if (is_stateful()) {
5556
state_h_ = tensor(tensor_shape(n_units_), static_cast<float_type>(0));
57+
use_avail_input_state_for_stateful_ = true;
5658
}
5759
}
5860

@@ -77,17 +79,26 @@ class gru_layer : public layer
7779
assertion(inputs.size() == 1 || inputs.size() == 2,
7880
"Invalid number of input tensors.");
7981

80-
tensor state_h = inputs.size() == 2
81-
? inputs[1]
82-
: is_stateful()
83-
? state_h_.unsafe_get_just()
84-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
85-
82+
// RNN behaivor since TF 2.1:
83+
// If an *initial state input is provided*, this is used always for non-stateful models
84+
// but only on reset for stateful models (including the very first call)
85+
// If *no input state is provided*, then initial state is 0 for non-stateful
86+
// and, for stateful, it carries the state from previous call, unless state-reset, in which case it set to 0
87+
bool initial_state_provided = inputs.size() == 2;
88+
bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_;
89+
bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state;
90+
// bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state;
91+
92+
tensor state_h = use_input_initial_state ? inputs[1] :
93+
use_last_state_for_initial_state ? state_h_.unsafe_get_just() :
94+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
95+
8696
const auto result = gru_impl(input, state_h, n_units_, use_bias_,
8797
reset_after_, return_sequences_, return_state_, weights_, recurrent_weights_,
8898
bias_, activation_, recurrent_activation_);
8999
if (is_stateful()) {
90100
state_h_ = state_h;
101+
use_avail_input_state_for_stateful_ = false;
91102
}
92103
return result;
93104
}
@@ -104,6 +115,7 @@ class gru_layer : public layer
104115
const float_vec recurrent_weights_;
105116
const float_vec bias_;
106117
mutable fplus::maybe<tensor> state_h_;
118+
mutable bool use_avail_input_state_for_stateful_;
107119
};
108120

109121
} // namespace internal

‎include/fdeep/layers/lstm_layer.hpp

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,9 @@ class lstm_layer : public layer
4343
recurrent_weights_(recurrent_weights),
4444
bias_(bias),
4545
state_h_(stateful ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>()),
46-
state_c_(stateful ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>())
46+
state_c_(stateful ? tensor(tensor_shape(n_units), static_cast<float_type>(0)) : fplus::nothing<tensor>()),
47+
use_avail_input_state_for_stateful_(true)
48+
4749
{
4850
}
4951

@@ -52,6 +54,7 @@ class lstm_layer : public layer
5254
if (is_stateful()) {
5355
state_h_ = tensor(tensor_shape(n_units_), static_cast<float_type>(0));
5456
state_c_ = tensor(tensor_shape(n_units_), static_cast<float_type>(0));
57+
use_avail_input_state_for_stateful_ = true;
5558
}
5659
}
5760

@@ -74,24 +77,31 @@ class lstm_layer : public layer
7477
assertion(inputs.size() == 1 || inputs.size() == 3,
7578
"Invalid number of input tensors.");
7679

77-
tensor state_h = inputs.size() == 3
78-
? inputs[1]
79-
: is_stateful()
80-
? state_h_.unsafe_get_just()
81-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
80+
// RNN behaivor since TF 2.1:
81+
// If an *initial state input is provided*, this is used always for non-stateful models
82+
// but only on reset for stateful models (including the very first call)
83+
// If *no input state is provided*, then initial state is 0 for non-stateful
84+
// and, for stateful, it carries the state from previous call, unless state-reset, in which case it set to 0
85+
bool initial_state_provided = inputs.size() == 3;
86+
bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_;
87+
bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state;
88+
// bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state;
89+
90+
tensor state_h = use_input_initial_state ? inputs[1] :
91+
use_last_state_for_initial_state ? state_h_.unsafe_get_just() :
92+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
8293

83-
tensor state_c = inputs.size() == 3
84-
? inputs[2]
85-
: is_stateful()
86-
? state_c_.unsafe_get_just()
87-
: tensor(tensor_shape(n_units_), static_cast<float_type>(0));
94+
tensor state_c = use_input_initial_state ? inputs[2] :
95+
use_last_state_for_initial_state ? state_c_.unsafe_get_just() :
96+
tensor(tensor_shape(n_units_), static_cast<float_type>(0)); // use_zero_initial_state
8897

8998
const auto result = lstm_impl(input, state_h, state_c,
9099
n_units_, use_bias_, return_sequences_, return_state_, weights_,
91100
recurrent_weights_, bias_, activation_, recurrent_activation_);
92101
if (is_stateful()) {
93102
state_h_ = state_h;
94103
state_c_ = state_c;
104+
use_avail_input_state_for_stateful_ = false;
95105
}
96106
return result;
97107
}
@@ -108,6 +118,7 @@ class lstm_layer : public layer
108118
const float_vec bias_;
109119
mutable fplus::maybe<tensor> state_h_;
110120
mutable fplus::maybe<tensor> state_c_;
121+
mutable bool use_avail_input_state_for_stateful_;
111122
};
112123

113124
} // namespace internal

0 commit comments

Comments
 (0)