From 9743a49aafebcda5d7cbdaba0d9fa8119da491bc Mon Sep 17 00:00:00 2001 From: Dobiasd Date: Tue, 26 Dec 2023 16:31:50 +0100 Subject: [PATCH] add comment --- include/fdeep/layers/multi_head_attention_layer.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/fdeep/layers/multi_head_attention_layer.hpp b/include/fdeep/layers/multi_head_attention_layer.hpp index b727f900..63ea5333 100644 --- a/include/fdeep/layers/multi_head_attention_layer.hpp +++ b/include/fdeep/layers/multi_head_attention_layer.hpp @@ -33,6 +33,8 @@ class multi_head_attention_layer : public layer //const tensor& query = input[0]; //const tensor& value = input[1]; //const tensor& key = input.size() > 2 ? input[2] : value; + // https://towardsdatascience.com/transformers-explained-visually-part-3-multi-head-attention-deep-dive-1c1ff1024853 + // https://dmol.pub/dl/attention.html#multi-head-attention-block // https://github.com/keras-team/keras/blob/v2.14.0/keras/layers/attention/multi_head_attention.py // https://gist.github.com/sevagh/b71d253a347a9b59c026580625452fc5 return input;