Local Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from model_X.bilstm_architectures import *\n",
    "from model_X.dense_architectures import *\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.optimizers import *\n",
    "from tensorflow.keras.models import Model, Sequential\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model_X.bilstm_architectures import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 100, 100)     1000        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d (SpatialDropo (None, 100, 100)     0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   (None, 100, 512)     731136      spatial_dropout1d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 100, 256)     493056      bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "scaled_dot_product_attention (S (None, 100, 512)     0           bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "scaled_dot_product_attention_1  (None, 100, 256)     0           bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 512)          0           scaled_dot_product_attention[0][0\n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 256)          0           scaled_dot_product_attention_1[0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 768)          0           global_max_pooling1d[0][0]       \n",
      "                                                                 global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1024)         787456      concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          524800      dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          131328      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 128)          32896       dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 64)           8256        dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 3)            195         dense_4[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 2,710,123\n",
      "Trainable params: 2,709,123\n",
      "Non-trainable params: 1,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "bilstm_layers = BiLSTMGRUAttention(10, 100, attention_type='ScaledDotProductAttention')(model_input)\n",
    "dense_layers = DenseLayerModel()(bilstm_layers)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 100, 100)     1000        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d (SpatialDropo (None, 100, 100)     0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   (None, 100, 512)     731136      spatial_dropout1d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 100, 256)     493056      bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "seq_self_attention (SeqSelfAtte (None, None, 512)    32833       bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "seq_self_attention_1 (SeqSelfAt (None, None, 256)    16449       bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 512)          0           seq_self_attention[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 256)          0           seq_self_attention_1[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 768)          0           global_max_pooling1d[0][0]       \n",
      "                                                                 global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1024)         787456      concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          524800      dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          131328      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 128)          32896       dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 64)           8256        dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 3)            195         dense_4[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 2,759,405\n",
      "Trainable params: 2,758,405\n",
      "Non-trainable params: 1,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "bilstm_layers = BiLSTMGRUSelfAttention(10, 100)(model_input)\n",
    "dense_layers = DenseLayerModel()(bilstm_layers)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 100, 100)     1000        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_1 (SpatialDro (None, 100, 100)     0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 100, 512)     731136      spatial_dropout1d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 100, 256)     493056      bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 512)          0           bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 256)          0           bidirectional_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 768)          0           global_max_pooling1d_2[0][0]     \n",
      "                                                                 global_max_pooling1d_3[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 1024)         787456      concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 512)          524800      dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 256)          131328      dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 128)          32896       dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 64)           8256        dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 3)            195         dense_10[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 2,710,123\n",
      "Trainable params: 2,709,123\n",
      "Non-trainable params: 1,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "bilstm_layers = BiLSTMGRUSpatialDropout1D(10, 100)(model_input)\n",
    "dense_layers = DenseLayerModel()(bilstm_layers)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 100, 100)     1000        input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_2 (SpatialDro (None, 100, 100)     0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "multi_head (MultiHead)          (None, 512, 3)       2193408     spatial_dropout1d_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multi_head_1 (MultiHead)        (None, 256, 3)       529920      spatial_dropout1d_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multi_head_attention (MultiHead (None, 512, 3)       48          multi_head[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multi_head_attention_1 (MultiHe (None, 256, 3)       48          multi_head_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_4 (GlobalM (None, 3)            0           multi_head_attention[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_5 (GlobalM (None, 3)            0           multi_head_attention_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 6)            0           global_max_pooling1d_4[0][0]     \n",
      "                                                                 global_max_pooling1d_5[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 1024)         7168        concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 512)          524800      dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 256)          131328      dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 128)          32896       dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 64)           8256        dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 3)            195         dense_16[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 3,429,067\n",
      "Trainable params: 3,428,067\n",
      "Non-trainable params: 1,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "bilstm_layers = BiLSTMGRUMultiHeadAttention(10, 100)(model_input)\n",
    "dense_layers = DenseLayerModel()(bilstm_layers)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 100, 100)     1000        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d (SpatialDropo (None, 100, 100)     0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda (Lambda)                 [(None, 100, 50), (N 0           spatial_dropout1d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   (None, 100, 512)     628736      lambda[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 100, 1024)    2306048     lambda[0][1]                     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 100, 1536)    0           bidirectional[0][0]              \n",
      "                                                                 bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 100, 256)     1279488     concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 256)          0           bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1024)         263168      global_max_pooling1d[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          524800      dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          131328      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 128)          32896       dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 64)           8256        dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 3)            195         dense_4[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 5,175,915\n",
      "Trainable params: 5,174,915\n",
      "Non-trainable params: 1,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "bilstm_layers = SplitBiLSTMGRUSpatialDropout1D(10, 100)(model_input)\n",
    "dense_layers = DenseLayerModel()(bilstm_layers)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 100, 100)     1000        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               [(None, 100, 50), (N 0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 100, 512)     628736      lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_4 (Bidirectional) (None, 100, 1024)    2306048     lambda_1[0][1]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 100, 1536)    0           bidirectional_3[0][0]            \n",
      "                                                                 bidirectional_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_5 (Bidirectional) (None, 100, 256)     1279488     concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 256)          0           bidirectional_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 1024)         263168      global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 512)          524800      dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 256)          131328      dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 128)          32896       dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 64)           8256        dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 3)            195         dense_10[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 5,175,915\n",
      "Trainable params: 5,174,915\n",
      "Non-trainable params: 1,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "bilstm_layers = SplitBiLSTMGRU(10, 100)(model_input)\n",
    "dense_layers = DenseLayerModel()(bilstm_layers)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 100)]             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1024)              103424    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 512)               524800    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 3)                 195       \n",
      "=================================================================\n",
      "Total params: 800,899\n",
      "Trainable params: 800,899\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "dense_layers = DenseLayerModel()(model_input)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 100)]             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1024)              103424    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 512)               524800    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 3)                 195       \n",
      "=================================================================\n",
      "Total params: 800,899\n",
      "Trainable params: 800,899\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "dense_layers = DenseLayerModelWithDropout()(model_input)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 100, 100)     1000        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d (SpatialDropo (None, 100, 100)     0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   (None, 100, 512)     731136      spatial_dropout1d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 100, 512)     0           bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 100, 256)     493056      dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 100, 256)     0           bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "seq_self_attention (SeqSelfAtte (None, None, 512)    32833       dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "seq_self_attention_1 (SeqSelfAt (None, None, 256)    16449       dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 512)          0           seq_self_attention[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 256)          0           seq_self_attention_1[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 768)          0           global_max_pooling1d[0][0]       \n",
      "                                                                 global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 1024)         787456      concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 512)          524800      dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 256)          131328      dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 128)          32896       dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 64)           8256        dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 3)            195         dense_10[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 2,759,405\n",
      "Trainable params: 2,758,405\n",
      "Non-trainable params: 1,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "bilstm_layers = BiLSTMGRUAttention(10, 100, dropout=0.2)(model_input)\n",
    "dense_layers = DenseLayerModel()(bilstm_layers)\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(<tf.Tensor 'concatenate_1/Identity:0' shape=(None, 768) dtype=float32>, <tf.Tensor 'seq_self_attention_2/Identity_1:0' shape=(None, None, None) dtype=float32>, <tf.Tensor 'seq_self_attention_3/Identity_1:0' shape=(None, None, None) dtype=float32>)\n",
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 100, 100)     1000        input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_1 (SpatialDro (None, 100, 100)     0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 100, 512)     731136      spatial_dropout1d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 100, 256)     493056      bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "seq_self_attention_2 (SeqSelfAt [(None, None, 512),  32833       bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "seq_self_attention_3 (SeqSelfAt [(None, None, 256),  16449       bidirectional_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 512)          0           seq_self_attention_2[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 256)          0           seq_self_attention_3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 768)          0           global_max_pooling1d_2[0][0]     \n",
      "                                                                 global_max_pooling1d_3[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 1024)         787456      concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 512)          524800      dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 256)          131328      dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 128)          32896       dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 64)           8256        dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 3)            195         dense_16[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 2,759,405\n",
      "Trainable params: 2,758,405\n",
      "Non-trainable params: 1,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "input_shape = (100,)\n",
    "model_input = Input(shape=input_shape)\n",
    "bilstm_layers = BiLSTMGRUAttention(10, 100, return_attention=True)(model_input)\n",
    "print(bilstm_layers)\n",
    "dense_layers = DenseLayerModel()(bilstm_layers[0])\n",
    "output = Dense(3, activation='softmax')(dense_layers)\n",
    "full_model = Model(inputs=model_input, outputs=output)\n",
    "print(full_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0,'./model_X/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers_utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = argparse.Namespace(vocab_size=1000,\n",
    "                            embed_dim=128,\n",
    "                            ff_dim=32,\n",
    "                            num_heads=8,\n",
    "                            rate=0.1,\n",
    "                            maxlen=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tf.keras.layers.Input(shape=(config.maxlen,))\n",
    "embedding_layer = TokenAndPositionEmbedding(config)(inputs)\n",
    "transformer_block = TransformerBlock(config)(embedding_layer)\n",
    "x = tf.keras.layers.GlobalAveragePooling1D()(transformer_block)\n",
    "x = tf.keras.layers.Dropout(0.1)(x)\n",
    "x = tf.keras.layers.Dense(20, activation=\"relu\")(x)\n",
    "x = tf.keras.layers.Dropout(0.1)(x)\n",
    "outputs = tf.keras.layers.Dense(2, activation=\"softmax\")(x)\n",
    "\n",
    "model = tf.keras.Model(inputs=inputs, outputs=outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Shape (TensorFlowOp [(2,)]               0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_strided_slice (Tens [()]                 0           tf_op_layer_Shape[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_range (TensorFlowOp [(None,)]            0           tf_op_layer_strided_slice[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 128, 128)     128000      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 128)          16384       tf_op_layer_range[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_add (TensorFlowOpLa [(None, 128, 128)]   0           embedding[0][0]                  \n",
      "                                                                 embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Shape_1 (TensorFlow [(3,)]               0           tf_op_layer_add[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_strided_slice_1 (Te [()]                 0           tf_op_layer_Shape_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 128, 128)     16512       tf_op_layer_add[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Reshape_1/shape (Te [(4,)]               0           tf_op_layer_strided_slice_1[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Reshape_1 (TensorFl [(None, None, 8, 16) 0           dense_1[0][0]                    \n",
      "                                                                 tf_op_layer_Reshape_1/shape[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_transpose_1 (Tensor [(None, 8, None, 16) 0           tf_op_layer_Reshape_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 128, 128)     16512       tf_op_layer_add[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Reshape/shape (Tens [(4,)]               0           tf_op_layer_strided_slice_1[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Shape_2 (TensorFlow [(4,)]               0           tf_op_layer_transpose_1[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Reshape (TensorFlow [(None, None, 8, 16) 0           dense[0][0]                      \n",
      "                                                                 tf_op_layer_Reshape/shape[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_strided_slice_2 (Te [()]                 0           tf_op_layer_Shape_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_transpose (TensorFl [(None, 8, None, 16) 0           tf_op_layer_Reshape[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Cast (TensorFlowOpL [()]                 0           tf_op_layer_strided_slice_2[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_MatMul (TensorFlowO [(None, 8, None, Non 0           tf_op_layer_transpose[0][0]      \n",
      "                                                                 tf_op_layer_transpose_1[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Sqrt (TensorFlowOpL [()]                 0           tf_op_layer_Cast[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 128, 128)     16512       tf_op_layer_add[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Reshape_2/shape (Te [(4,)]               0           tf_op_layer_strided_slice_1[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_truediv (TensorFlow [(None, 8, None, Non 0           tf_op_layer_MatMul[0][0]         \n",
      "                                                                 tf_op_layer_Sqrt[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Reshape_2 (TensorFl [(None, None, 8, 16) 0           dense_2[0][0]                    \n",
      "                                                                 tf_op_layer_Reshape_2/shape[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax (TensorFlow [(None, 8, None, Non 0           tf_op_layer_truediv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_transpose_2 (Tensor [(None, 8, None, 16) 0           tf_op_layer_Reshape_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_MatMul_1 (TensorFlo [(None, 8, None, 16) 0           tf_op_layer_Softmax[0][0]        \n",
      "                                                                 tf_op_layer_transpose_2[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_transpose_3 (Tensor [(None, None, 8, 16) 0           tf_op_layer_MatMul_1[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Reshape_3/shape (Te [(3,)]               0           tf_op_layer_strided_slice_1[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Reshape_3 (TensorFl [(None, None, 128)]  0           tf_op_layer_transpose_3[0][0]    \n",
      "                                                                 tf_op_layer_Reshape_3/shape[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, None, 128)    16512       tf_op_layer_Reshape_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, None, 128)    0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_add_1 (TensorFlowOp [(None, 128, 128)]   0           tf_op_layer_add[0][0]            \n",
      "                                                                 dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization (LayerNorma (None, 128, 128)     256         tf_op_layer_add_1[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "sequential (Sequential)         (None, 128, 128)     8352        layer_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 128, 128)     0           sequential[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_add_2 (TensorFlowOp [(None, 128, 128)]   0           layer_normalization[0][0]        \n",
      "                                                                 dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_1 (LayerNor (None, 128, 128)     256         tf_op_layer_add_2[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d (Globa (None, 128)          0           layer_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 128)          0           global_average_pooling1d[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 20)           2580        dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 20)           0           dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 2)            42          dropout_3[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 221,918\n",
      "Trainable params: 205,534\n",
      "Non-trainable params: 16,384\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import tensorflow as tf\n",
    "import argparse\n",
    "sys.path.insert(0,'./model_X/')\n",
    "from transformers_utils import *\n",
    "from transformers_architectures import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = argparse.Namespace(vocab_size=1000,\n",
    "                            embed_dim=512,\n",
    "                            ff_dim=32,\n",
    "                            num_heads=8,\n",
    "                            rate=0.1,\n",
    "                            maxlen=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tf.keras.layers.Input(shape=(config.maxlen,))\n",
    "pooled_output,sequence_output = VanillaTransformer(config)(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'pooler_transform_1/Identity:0' shape=(None, 512) dtype=float32>,\n",
       " <tf.Tensor 'layer_normalization_3/Identity:0' shape=(None, 128, 512) dtype=float32>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pooled_output, sequence_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.5 64-bit ('tf2_env': venv)",
   "language": "python",
   "name": "python37564bittf2envvenv9baad5f63bf544adb456767f347f0830"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}