seq2seq_keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, LSTM, Dense, Embedding\n",
    "import numpy as np\n",
    "from data_utils import Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128  # Batch size for training.\n",
    "epochs = 100  # Number of epochs to train for.\n",
    "latent_dim = 256  # Latent dimensionality of the encoding space.\n",
    "num_samples = 10000  # Number of samples to train on.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid data size= 500\n",
      "Train data size= 5413\n"
     ]
    }
   ],
   "source": [
    "# VQA_MED\n",
    "val_images, val_questions, val_answers = Utility.read_dataset(\"Valid\")\n",
    "train_images, train_questions, train_answers = Utility.read_dataset(\"Train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eng</th>\n",
       "      <th>fr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>368</th>\n",
       "      <td>what shows fluid filled appendix with peri cae...</td>\n",
       "      <td>ct image</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2547</th>\n",
       "      <td>what does the mass show ?</td>\n",
       "      <td>combination of fat calcification and soft tissue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2598</th>\n",
       "      <td>what shows the involvement of the right iliac ...</td>\n",
       "      <td>axial mri</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154</th>\n",
       "      <td>what is the post operative ct scan of the head...</td>\n",
       "      <td>the complete resection of the metastatic lesio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2667</th>\n",
       "      <td>what show contrast enhancement in both joint s...</td>\n",
       "      <td>post contrast images</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>727</th>\n",
       "      <td>what shows diffuse ground glass opacities ?</td>\n",
       "      <td>ct thorax</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4335</th>\n",
       "      <td>what shows a lesion with peripheral ossificati...</td>\n",
       "      <td>ct of the thigh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5179</th>\n",
       "      <td>what does multiplanar ct reconstruction and su...</td>\n",
       "      <td>vessels</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3542</th>\n",
       "      <td>what does axial mri image show intense enhance...</td>\n",
       "      <td>of the mass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>what does sagittal mri image show the severe c...</td>\n",
       "      <td>of the cord</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    eng  \\\n",
       "368   what shows fluid filled appendix with peri cae...   \n",
       "2547                          what does the mass show ?   \n",
       "2598  what shows the involvement of the right iliac ...   \n",
       "1154  what is the post operative ct scan of the head...   \n",
       "2667  what show contrast enhancement in both joint s...   \n",
       "727         what shows diffuse ground glass opacities ?   \n",
       "4335  what shows a lesion with peripheral ossificati...   \n",
       "5179  what does multiplanar ct reconstruction and su...   \n",
       "3542  what does axial mri image show intense enhance...   \n",
       "1022  what does sagittal mri image show the severe c...   \n",
       "\n",
       "                                                     fr  \n",
       "368                                            ct image  \n",
       "2547   combination of fat calcification and soft tissue  \n",
       "2598                                          axial mri  \n",
       "1154  the complete resection of the metastatic lesio...  \n",
       "2667                               post contrast images  \n",
       "727                                           ct thorax  \n",
       "4335                                    ct of the thigh  \n",
       "5179                                            vessels  \n",
       "3542                                        of the mass  \n",
       "1022                                        of the cord  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "lines = pd.DataFrame({'eng':train_questions, 'fr':train_answers})\n",
    "lines.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Clean\n",
    "#import string\n",
    "#exclude = set(string.punctuation)\n",
    "#lines.eng=lines.eng.apply(lambda x: ''.join(ch for ch in x if ch not in exclude))\n",
    "#lines.fr=lines.fr.apply(lambda x: ''.join(ch for ch in x if ch not in exclude))\n",
    "\n",
    "lines.fr = lines.fr.apply(lambda x : 'START_ '+ x + ' _END')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eng</th>\n",
       "      <th>fr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>what does mri show ?</td>\n",
       "      <td>START_ lesion at tail of pancreas _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>where does axial section mri abdomen show hypo...</td>\n",
       "      <td>START_ in distal pancreas _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>what do the arrows denote in the noncontrast c...</td>\n",
       "      <td>START_ complex fluid collection with layering ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>what was normal ?</td>\n",
       "      <td>START_ blood supply to the brain _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>what shows evidence of a contained rupture ?</td>\n",
       "      <td>START_ repeat ct scan of the abdomen _END</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 eng  \\\n",
       "0                               what does mri show ?   \n",
       "1  where does axial section mri abdomen show hypo...   \n",
       "2  what do the arrows denote in the noncontrast c...   \n",
       "3                                  what was normal ?   \n",
       "4       what shows evidence of a contained rupture ?   \n",
       "\n",
       "                                                  fr  \n",
       "0             START_ lesion at tail of pancreas _END  \n",
       "1                     START_ in distal pancreas _END  \n",
       "2  START_ complex fluid collection with layering ...  \n",
       "3              START_ blood supply to the brain _END  \n",
       "4          START_ repeat ct scan of the abdomen _END  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_eng_words=set()\n",
    "for eng in lines.eng:\n",
    "    for word in eng.split():\n",
    "        if word not in all_eng_words:\n",
    "            all_eng_words.add(word)\n",
    "    \n",
    "all_french_words=set()\n",
    "for fr in lines.fr:\n",
    "    for word in fr.split():\n",
    "        if word not in all_french_words:\n",
    "            all_french_words.add(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3332, 3270)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_eng_words), len(all_french_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Answers\n",
    "lenght_list=[]\n",
    "for l in lines.fr:\n",
    "    lenght_list.append(len(l.split(' ')))\n",
    "np.max(lenght_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Questions\n",
    "lenght_list=[]\n",
    "for l in lines.eng:\n",
    "    lenght_list.append(len(l.split(' ')))\n",
    "np.max(lenght_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_words = sorted(list(all_eng_words))\n",
    "target_words = sorted(list(all_french_words))\n",
    "num_encoder_tokens = len(all_eng_words)\n",
    "num_decoder_tokens = len(all_french_words)\n",
    "# del all_eng_words, all_french_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_token_index = dict(\n",
    "    [(word, i) for i, word in enumerate(input_words)])\n",
    "target_token_index = dict(\n",
    "    [(word, i) for i, word in enumerate(target_words)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "283208160"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lines.fr)*16*num_decoder_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_input_data = np.zeros(\n",
    "    (len(lines.eng), 29),\n",
    "    dtype='float32')\n",
    "decoder_input_data = np.zeros(\n",
    "    (len(lines.fr), 28),\n",
    "    dtype='float32')\n",
    "decoder_target_data = np.zeros(\n",
    "    (len(lines.fr), 28, num_decoder_tokens),\n",
    "    dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (input_text, target_text) in enumerate(zip(lines.eng, lines.fr)):\n",
    "    for t, word in enumerate(input_text.split()):\n",
    "        encoder_input_data[i, t] = input_token_index[word]\n",
    "    for t, word in enumerate(target_text.split()):\n",
    "        # decoder_target_data is ahead of decoder_input_data by one timestep\n",
    "        decoder_input_data[i, t] = target_token_index[word]\n",
    "        if t > 0:\n",
    "            # decoder_target_data will be ahead by one timestep\n",
    "            # and will not include the start character.\n",
    "            decoder_target_data[i, t - 1, target_token_index[word]] = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 50\n",
    "from keras.layers import Input, LSTM, Embedding, Dense\n",
    "from keras.models import Model\n",
    "from keras.utils import plot_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoder model\n",
    "encoder_inputs = Input(shape=(None,))\n",
    "en_x=  Embedding(num_encoder_tokens, embedding_size)(encoder_inputs)\n",
    "encoder = LSTM(50, return_state=True)\n",
    "encoder_outputs, state_h, state_c = encoder(en_x)\n",
    "# We discard `encoder_outputs` and only keep the states.\n",
    "encoder_states = [state_h, state_c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Decoder model\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "\n",
    "dex=  Embedding(num_decoder_tokens, embedding_size)\n",
    "\n",
    "final_dex= dex(decoder_inputs)\n",
    "\n",
    "\n",
    "decoder_lstm = LSTM(50, return_sequences=True, return_state=True)\n",
    "\n",
    "decoder_outputs, _, _ = decoder_lstm(final_dex,\n",
    "                                     initial_state=encoder_states)\n",
    "\n",
    "decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
    "\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 50)     166600      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, None, 50)     163500      input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, 50), (None,  20200       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   [(None, None, 50), ( 20200       embedding_2[0][0]                \n",
      "                                                                 lstm_1[0][1]                     \n",
      "                                                                 lstm_1[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, None, 3270)   166770      lstm_2[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 537,270\n",
      "Trainable params: 537,270\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "5413/5413 [==============================] - 10s 2ms/step - loss: 1.6767 - acc: 0.0343\n",
      "Epoch 2/5\n",
      "5413/5413 [==============================] - 7s 1ms/step - loss: 1.3921 - acc: 0.0357\n",
      "Epoch 3/5\n",
      "5413/5413 [==============================] - 7s 1ms/step - loss: 1.3667 - acc: 0.0357\n",
      "Epoch 4/5\n",
      "5413/5413 [==============================] - 7s 1ms/step - loss: 1.3452 - acc: 0.0373\n",
      "Epoch 5/5\n",
      "5413/5413 [==============================] - 7s 1ms/step - loss: 1.3262 - acc: 0.0397\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f907398bf60>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit([encoder_input_data, decoder_input_data], decoder_target_data,\n",
    "          batch_size=128,\n",
    "          epochs=5,\n",
    "          validation_split=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, None, 50)          166600    \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                [(None, 50), (None, 50),  20200     \n",
      "=================================================================\n",
      "Total params: 186,800\n",
      "Trainable params: 186,800\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoder_model = Model(encoder_inputs, encoder_states)\n",
    "encoder_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder_state_input_h = Input(shape=(50,))\n",
    "decoder_state_input_c = Input(shape=(50,))\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "\n",
    "final_dex2= dex(decoder_inputs)\n",
    "\n",
    "decoder_outputs2, state_h2, state_c2 = decoder_lstm(final_dex2, initial_state=decoder_states_inputs)\n",
    "decoder_states2 = [state_h2, state_c2]\n",
    "decoder_outputs2 = decoder_dense(decoder_outputs2)\n",
    "decoder_model = Model(\n",
    "    [decoder_inputs] + decoder_states_inputs,\n",
    "    [decoder_outputs2] + decoder_states2)\n",
    "\n",
    "# Reverse-lookup token index to decode sequences back to\n",
    "# something readable.\n",
    "reverse_input_char_index = dict(\n",
    "    (i, char) for char, i in input_token_index.items())\n",
    "reverse_target_char_index = dict(\n",
    "    (i, char) for char, i in target_token_index.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "    # Generate empty target sequence of length 1.\n",
    "    target_seq = np.zeros((1,1))\n",
    "    # Populate the first character of target sequence with the start character.\n",
    "    target_seq[0, 0] = target_token_index['START_']\n",
    "\n",
    "    # Sampling loop for a batch of sequences\n",
    "    # (to simplify, here we assume a batch of size 1).\n",
    "    stop_condition = False\n",
    "    decoded_sentence = ''\n",
    "    while not stop_condition:\n",
    "        output_tokens, h, c = decoder_model.predict(\n",
    "            [target_seq] + states_value)\n",
    "\n",
    "        # Sample a token\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_char = reverse_target_char_index[sampled_token_index]\n",
    "        decoded_sentence += ' '+sampled_char\n",
    "\n",
    "        # Exit condition: either hit max length\n",
    "        # or find stop character.\n",
    "        if (sampled_char == '_END' or\n",
    "           len(decoded_sentence) > 52):\n",
    "            stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1,1))\n",
    "        target_seq[0, 0] = sampled_token_index\n",
    "\n",
    "        # Update states\n",
    "        states_value = [h, c]\n",
    "\n",
    "    return decoded_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    what does mri show ?\n",
      "Name: eng, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(lines.eng[0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-\n",
      "Input sentence: what does mri show ?\n",
      "Actual sentence: START_ lesion at tail of pancreas _END\n",
      "Decoded sentence:  the the _END\n",
      "-\n",
      "Input sentence: where does axial section mri abdomen show hypoechoic mass ?\n",
      "Actual sentence: START_ in distal pancreas _END\n",
      "Decoded sentence:  the _END\n",
      "-\n",
      "Input sentence: what do the arrows denote in the noncontrast ct scan image of pelvis ?\n",
      "Actual sentence: START_ complex fluid collection with layering consistent with hematoma _END\n",
      "Decoded sentence:  the the _END\n",
      "-\n",
      "Input sentence: what was normal ?\n",
      "Actual sentence: START_ blood supply to the brain _END\n",
      "Decoded sentence:  the the the _END\n",
      "-\n",
      "Input sentence: what shows evidence of a contained rupture ?\n",
      "Actual sentence: START_ repeat ct scan of the abdomen _END\n",
      "Decoded sentence:  the the the _END\n",
      "-\n",
      "Input sentence: what does preoperative ct demonstrate ?\n",
      "Actual sentence: START_ severe loosening of the pedicle screws at l3 and l5 _END\n",
      "Decoded sentence:  the the the _END\n",
      "-\n",
      "Input sentence: what does the axial contracted CT section show ?\n",
      "Actual sentence: START_ hypodense nodular lesion in the right adrenal gland _END\n",
      "Decoded sentence:  the the _END\n",
      "-\n",
      "Input sentence: where does sagittal reformatted ct scan of the pelvis show a contrastfilled vagina ?\n",
      "Actual sentence: START_ separate and posterior to the urinary bladder _END\n",
      "Decoded sentence:  the _END\n",
      "-\n",
      "Input sentence: what does coronal ct scan demonstrate ?\n",
      "Actual sentence: START_ incisional lumbar hernia _END\n",
      "Decoded sentence:  the the _END\n",
      "-\n",
      "Input sentence: what shows site and size of infarct in three study patients ?\n",
      "Actual sentence: START_ magnetic resonance imaging _END\n",
      "Decoded sentence:  the the _END\n"
     ]
    }
   ],
   "source": [
    "for seq_index in range(10):\n",
    "    input_seq = encoder_input_data[seq_index: seq_index + 1]\n",
    "    decoded_sentence = decode_sequence(input_seq)\n",
    "    print('-')\n",
    "    print('Input sentence:', lines.eng[seq_index])\n",
    "    print('Actual sentence:', lines.fr[seq_index])\n",
    "    print('Decoded sentence:', decoded_sentence)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid data size= 500\n",
      "Train data size= 5413\n"
     ]
    }
   ],
   "source": [
    "\n",
    "questions_vocab = set()\n",
    "answers_vocab = set()\n",
    "\n",
    "for i in range(len(train_answers)):\n",
    "    train_answers[i] = \"<go> \"+train_answers[i]+\" <eos>\"\n",
    "    \n",
    "for q,a in zip(val_questions, val_answers):\n",
    "    for w in q.split():\n",
    "        if w not in questions_vocab:\n",
    "            questions_vocab.add(w)\n",
    "    for w in a.split():\n",
    "        if w not in answers_vocab:\n",
    "            answers_vocab.add(w)\n",
    "            \n",
    "\n",
    "for q,a in zip(train_questions, train_answers):\n",
    "    for w in q.split():\n",
    "        if w not in questions_vocab:\n",
    "            questions_vocab.add(w)\n",
    "    for w in a.split():\n",
    "        if w not in answers_vocab:\n",
    "            answers_vocab.add(w)\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<go> complex fluid collection with layering consistent with hematoma <eos>'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_answers[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#VQA-MED\n",
    "\n",
    "input_characters = sorted(list(questions_vocab))\n",
    "target_characters = sorted(list(answers_vocab))\n",
    "num_encoder_tokens = len(questions_vocab)\n",
    "num_decoder_tokens = len(answers_vocab)\n",
    "max_encoder_seq_length = max([len(txt.split()) for txt in train_questions])\n",
    "max_decoder_seq_length = max([len(txt.split()) for txt in train_answers])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input_words = sorted(list(input_characters))\n",
    "# target_words = sorted(list(target_characters))\n",
    "# num_encoder_tokens = len(input_characters)\n",
    "# num_decoder_tokens = len(target_characters)\n",
    "# max_encoder_seq_length = max([len(txt) for txt in input_texts])\n",
    "# max_decoder_seq_length = max([len(txt) for txt in target_texts])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples: 5413\n",
      "Number of unique input tokens: 3400\n",
      "Number of unique output tokens: 3381\n",
      "Max sequence length for inputs: 29\n",
      "Max sequence length for outputs: 28\n"
     ]
    }
   ],
   "source": [
    "print('Number of samples:', len(train_answers))\n",
    "print('Number of unique input tokens:', num_encoder_tokens)\n",
    "print('Number of unique output tokens:', num_decoder_tokens)\n",
    "print('Max sequence length for inputs:', max_encoder_seq_length)\n",
    "print('Max sequence length for outputs:', max_decoder_seq_length)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_token_index = dict(\n",
    "    [(char, i) for i, char in enumerate(questions_vocab)])\n",
    "target_token_index = dict(\n",
    "    [(char, i) for i, char in enumerate(answers_vocab)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ques_wor2tok = dict(\n",
    "    [(char, i) for i, char in enumerate(questions_vocab)])\n",
    "ans_wor2tok = dict(\n",
    "    [(char, i) for i, char in enumerate(answers_vocab)])\n",
    "\n",
    "\n",
    "ques_tok2wor = dict(\n",
    "    (i, char) for char, i in ques_wor2tok.items())\n",
    "ans_tok2wor = dict(\n",
    "    (i, char) for char, i in ans_wor2tok.items())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "ques_tok2wor[3400] = '<pad>'\n",
    "ans_tok2wor[3381] = '<pad>'\n",
    "\n",
    "ques_wor2tok['<pad>'] = 3400\n",
    "ans_wor2tok['<pad>'] = 3381"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'<pad>'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-313f9be9452e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mques_wor2tok\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"<pad>\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m: '<pad>'"
     ]
    }
   ],
   "source": [
    "ques_wor2tok[\"<pad>\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_input_data = np.zeros(\n",
    "    (len(train_answers), max_encoder_seq_length, num_encoder_tokens),\n",
    "    dtype='float32')\n",
    "decoder_input_data = np.zeros(\n",
    "    (len(train_answers), max_decoder_seq_length, num_decoder_tokens),\n",
    "    dtype='float32')\n",
    "decoder_target_data = np.zeros(\n",
    "    (len(train_answers), max_decoder_seq_length, num_decoder_tokens),\n",
    "    dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_input_data = np.zeros(\n",
    "    (len(train_answers), max_encoder_seq_length),\n",
    "    dtype='float32')\n",
    "decoder_input_data = np.zeros(\n",
    "    (len(train_answers), max_decoder_seq_length),\n",
    "    dtype='float32')\n",
    "decoder_target_data = np.zeros(\n",
    "    (len(train_answers), max_decoder_seq_length),\n",
    "    dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "ei = []\n",
    "di = []\n",
    "dt = []\n",
    "for ques, ans in zip(train_questions, train_answers):\n",
    "    ques_token = [ques_wor2tok[word] for word in ques.strip().split()]\n",
    "    ei.append(ques_token + [ques_wor2tok[\"<pad>\"]]* (max_encoder_seq_length - len(ques_token)))\n",
    "    \n",
    "    ans_token = [ans_wor2tok[word] for word in ans.strip().split()]\n",
    "    di.append(ans_token + [ans_wor2tok[\"<pad>\"]]* (max_decoder_seq_length - len(ans_token)))\n",
    "    \n",
    "    ans_token_dt = ans_token[1:]\n",
    "    dt.append(ans_token_dt + [ans_wor2tok[\"<pad>\"]]* (max_decoder_seq_length - len(ans_token_dt)))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lesion'"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans_tok2wor[di[0][1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):\n",
    "#     for t, char in enumerate(input_text):\n",
    "#         encoder_input_data[i, t, input_token_index[char]] = 1.\n",
    "#     for t, char in enumerate(target_text):\n",
    "#         # decoder_target_data is ahead of decoder_input_data by one timestep\n",
    "#         decoder_input_data[i, t, target_token_index[char]] = 1.\n",
    "#         if t > 0:\n",
    "#             # decoder_target_data will be ahead by one timestep\n",
    "#             # and will not include the start character.\n",
    "#             decoder_target_data[i, t - 1, target_token_index[char]] = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (ques, ans) in enumerate(zip(train_questions, train_answers)):\n",
    "    for t, char in enumerate(ques.split()):\n",
    "        encoder_input_data[i, t, input_token_index[char]] = 1.\n",
    "    for t, char in enumerate(ans.split()):\n",
    "        # decoder_target_data is ahead of decoder_input_data by one timestep\n",
    "        decoder_input_data[i, t, target_token_index[char]] = 1.\n",
    "        if t > 0:\n",
    "            # decoder_target_data will be ahead by one timestep\n",
    "            # and will not include the start character.\n",
    "            decoder_target_data[i, t - 1, target_token_index[char]] = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "object of type 'int' has no len()",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-93-7e6871b741ac>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mei\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: object of type 'int' has no len()"
     ]
    }
   ],
   "source": [
    "len(ei[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with tf.device(\"/gpu:0\"):\n",
    "# Define an input sequence and process it.\n",
    "encoder_inputs = Input(shape=(None,))\n",
    "x = Embedding(num_encoder_tokens, latent_dim)(encoder_inputs)\n",
    "x, state_h, state_c = LSTM(latent_dim,\n",
    "                           return_state=True)(x)\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "#encoder = LSTM(latent_dim, return_state=True)\n",
    "#    encoder_outputs, state_h, state_c = encoder(encoder_inputs)\n",
    "# We discard `encoder_outputs` and only keep the states.\n",
    "#encoder_states = [state_h, state_c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define an input sequence and process it.\n",
    "encoder_inputs = Input(shape=(None,))\n",
    "x = Embedding(num_encoder_tokens, latent_dim)(encoder_inputs)\n",
    "x, state_h, state_c = LSTM(latent_dim,\n",
    "                           return_state=True)(x)\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "# Set up the decoder, using `encoder_states` as initial state.\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "x = Embedding(num_decoder_tokens, latent_dim)(decoder_inputs)\n",
    "x = LSTM(latent_dim, return_sequences=True)(x, initial_state=encoder_states)\n",
    "decoder_outputs = Dense(num_decoder_tokens, activation='softmax')(x)\n",
    "\n",
    "# Define the model that will turn\n",
    "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with tf.device(\"/gpu:1\"):\n",
    "# Set up the decoder, using `encoder_states` as initial state.\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "# We set up our decoder to return full output sequences,\n",
    "# and to return internal states as well. We don't use the\n",
    "# return states in the training model, but we will use them in inference.\n",
    "x = Embedding(num_decoder_tokens, latent_dim)(decoder_inputs)\n",
    "x = LSTM(latent_dim, return_sequences=True)(x, initial_state=encoder_states)\n",
    "decoder_outputs = Dense(num_decoder_tokens, activation='softmax')(x)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)\n",
    "# decoder_outputs, _, _ = decoder_lstm(decoder_inputs,\n",
    "#                                  initial_state=encoder_states)\n",
    "# decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
    "# decoder_outputs = decoder_dense(decoder_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the model that will turn\n",
    "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "from keras.utils.training_utils import multi_gpu_model\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Error when checking target: expected dense_6 to have 3 dimensions, but got array with shape (5413, 28)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-87-96f6bfcf8658>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rmsprop'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'categorical_crossentropy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'mae'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mei\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1628\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1629\u001b[0m             \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1630\u001b[0;31m             batch_size=batch_size)\n\u001b[0m\u001b[1;32m   1631\u001b[0m         \u001b[0;31m# Prepare validation data.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1632\u001b[0m         \u001b[0mdo_validation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)\u001b[0m\n\u001b[1;32m   1478\u001b[0m                                     \u001b[0moutput_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1479\u001b[0m                                     \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1480\u001b[0;31m                                     exception_prefix='target')\n\u001b[0m\u001b[1;32m   1481\u001b[0m         sample_weights = _standardize_sample_weights(sample_weight,\n\u001b[1;32m   1482\u001b[0m                                                      self._feed_output_names)\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m    111\u001b[0m                         \u001b[0;34m': expected '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' to have '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m                         \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' dimensions, but got array '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m                         'with shape ' + str(data_shape))\n\u001b[0m\u001b[1;32m    114\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m                     \u001b[0mdata_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_shape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Error when checking target: expected dense_6 to have 3 dimensions, but got array with shape (5413, 28)"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['mae'])\n",
    "model.fit([np.array(ei), np.array(di)], np.array(dt), batch_size=batch_size, epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'lstm_25/transpose_1:0' shape=(?, ?, 256) dtype=float32>"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('s2s.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Layer lstm_2 expects 19 inputs, but it received 3 input tensors. Input received: [<tf.Tensor 'input_32:0' shape=(?, ?) dtype=float32>, <tf.Tensor 'input_45:0' shape=(?, ?, ?, 256) dtype=float32>, <tf.Tensor 'input_46:0' shape=(?, ?, ?, 256) dtype=float32>]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-119-3c0fdea38a71>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m decoder_outputs, state_h, state_c = decoder_lstm(\n\u001b[0;32m---> 18\u001b[0;31m     decoder_inputs, initial_state=decoder_states_inputs)\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0mdecoder_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstate_h\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate_c\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mdecoder_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecoder_dense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdecoder_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/layers/recurrent.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, initial_state, constants, **kwargs)\u001b[0m\n\u001b[1;32m    536\u001b[0m             \u001b[0moriginal_input_spec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_spec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_spec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfull_input_spec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 538\u001b[0;31m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRNN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfull_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    539\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_spec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moriginal_input_spec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/topology.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, **kwargs)\u001b[0m\n\u001b[1;32m    601\u001b[0m             \u001b[0;31m# Raise exceptions in case the input is not compatible\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m             \u001b[0;31m# with the input_spec set at build time.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 603\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massert_input_compatibility\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    604\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    605\u001b[0m             \u001b[0;31m# Handle mask propagation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/topology.py\u001b[0m in \u001b[0;36massert_input_compatibility\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m    460\u001b[0m                              \u001b[0;34m'but it received '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    461\u001b[0m                              \u001b[0;34m' input tensors. Input received: '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 462\u001b[0;31m                              str(inputs))\n\u001b[0m\u001b[1;32m    463\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0minput_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_spec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    464\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mspec\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Layer lstm_2 expects 19 inputs, but it received 3 input tensors. Input received: [<tf.Tensor 'input_32:0' shape=(?, ?) dtype=float32>, <tf.Tensor 'input_45:0' shape=(?, ?, ?, 256) dtype=float32>, <tf.Tensor 'input_46:0' shape=(?, ?, ?, 256) dtype=float32>]"
     ]
    }
   ],
   "source": [
    "# Next: inference mode (sampling).\n",
    "# Here's the drill:\n",
    "# 1) encode input and retrieve initial decoder state\n",
    "# 2) run one step of decoder with this initial state\n",
    "# and a \"start of sequence\" token as target.\n",
    "# Output will be the next target token\n",
    "# 3) Repeat with the current target token and current states\n",
    "\n",
    "# Define sampling models\n",
    "encoder_model = Model(encoder_inputs, encoder_states)\n",
    "\n",
    "decoder_state_input_h = Input(shape=(None, None, 256))\n",
    "decoder_state_input_c = Input(shape=(None, None, 256))\n",
    "\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "\n",
    "decoder_outputs, state_h, state_c = decoder_lstm(\n",
    "    decoder_inputs, initial_state=decoder_states_inputs)\n",
    "decoder_states = [state_h, state_c]\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "decoder_model = Model(\n",
    "    [decoder_inputs] + decoder_states_inputs,\n",
    "    [decoder_outputs] + decoder_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Reverse-lookup token index to decode sequences back to\n",
    "# something readable.\n",
    "reverse_input_char_index = dict(\n",
    "    (i, char) for char, i in input_token_index.items())\n",
    "reverse_target_char_index = dict(\n",
    "    (i, char) for char, i in target_token_index.items())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "\n",
    "    # Generate empty target sequence of length 1.\n",
    "    target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
    "    # Populate the first character of target sequence with the start character.\n",
    "    target_seq[0, 0, target_token_index['<go>']] = 1.\n",
    "\n",
    "    # Sampling loop for a batch of sequences\n",
    "    # (to simplify, here we assume a batch of size 1).\n",
    "    stop_condition = False\n",
    "    decoded_sentence = ''\n",
    "    while not stop_condition:\n",
    "        output_tokens, h, c = decoder_model.predict(\n",
    "            [target_seq] + states_value)\n",
    "\n",
    "        # Sample a token\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_char = reverse_target_char_index[sampled_token_index]\n",
    "        decoded_sentence += sampled_char+\" \"\n",
    "\n",
    "        # Exit condition: either hit max length\n",
    "        # or find stop character.\n",
    "        if (sampled_char == '<eos>' or\n",
    "           len(decoded_sentence.split()) > max_decoder_seq_length):\n",
    "            stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
    "        target_seq[0, 0, sampled_token_index] = 1.\n",
    "\n",
    "        # Update states\n",
    "        states_value = [h, c]\n",
    "\n",
    "    return decoded_sentence\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/tmp/tmp624pcnsr\n"
     ]
    }
   ],
   "source": [
    "from bleu import moses_multi_bleu\n",
    "bleus = moses_multi_bleu([\"Hello world Hello world\"], [\"Hello world Hello world\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 , out of 10\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'ndim'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-81-80773163c583>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m     model.fit([ei, di], dt,\n\u001b[1;32m      8\u001b[0m           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m           epochs=1)\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;36m5\u001b[0m \u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1628\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1629\u001b[0m             \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1630\u001b[0;31m             batch_size=batch_size)\n\u001b[0m\u001b[1;32m   1631\u001b[0m         \u001b[0;31m# Prepare validation data.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1632\u001b[0m         \u001b[0mdo_validation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)\u001b[0m\n\u001b[1;32m   1474\u001b[0m                                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_feed_input_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1475\u001b[0m                                     \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1476\u001b[0;31m                                     exception_prefix='input')\n\u001b[0m\u001b[1;32m   1477\u001b[0m         y = _standardize_input_data(y, self._feed_output_names,\n\u001b[1;32m   1478\u001b[0m                                     \u001b[0moutput_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m     74\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'DataFrame'\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/d2msa/virtualenv/tf_cuda9/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     74\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'DataFrame'\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'ndim'"
     ]
    }
   ],
   "source": [
    "epochs = 10\n",
    "# Run training\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['mae'])\n",
    "\n",
    "for i in range(epochs):\n",
    "    print(\"Epoch\",str(i),\", out of\", str(epochs))\n",
    "    model.fit([ei, di], dt,\n",
    "          batch_size=batch_size,\n",
    "          epochs=1)\n",
    "    \n",
    "    if (i+1)%5 ==0:\n",
    "        print(\"Testing ...\")\n",
    "        ref = []\n",
    "        pred = []\n",
    "        for seq_index in range(10):\n",
    "            \n",
    "            input_seq = encoder_input_data[seq_index: seq_index + 1]\n",
    "            decoded_sentence = decode_sequence(input_seq)\n",
    "            \n",
    "            reference = train_answers[seq_index].replace(\"<eos>\", \"\")\n",
    "            reference = reference.replace(\"<go>\", \"\")\n",
    "            ref.append(reference.strip())\n",
    "            \n",
    "            predicted = decoded_sentence.replace(\"<eos>\", \"\")\n",
    "            predicted = predicted.replace(\"<go>\", \"\")\n",
    "            pred.append(predicted.strip())\n",
    "            \n",
    "            if seq_index <=10:\n",
    "                print('\\n-')\n",
    "                print('Input Question  :', train_questions[seq_index])\n",
    "                print('Traget Answer   :', reference)\n",
    "                print('Predicted Answer:', predicted)\n",
    "                \n",
    "            \n",
    "        bleus = moses_multi_bleu(ref, pred)\n",
    "        print(\"Bleu:\", str(bleus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_decoder_seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}