test.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled7.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "go5ez0gq7OBs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7cda3b74-0778-4ffc-c2ba-2895696237b0"
      },
      "source": [
        "!pip install tensorflow_text"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow_text\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c0/ed/bbb51e9eccca0c2bfdf9df66e54cdff563b6f32daed9255da9b9a541368f/tensorflow_text-2.5.0-cp37-cp37m-manylinux1_x86_64.whl (4.3MB)\n",
            "\u001b[K     |████████████████████████████████| 4.3MB 4.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (0.12.0)\n",
            "Requirement already satisfied: tensorflow<2.6,>=2.5.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (2.5.0)\n",
            "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-hub>=0.8.0->tensorflow_text) (1.19.5)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-hub>=0.8.0->tensorflow_text) (3.12.4)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (1.1.2)\n",
            "Requirement already satisfied: tensorboard~=2.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (2.5.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (1.1.0)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (0.12.0)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (1.15.0)\n",
            "Requirement already satisfied: grpcio~=1.34.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (1.34.1)\n",
            "Requirement already satisfied: tensorflow-estimator<2.6.0,>=2.5.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (2.5.0)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (3.3.0)\n",
            "Requirement already satisfied: keras-nightly~=2.5.0.dev in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (2.5.0.dev2021032900)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (0.36.2)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (0.2.0)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (1.6.3)\n",
            "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (3.1.0)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (1.12)\n",
            "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (0.4.0)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (3.7.4.3)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text) (1.12.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.8.0->tensorflow-hub>=0.8.0->tensorflow_text) (57.0.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (0.6.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (1.0.1)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (1.31.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (2.23.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (3.3.4)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (1.8.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (0.4.4)\n",
            "Requirement already satisfied: cached-property; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow<2.6,>=2.5.0->tensorflow_text) (1.5.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (4.2.2)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (4.7.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (0.2.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (2.10)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (4.5.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (1.3.0)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.7/dist-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (0.4.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (3.4.1)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text) (3.1.1)\n",
            "Installing collected packages: tensorflow-text\n",
            "Successfully installed tensorflow-text-2.5.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j-J-39DUCZbE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eaa046bb-3e7e-4240-9531-a2d87e85c2ec"
      },
      "source": [
        "!pip install -q tf-models-official==2.4.0"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.1MB 4.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 6.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 215kB 36.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 102kB 9.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2MB 36.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 358kB 13.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 645kB 34.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 686kB 36.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 38.2MB 78kB/s \n",
            "\u001b[?25h  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for py-cpuinfo (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YOohqTd_C0N3"
      },
      "source": [
        "from official.nlp import bert"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9uXdit3Q8Blh"
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text\n",
        "\n",
        "# We'll generate plots of attention in order to see which parts of an image\n",
        "# our model focuses on during captioning\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Scikit-learn includes many helpful utilities\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.utils import shuffle\n",
        "\n",
        "import re\n",
        "import numpy as np\n",
        "import os\n",
        "import time\n",
        "import json\n",
        "from glob import glob\n",
        "from PIL import Image\n",
        "import pickle"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tRQe75uQyQU3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "314a649c-72e0-4fc9-be95-3bb21873ff80"
      },
      "source": [
        "print(\"Num GPUs Available: \", tf.config.list_physical_devices('GPU'))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Num GPUs Available:  [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "crcC56htzJbn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c1c6c33b-8566-485e-c996-6481793d3846"
      },
      "source": [
        "from tensorflow.python.client import device_lib\n",
        "print(device_lib.list_local_devices())"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[name: \"/device:CPU:0\"\n",
            "device_type: \"CPU\"\n",
            "memory_limit: 268435456\n",
            "locality {\n",
            "}\n",
            "incarnation: 701163046018305930\n",
            ", name: \"/device:GPU:0\"\n",
            "device_type: \"GPU\"\n",
            "memory_limit: 11344216064\n",
            "locality {\n",
            "  bus_id: 1\n",
            "  links {\n",
            "  }\n",
            "}\n",
            "incarnation: 5390908927222675215\n",
            "physical_device_desc: \"device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7\"\n",
            "]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "udmJN-IHFSRN"
      },
      "source": [
        "annotation_zip = tf.keras.utils.get_file('captions.zip',\n",
        "                                          cache_subdir=os.path.abspath('.'),\n",
        "                                          origin = 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',\n",
        "                                          extract = True)\n",
        "annotation_file = os.path.dirname(annotation_zip)+'/annotations/captions_train2014.json'"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sKjbC6xNFb_M"
      },
      "source": [
        "name_of_zip = 'train2014.zip'\n",
        "if not os.path.exists(os.path.abspath('.') + '/' + name_of_zip):\n",
        "  image_zip = tf.keras.utils.get_file(name_of_zip,\n",
        "                                      cache_subdir=os.path.abspath('.'),\n",
        "                                      origin = 'http://images.cocodataset.org/zips/train2014.zip',\n",
        "                                      extract = True)\n",
        "  PATH = os.path.dirname(image_zip)+'/train2014/'\n",
        "else:\n",
        "\n",
        "  PATH = os.path.abspath('.')+'/train2014/'"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "roEeJAMoJSkG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2c429577-6e4f-425c-a854-60b50cadd0d1"
      },
      "source": [
        "print(PATH)\n",
        "!wget https://s3.amazonaws.com/cvmlp/vqa/mscoco/vqa/v2_Questions_Train_mscoco.zip\n",
        "! unzip -a v2_Questions_Train_mscoco.zip\n",
        "!wget https://s3.amazonaws.com/cvmlp/vqa/mscoco/vqa/v2_Annotations_Train_mscoco.zip\n",
        "! unzip -a v2_Annotations_Train_mscoco.zip"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/train2014/\n",
            "--2021-06-18 09:25:56--  https://s3.amazonaws.com/cvmlp/vqa/mscoco/vqa/v2_Questions_Train_mscoco.zip\n",
            "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.12.118\n",
            "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.12.118|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 7239401 (6.9M) [application/zip]\n",
            "Saving to: ‘v2_Questions_Train_mscoco.zip.2’\n",
            "\n",
            "v2_Questions_Train_ 100%[===================>]   6.90M  4.25MB/s    in 1.6s    \n",
            "\n",
            "2021-06-18 09:25:58 (4.25 MB/s) - ‘v2_Questions_Train_mscoco.zip.2’ saved [7239401/7239401]\n",
            "\n",
            "Archive:  v2_Questions_Train_mscoco.zip\n",
            "replace v2_OpenEnded_mscoco_train2014_questions.json? [y]es, [n]o, [A]ll, [N]one, [r]ename: y\n",
            "  inflating: v2_OpenEnded_mscoco_train2014_questions.json  [text]  \n",
            "--2021-06-18 09:26:54--  https://s3.amazonaws.com/cvmlp/vqa/mscoco/vqa/v2_Annotations_Train_mscoco.zip\n",
            "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.107.166\n",
            "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.107.166|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 21708861 (21M) [application/zip]\n",
            "Saving to: ‘v2_Annotations_Train_mscoco.zip.2’\n",
            "\n",
            "v2_Annotations_Trai 100%[===================>]  20.70M  7.56MB/s    in 2.7s    \n",
            "\n",
            "2021-06-18 09:26:58 (7.56 MB/s) - ‘v2_Annotations_Train_mscoco.zip.2’ saved [21708861/21708861]\n",
            "\n",
            "Archive:  v2_Annotations_Train_mscoco.zip\n",
            "replace v2_mscoco_train2014_annotations.json? [y]es, [n]o, [A]ll, [N]one, [r]ename: y\n",
            "  inflating: v2_mscoco_train2014_annotations.json  [text]  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oG1RVFrAIVVG"
      },
      "source": [
        "import collections\n",
        "import operator\n",
        "# read the json file\n",
        "annotation_file = 'v2_mscoco_train2014_annotations.json'\n",
        "with open(annotation_file, 'r') as f:\n",
        "    annotations = json.load(f)\n",
        "\n",
        "# storing the captions and the image name in vectors\n",
        "all_answers = []\n",
        "all_answers_qids = []\n",
        "all_img_name_vector = []\n",
        "all_accepted_answers = []\n",
        "for annot in annotations['annotations']:\n",
        "#     print(annot)\n",
        "#     break\n",
        "    ans_dic = collections.defaultdict(int)\n",
        "    for each in annot['answers']:\n",
        "      diffans = each['answer']\n",
        "      if diffans in ans_dic:\n",
        "        #print(each['answer_confidence'])\n",
        "        if each['answer_confidence']=='yes':\n",
        "          ans_dic[diffans]+=4\n",
        "        if each['answer_confidence']=='maybe':\n",
        "          ans_dic[diffans]+= 2\n",
        "        if each['answer_confidence']=='no':\n",
        "          ans_dic[diffans]+= 1\n",
        "      else:\n",
        "        if each['answer_confidence']=='yes':\n",
        "          ans_dic[diffans]= 4\n",
        "        if each['answer_confidence']=='maybe':\n",
        "          ans_dic[diffans]= 2\n",
        "        if each['answer_confidence']=='no':\n",
        "          ans_dic[diffans]= 1\n",
        "#     print(ans_dic.items()) \n",
        "#     break\n",
        "    all_accepted_answers.append(['~ '+a[0] +' `' for a in ans_dic.items()])\n",
        "    most_fav = max(ans_dic.items(), key=operator.itemgetter(1))[0]\n",
        "    #print(most_fav)\n",
        "    caption = '~ ' + most_fav + ' `' #each['answer']\n",
        "    \n",
        "    image_id = annot['image_id']\n",
        "    question_id = annot['question_id']\n",
        "    full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n",
        "\n",
        "    all_img_name_vector.append(full_coco_image_path)\n",
        "    all_answers.append(caption)\n",
        "    all_answers_qids.append(question_id)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V4TO9ooBwDoH"
      },
      "source": [
        "# read the json file\n",
        "question_file = 'v2_OpenEnded_mscoco_train2014_questions.json'\n",
        "with open(question_file, 'r') as f:\n",
        "    questions = json.load(f)\n",
        "\n",
        "# storing the captions and the image name in vectors\n",
        "question_ids =[]\n",
        "all_questions = []\n",
        "all_img_name_vector_2 = []\n",
        "\n",
        "for annot in questions['questions']:\n",
        "    caption = annot['question']\n",
        "    image_id = annot['image_id']\n",
        "    full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n",
        "\n",
        "    all_img_name_vector_2.append(full_coco_image_path)\n",
        "    all_questions.append(caption)\n",
        "    question_ids.append(annot['question_id'])"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NAHeIJ8V7a4n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a76d0908-a692-4ce8-8ae5-ed89ab268b03"
      },
      "source": [
        "bert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8' \n",
        "\n",
        "map_name_to_handle = {\n",
        "    'bert_en_uncased_L-12_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3',\n",
        "    'bert_en_cased_L-12_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3',\n",
        "    'bert_multi_cased_L-12_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3',\n",
        "    'small_bert/bert_en_uncased_L-2_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1',\n",
        "    'small_bert/bert_en_uncased_L-2_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1',\n",
        "    'small_bert/bert_en_uncased_L-2_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1',\n",
        "    'small_bert/bert_en_uncased_L-2_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1',\n",
        "    'small_bert/bert_en_uncased_L-4_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1',\n",
        "    'small_bert/bert_en_uncased_L-4_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1',\n",
        "    'small_bert/bert_en_uncased_L-4_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1',\n",
        "    'small_bert/bert_en_uncased_L-4_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1',\n",
        "    'small_bert/bert_en_uncased_L-6_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1',\n",
        "    'small_bert/bert_en_uncased_L-6_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1',\n",
        "    'small_bert/bert_en_uncased_L-6_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1',\n",
        "    'small_bert/bert_en_uncased_L-6_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1',\n",
        "    'small_bert/bert_en_uncased_L-8_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1',\n",
        "    'small_bert/bert_en_uncased_L-8_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1',\n",
        "    'small_bert/bert_en_uncased_L-8_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1',\n",
        "    'small_bert/bert_en_uncased_L-8_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1',\n",
        "    'small_bert/bert_en_uncased_L-10_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1',\n",
        "    'small_bert/bert_en_uncased_L-10_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1',\n",
        "    'small_bert/bert_en_uncased_L-10_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1',\n",
        "    'small_bert/bert_en_uncased_L-10_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1',\n",
        "    'small_bert/bert_en_uncased_L-12_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1',\n",
        "    'small_bert/bert_en_uncased_L-12_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1',\n",
        "    'small_bert/bert_en_uncased_L-12_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1',\n",
        "    'small_bert/bert_en_uncased_L-12_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1',\n",
        "    'albert_en_base':\n",
        "        'https://tfhub.dev/tensorflow/albert_en_base/2',\n",
        "    'electra_small':\n",
        "        'https://tfhub.dev/google/electra_small/2',\n",
        "    'electra_base':\n",
        "        'https://tfhub.dev/google/electra_base/2',\n",
        "    'experts_pubmed':\n",
        "        'https://tfhub.dev/google/experts/bert/pubmed/2',\n",
        "    'experts_wiki_books':\n",
        "        'https://tfhub.dev/google/experts/bert/wiki_books/2',\n",
        "    'talking-heads_base':\n",
        "        'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1',\n",
        "}\n",
        "\n",
        "map_model_to_preprocess = {\n",
        "    'bert_en_uncased_L-12_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'bert_en_cased_L-12_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-2_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-2_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-2_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-2_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-4_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-4_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-4_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-4_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-6_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-6_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-6_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-6_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-8_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-8_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-8_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-8_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-10_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-10_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-10_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-10_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-12_H-128_A-2':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-12_H-256_A-4':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-12_H-512_A-8':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'small_bert/bert_en_uncased_L-12_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'bert_multi_cased_L-12_H-768_A-12':\n",
        "        'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3',\n",
        "    'albert_en_base':\n",
        "        'https://tfhub.dev/tensorflow/albert_en_preprocess/3',\n",
        "    'electra_small':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'electra_base':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'experts_pubmed':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'experts_wiki_books':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "    'talking-heads_base':\n",
        "        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n",
        "}\n",
        "\n",
        "tfhub_handle_encoder = map_name_to_handle[bert_model_name]\n",
        "tfhub_handle_preprocess = map_model_to_preprocess[bert_model_name]\n",
        "\n",
        "print(f'BERT model selected           : {tfhub_handle_encoder}')\n",
        "print(f'Preprocess model auto-selected: {tfhub_handle_preprocess}')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BERT model selected           : https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1\n",
            "Preprocess model auto-selected: https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AaFmJp4V7mw5"
      },
      "source": [
        "bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EAUQ42-RzKZl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "218f426f-b8ea-4e08-faff-63a8847ef713"
      },
      "source": [
        "text_preprocessed = bert_preprocess_model(all_questions[:10])\n",
        "\n",
        "print(f'Keys       : {list(text_preprocessed.keys())}')\n",
        "print(f'Shape      : {text_preprocessed[\"input_word_ids\"].shape}')\n",
        "print(f'Word Ids   : {text_preprocessed[\"input_word_ids\"][0, :12]}')\n",
        "print(f'Input Mask : {text_preprocessed[\"input_mask\"][0, :12]}')\n",
        "print(f'Type Ids   : {text_preprocessed[\"input_type_ids\"][0, :12]}')\n",
        "\n",
        "word_ids = text_preprocessed[\"input_word_ids\"]"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Keys       : ['input_word_ids', 'input_mask', 'input_type_ids']\n",
            "Shape      : (10, 128)\n",
            "Word Ids   : [ 101 2054 2003 2023 6302 2579 2559 2083 1029  102    0    0]\n",
            "Input Mask : [1 1 1 1 1 1 1 1 1 1 0 0]\n",
            "Type Ids   : [0 0 0 0 0 0 0 0 0 0 0 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pvTcHwxN8Wkn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "454dce68-ac19-4007-c240-f501b68f7fa3"
      },
      "source": [
        "text_preprocessed[\"input_word_ids\"]"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10, 128), dtype=int32, numpy=\n",
              "array([[ 101, 2054, 2003, ...,    0,    0,    0],\n",
              "       [ 101, 2054, 2597, ...,    0,    0,    0],\n",
              "       [ 101, 2054, 3609, ...,    0,    0,    0],\n",
              "       ...,\n",
              "       [ 101, 2054, 2003, ...,    0,    0,    0],\n",
              "       [ 101, 2003, 1996, ...,    0,    0,    0],\n",
              "       [ 101, 2003, 1996, ...,    0,    0,    0]], dtype=int32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Khxg4s0y8vXK"
      },
      "source": [
        "bert_model = hub.KerasLayer(tfhub_handle_encoder)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jm_jKaY6F4w7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3703736-769f-41c5-ebfd-f04550768d9e"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.7.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.5.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (5.4.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: huggingface-hub==0.0.8 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6At5RZ0jauBm",
        "outputId": "c95e40da-fd14-44f5-aa74-b758b371d4d9"
      },
      "source": [
        "print(all_questions[:10])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['What is this photo taken looking through?', 'What position is this man playing?', 'What color is the players shirt?', 'Is this man a professional baseball player?', 'What color is the snow?', 'What is the person doing?', 'What color is the persons headwear?', \"What is in the person's hand?\", 'Is the dog waiting?', 'Is the dog looking at a tennis ball or frisbee?']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g9yhCcgsF7OS",
        "outputId": "d6cdc754-9a0a-4efb-f0df-fefbb428df96"
      },
      "source": [
        "from transformers import BertTokenizer, BertModel\n",
        "tz = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "text = [tz.convert_tokens_to_ids(tz.tokenize(all_questions[i])) for i in range(10)]\n",
        "tokens = tz.convert_ids_to_tokens([1066,2509,2054])\n",
        "print(text)\n",
        "print(word_ids)\n",
        "print(tokens)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[2054, 2003, 2023, 6302, 2579, 2559, 2083, 1029], [2054, 2597, 2003, 2023, 2158, 2652, 1029], [2054, 3609, 2003, 1996, 2867, 3797, 1029], [2003, 2023, 2158, 1037, 2658, 3598, 2447, 1029], [2054, 3609, 2003, 1996, 4586, 1029], [2054, 2003, 1996, 2711, 2725, 1029], [2054, 3609, 2003, 1996, 5381, 2132, 16689, 1029], [2054, 2003, 1999, 1996, 2711, 1005, 1055, 2192, 1029], [2003, 1996, 3899, 3403, 1029], [2003, 1996, 3899, 2559, 2012, 1037, 5093, 3608, 2030, 10424, 2483, 11306, 1029]]\n",
            "tf.Tensor(\n",
            "[[ 101 2054 2003 ...    0    0    0]\n",
            " [ 101 2054 2597 ...    0    0    0]\n",
            " [ 101 2054 3609 ...    0    0    0]\n",
            " ...\n",
            " [ 101 2054 2003 ...    0    0    0]\n",
            " [ 101 2003 1996 ...    0    0    0]\n",
            " [ 101 2003 1996 ...    0    0    0]], shape=(10, 128), dtype=int32)\n",
            "['~', '##3', 'what']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QIB1q9j981jf",
        "outputId": "88e957c7-4060-4493-a30f-c9bea7efef69"
      },
      "source": [
        "bert_results = bert_model(text_preprocessed)\n",
        "\n",
        "print(f'Loaded BERT: {tfhub_handle_encoder}')\n",
        "print(f'Pooled Outputs Shape:{bert_results[\"pooled_output\"].shape}')\n",
        "print(f'Pooled Outputs Values:{bert_results[\"pooled_output\"][0, :12]}')\n",
        "print(f'Sequence Outputs Shape:{bert_results[\"sequence_output\"].shape}')\n",
        "print(f'Sequence Outputs Values:{bert_results[\"sequence_output\"][0, :12]}')\n",
        "\n",
        "final_output = tf.keras.layers.Dense(1024,activation='relu')(bert_results[\"pooled_output\"])\n",
        "print(final_output)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded BERT: https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1\n",
            "Pooled Outputs Shape:(10, 512)\n",
            "Pooled Outputs Values:[ 0.9864783   0.929076    0.0477565   0.23734523  0.18002531  0.35409155\n",
            "  0.99114573 -0.9915475  -0.09610949 -0.8664898  -0.27757415 -0.99276334]\n",
            "Sequence Outputs Shape:(10, 128, 512)\n",
            "Sequence Outputs Values:[[-0.27141997  0.06498506  0.9456969  ... -1.2558293   1.1619257\n",
            "   0.74685216]\n",
            " [-0.18597859 -0.39936352  0.07471289 ...  0.40324527 -0.2771676\n",
            "   0.28966343]\n",
            " [ 0.2401818   0.01206819  0.20339519 ... -0.84410834 -0.24459103\n",
            "   0.00836066]\n",
            " ...\n",
            " [ 0.07787731 -0.13147     1.1650044  ... -0.78934014  1.3759407\n",
            "   0.8701567 ]\n",
            " [-0.03311682 -0.46155715 -0.04248394 ... -0.03216039  1.2840161\n",
            "   0.30595353]\n",
            " [-0.04731586 -0.38663828  0.03326248 ... -0.01138435  1.2326314\n",
            "   0.17593908]]\n",
            "tf.Tensor(\n",
            "[[0.17423135 0.31981978 0.         ... 0.         0.         0.        ]\n",
            " [0.08318952 0.5416185  0.         ... 0.         0.         0.        ]\n",
            " [0.39461094 0.3509038  0.         ... 0.         0.         0.        ]\n",
            " ...\n",
            " [0.00402315 0.45896953 0.         ... 0.         0.04866374 0.        ]\n",
            " [0.14871545 0.17982769 0.         ... 0.         0.07363455 0.        ]\n",
            " [0.3151893  0.         0.         ... 0.         0.25284034 0.        ]], shape=(10, 1024), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "meKTcnAD4_fj",
        "outputId": "e356a790-db49-497f-a3b3-02a21f56d681"
      },
      "source": [
        "# shuffling the captions and image_names together\n",
        "# setting a random state\n",
        "\n",
        "\n",
        "train_answers, train_questions, img_name_vector,train_accepted_answers = shuffle(all_answers,all_questions,\n",
        "                                          all_img_name_vector,all_accepted_answers,\n",
        "                                          random_state=1)\n",
        "\n",
        "# selecting the first 30000 captions from the shuffled set\n",
        "num_examples = 1000\n",
        "train_answers = train_answers[:num_examples]\n",
        "train_questions = train_questions[:num_examples]\n",
        "img_name_vector = img_name_vector[:num_examples]\n",
        "train_accepted_answers = train_accepted_answers[:num_examples]\n",
        "print(img_name_vector[0],train_questions[0],train_answers[0])\n",
        "\n",
        "print(len(img_name_vector),len(train_questions),len(train_answers))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/train2014/COCO_train2014_000000573744.jpg Is this a happy animal? ~ yes `\n",
            "1000 1000 1000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NpxWtNyHzbNx",
        "outputId": "8e2b36ff-1aaf-49e2-8d7c-8c5ec65019bb"
      },
      "source": [
        "print(train_answers[:50])"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['~ yes `', '~ arabic `', '~ lj 33472 `', '~ yes `', '~ sheep `', '~ tennis racket `', '~ apartment `', '~ no `', '~ no `', '~ no `', '~ cow `', '~ no `', '~ car `', '~ bed `', '~ yes `', '~ 0 `', '~ panda `', '~ no `', '~ brown `', '~ no `', '~ 1:25 `', '~ fat cats `', '~ yes `', '~ baseball `', '~ sink `', '~ no `', '~ no `', '~ yes `', '~ yes `', '~ yes `', '~ orange `', '~ no `', '~ fish `', '~ yes `', '~ star `', '~ yes `', '~ none `', '~ yellow `', '~ school `', '~ 2 `', '~ no `', '~ lancashire `', '~ broccoli `', '~ no `', '~ house `', '~ bananas `', '~ american eagle `', '~ arrow `', '~ 100 `', '~ playing wii `']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g8J19kOb5Tpe"
      },
      "source": [
        "def load_image(image_path):\n",
        "    img = tf.io.read_file(image_path)\n",
        "    img = tf.image.decode_jpeg(img, channels=3)\n",
        "    #224 x 224 for VGG 299x299 for Inception\n",
        "    img = tf.image.resize(img, (224, 224))\n",
        "    img = tf.keras.applications.inception_v3.preprocess_input(img)\n",
        "    return img, image_path"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RU9mDHt95UiK"
      },
      "source": [
        "image_model = tf.keras.applications.VGG16(include_top=False,\n",
        "                                                weights='imagenet',input_shape = (224,224,3))\n",
        "new_input = image_model.input\n",
        "hidden_layer = image_model.layers[-1].output\n",
        "\n",
        "image_features_extract_model = tf.keras.Model(new_input, hidden_layer)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0vgswpVP5hyS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5e4bd6d1-48aa-467a-bc94-b71eebd995cf"
      },
      "source": [
        "encode_train = sorted(set(img_name_vector))\n",
        "\n",
        "# feel free to change the batch_size according to your system configuration\n",
        "image_dataset = tf.data.Dataset.from_tensor_slices(encode_train)\n",
        "image_dataset = image_dataset.map(\n",
        "  load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(16)\n",
        "\n",
        "for img, path in image_dataset:\n",
        "  batch_features = image_features_extract_model(img)\n",
        "  #print(batch_features.shape)\n",
        "  batch_features = tf.reshape(batch_features,\n",
        "                              (batch_features.shape[0], -1, batch_features.shape[3]))\n",
        "  print(batch_features.shape)\n",
        "\n",
        "  for bf, p in zip(batch_features, path):\n",
        "    path_of_feature = p.numpy().decode(\"utf-8\")\n",
        "    np.save(path_of_feature, bf.numpy())"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(16, 49, 512)\n",
            "(14, 49, 512)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FO6MUBkA7JFw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d5620801-f8b4-4f8f-897f-6271d84177d8"
      },
      "source": [
        "y = bert_preprocess_model(train_answers)\n",
        "print(f'Shape      : {y[\"input_word_ids\"].shape}')\n",
        "print(f'Word Ids   : {y[\"input_word_ids\"][0, :12]}')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape      : (1000, 128)\n",
            "Word Ids   : [ 101 1066 2748 1036  102    0    0    0    0    0    0    0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ly5iYLPpzMdI",
        "outputId": "ae817561-fe06-4867-f67a-b24fb41cd03c"
      },
      "source": [
        "print(f'Shape      : {y[\"input_word_ids\"].shape}')\n",
        "print(f'Word Ids   : {y[\"input_word_ids\"][:3, :12]}')"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape      : (1000, 128)\n",
            "Word Ids   : [[  101  1066  2748  1036   102     0     0     0     0     0     0     0]\n",
            " [  101  1066  5640  1036   102     0     0     0     0     0     0     0]\n",
            " [  101  1066  1048  3501 29562  2581  2475  1036   102     0     0     0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2GIJm6Hl9JY8"
      },
      "source": [
        "img_name_train, img_name_val, question_train, question_val,answer_train, answer_val  = train_test_split(img_name_vector,\n",
        "                                                                    train_questions,\n",
        "                                                                    train_answers,\n",
        "                                                                    test_size=0.1,\n",
        "                                                                    random_state=0)"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BjPxrynJMcNt",
        "outputId": "ad5e1378-d067-43cc-fad4-299b7324884b"
      },
      "source": [
        "print(img_name_train[0])\n",
        "print(question_train[0])\n",
        "print(answer_train[0])"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/train2014/COCO_train2014_000000081930.jpg\n",
            "Is this picture taken in winter?\n",
            "~ no `\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IVyVcaWC9oEC",
        "outputId": "573a0e81-700a-418e-d5fd-c962d69d21db"
      },
      "source": [
        "print(len(img_name_train), len(img_name_val), len(question_train), len(question_val),len(answer_train), len(answer_val))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "900 100 900 100 900 100\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "74owYhjrEubU",
        "outputId": "a86f4cff-0932-4a45-b6c6-bb63afe88729"
      },
      "source": [
        "#type(img_name_train)\n",
        "#type(question_train)\n",
        "type(answer_train)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "list"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-YkuMIDtewB9",
        "outputId": "5fc2b077-81f1-4e7d-8e77-2799bd5f6b05"
      },
      "source": [
        "test_tokenize_txt = train_answers[:10]\n",
        "test_tokenized = [tz.encode_plus(\n",
        "    text=test_tokenize_txt[i],  # the sentence to be encoded\n",
        "    max_length = 128,  # maximum length of a sentence\n",
        "    add_special_tokens=False,\n",
        "    pad_to_max_length=True,  # Add [PAD]s\n",
        "    return_attention_mask = True,  # Generate the attention mask  # ask the function to return PyTorch tensors\n",
        ")['input_ids'] for i in range(10)]\n",
        "print(test_tokenized)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1066, 2748, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 5640, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 1048, 3501, 29562, 2581, 2475, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 2748, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 8351, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 5093, 14513, 3388, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 4545, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 2053, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 2053, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1066, 2053, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2111: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d4UFOXmE9vdx"
      },
      "source": [
        "BATCH_SIZE = 32 #2 #64\n",
        "BUFFER_SIZE = 300 #1000\n",
        "num_steps = len(img_name_train) // BATCH_SIZE\n",
        "# shape of the vector extracted from VGG is (49, 512)\n",
        "# these two variables represent that\n",
        "features_shape = 512\n",
        "attention_features_shape = 49"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BssbQ3EmDNK9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "05cbf0a7-f50f-418e-f85a-c731a5e71413"
      },
      "source": [
        "#question_train = bert_preprocess_model(question_train)[\"input_word_ids\"]\n",
        "#question_val = bert_preprocess_model(question_val)[\"input_word_ids\"]\n",
        "answer_train =  [tz.encode_plus(\n",
        "    text=answer_train[i],  # the sentence to be encoded\n",
        "    max_length = 128,  # maximum length of a sentence\n",
        "    add_special_tokens=False,\n",
        "    pad_to_max_length=True,  # Add [PAD]s\n",
        "    return_attention_mask = True,  # Generate the attention mask  # ask the function to return PyTorch tensors\n",
        ")['input_ids'] for i in range(len(answer_train))]#bert_preprocess_model(answer_train)[\"input_word_ids\"]\n",
        "answer_val = [tz.encode_plus(\n",
        "    text=answer_val[i],  # the sentence to be encoded\n",
        "    max_length = 128,  # maximum length of a sentence\n",
        "    add_special_tokens=False,\n",
        "    pad_to_max_length=True,  # Add [PAD]s\n",
        "    return_attention_mask = True,  # Generate the attention mask  # ask the function to return PyTorch tensors\n",
        ")['input_ids'] for i in range(len(answer_val))]#bert_preprocess_model(answer_val)[\"input_word_ids\"]"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2111: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eyxz4aR4HLI9"
      },
      "source": [
        "#question_train = tf.cast(question_train,dtype=tf.float32)\n",
        "#question_val = tf.cast(question_val,dtype=tf.float32)\n",
        "answer_train = tf.cast(answer_train,dtype=tf.float32)\n",
        "answer_val = tf.cast(answer_val,dtype=tf.float32)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qacFcPyGH2Q5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "89e4bc66-e084-44aa-ec45-c90b312e6bec"
      },
      "source": [
        "print(answer_val)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[[ 1066.  1018.  1036. ...     0.     0.     0.]\n",
            " [ 1066.  2748.  1036. ...     0.     0.     0.]\n",
            " [ 1066.  9610. 14536. ...     0.     0.     0.]\n",
            " ...\n",
            " [ 1066.  2748.  1036. ...     0.     0.     0.]\n",
            " [ 1066.  4153.  1036. ...     0.     0.     0.]\n",
            " [ 1066. 10777.  1036. ...     0.     0.     0.]], shape=(100, 128), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BNTgONSQ-cmz"
      },
      "source": [
        "def map_func(img_name, cap,ans):\n",
        "  img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n",
        "  return img_tensor, cap,ans"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "afLX2U0OD-HC"
      },
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices((img_name_train, question_train, answer_train))\n",
        "\n",
        "dataset = dataset.map(lambda item1, item2, item3: tf.numpy_function(map_func, [item1, item2, item3], [tf.float32, tf.string, tf.float32]),\n",
        "          num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
        "dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kCue8H_QIXE6",
        "outputId": "28df67bb-d49d-41d5-b39c-397891eb1468"
      },
      "source": [
        "i = 0\n",
        "for ele1,ele2,ele3 in dataset:\n",
        "  if i != 1:\n",
        "    print(ele1.shape)\n",
        "    print(ele2.shape)\n",
        "    print(ele3.shape)\n",
        "    i += 1\n",
        "  else:\n",
        "    break"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(32, 49, 512)\n",
            "(32,)\n",
            "(32, 128)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "35NuQdwlNFt0"
      },
      "source": [
        "def gru(units):\n",
        "  # If you have a GPU, we recommend using CuDNNGRU(provides a 3x speedup than GRU)\n",
        "  # the code automatically does that.\n",
        "  return tf.keras.layers.GRU(units, \n",
        "                               return_sequences=True, \n",
        "                               return_state=True, \n",
        "                               recurrent_activation='sigmoid', \n",
        "                               recurrent_initializer='glorot_uniform')"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nen1Osk46hhN"
      },
      "source": [
        "class Encoder(tf.keras.Model):\n",
        "    def __init__(self):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.bert_preprocess = hub.KerasLayer(tfhub_handle_preprocess)\n",
        "        self.bert_model = hub.KerasLayer(tfhub_handle_encoder)\n",
        "        \n",
        "    def call(self, x, hidden):#hidden shape = (b,49,512)\n",
        "        x = self.bert_preprocess(x)\n",
        "        x = self.bert_model(x)[\"pooled_output\"]\n",
        "        return hidden, x"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jk1rsdW59lVc"
      },
      "source": [
        "class Decoder(tf.keras.Model):\n",
        "    def __init__(self, vocab_size=30522, dec_units=512):\n",
        "        super(Decoder, self).__init__()\n",
        "        self.dec_units = dec_units\n",
        "        self.bert_preprocess = hub.KerasLayer(tfhub_handle_preprocess)\n",
        "        self.bert_model = hub.KerasLayer(tfhub_handle_encoder)\n",
        "        self.gru = gru(self.dec_units)\n",
        "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
        "        \n",
        "        # used for attention\n",
        "        self.W1 = tf.keras.layers.Dense(self.dec_units)\n",
        "        self.W2 = tf.keras.layers.Dense(self.dec_units)\n",
        "        self.V = tf.keras.layers.Dense(1)\n",
        "        \n",
        "    def call(self, x, hidden, enc_output):\n",
        "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
        "        \n",
        "        # hidden shape == (batch_size, hidden size)\n",
        "        # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
        "        # we are doing this to perform addition to calculate the score\n",
        "        print(\"Here 1\")\n",
        "        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
        "        \n",
        "        print(\"Here 2\")\n",
        "        # score shape == (batch_size, max_length, hidden_size)\n",
        "        score = tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis))\n",
        "        \n",
        "        print(\"Here 3\")\n",
        "        # attention_weights shape == (batch_size, max_length, 1)\n",
        "        # we get 1 at the last axis because we are applying score to self.V\n",
        "        attention_weights = tf.nn.softmax(self.V(score), axis=1)\n",
        "        \n",
        "        print(\"Here 4\")\n",
        "        # context_vector shape after sum == (batch_size, hidden_size)\n",
        "        context_vector = attention_weights * enc_output\n",
        "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
        "        #print(\"Hallo jeee\")\n",
        "        \n",
        "        print(\"Here 5\")\n",
        "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
        "        x = self.bert_preprocess(x)\n",
        "        print(\"Preproc op: \",x['input_word_ids'].shape)\n",
        "        #print(\"Hallo Ahuhuh\")\n",
        "        print(\"Here 6\")\n",
        "        x = self.bert_model(x)[\"pooled_output\"]\n",
        "        print(\"Model op: \",x.shape)\n",
        "        print(\"Here 7\")\n",
        "        #print(\"Ahuhuh\")\n",
        "        x = tf.expand_dims(x,axis=1)\n",
        "        print(\"Model op expanded: \",x.shape)\n",
        "\n",
        "        print(\"Here 8\")\n",
        "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
        "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
        "        print(\"Model op concat: \",x.shape)\n",
        "\n",
        "        print(\"Here 9\")\n",
        "        # passing the concatenated vector to the GRU\n",
        "        output, state = self.gru(x)\n",
        "        print(\"GRU op: \",output.shape)\n",
        "        print(\"GRU state op: \",state.shape)\n",
        "        \n",
        "        print(\"Here 10\")\n",
        "        # output shape == (batch_size * max_length, hidden_size)\n",
        "        output = tf.reshape(output, (-1, output.shape[2]))\n",
        "        print(\"GRU op reshaped: \",output.shape)\n",
        "\n",
        "        print(\"Here 11\")\n",
        "        # output shape == (batch_size * max_length, vocab)\n",
        "        x = self.fc(output)\n",
        "        print(\"Here 12\")\n",
        "        print(\"Final op: \",x.shape)\n",
        "        \n",
        "        return x, state, attention_weights\n"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QnGPb1FzKbwm"
      },
      "source": [
        "encoder = Encoder()\n",
        "decoder = Decoder()\n",
        "test_bert_model = encoder.bert_model\n",
        "test_bert_preproc_model = encoder.bert_preprocess"
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uu2wF9RqMeTD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73da2096-4c39-42a7-e01c-9c97f991265f"
      },
      "source": [
        "i = 0\n",
        "loss = 0\n",
        "for (b,(img,ques,ans)) in enumerate(dataset):\n",
        "  if i != 1:\n",
        "    #with tf.GradientTape() as tape:\n",
        "    #dec_input = tf.expand_dims([\"~\"] * BATCH_SIZE, 1)\n",
        "      \n",
        "        #print(\"answer:\",ans)\n",
        "        enc_output_img, enc_output_ques = encoder(ques, img)\n",
        "        #print(enc_output_img.shape)\n",
        "                \n",
        "        dec_hidden = enc_output_ques\n",
        "        print(\"Enc output ques\",dec_hidden.shape)\n",
        "                \n",
        "        #dec_input = tf.expand_dims([\"~\"] * BATCH_SIZE, 1)  \n",
        "        dec_input = [\"~\"] * BATCH_SIZE    \n",
        "        #print(len(dec_input))\n",
        "                \n",
        "        # Teacher forcing - feeding the target as the next input\n",
        "        for t in range(1, ans.shape[1]):\n",
        "            # passing enc_output to the decoder\n",
        "            predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output_img)\n",
        "            #print(predictions.shape)\n",
        "            \n",
        "            loss += loss_function(ans[:, t], predictions, loss_object)\n",
        "            \n",
        "            # using teacher forcing\n",
        "            #print(ans[:, t])\n",
        "            temp_inp = ans[:, t]\n",
        "            temp_inp = tz.convert_ids_to_tokens(temp_inp)\n",
        "            print(\"Temp input\", temp_inp)\n",
        "            dec_input = temp_inp#tf.expand_dims(ans[:, t], 1)\n",
        "\n",
        "        i += 1\n",
        "  else:\n",
        "      break"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enc output ques (32, 512)\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['fork', 'fr', 'left', 'china', '2', 'no', '3', 'apple', 'inside', 'no', '6', 'no', 'orange', 'star', 'he', '1', 'yes', 'yes', 'bananas', 'decoration', 'blue', 'white', 'kite', 'skate', 'moving', '8', 'car', 'books', 'yes', 'fabric', 'nothing', 'black']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['`', '##is', '`', '`', '`', '`', '`', '`', '`', '`', '`', '`', '`', '`', 'isn', '`', '`', '`', '`', '`', '`', '`', '##s', '##boarding', '`', '`', '`', '`', '`', '`', '`', '`']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '##bee', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', \"'\", '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '`', '`', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '`', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', 't', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '`', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "Here 1\n",
            "Here 2\n",
            "Here 3\n",
            "Here 4\n",
            "Here 5\n",
            "Preproc op:  (32, 128)\n",
            "Here 6\n",
            "Model op:  (32, 512)\n",
            "Here 7\n",
            "Model op expanded:  (32, 1, 512)\n",
            "Here 8\n",
            "Model op concat:  (32, 1, 1024)\n",
            "Here 9\n",
            "GRU op:  (32, 1, 512)\n",
            "GRU state op:  (32, 512)\n",
            "Here 10\n",
            "GRU op reshaped:  (32, 512)\n",
            "Here 11\n",
            "Here 12\n",
            "Final op:  (32, 30522)\n",
            "Temp input ['[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2rWsS6At6ghG"
      },
      "source": [
        "optimizer = tf.keras.optimizers.Adam()\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
        "def loss_function(real, pred,loss_object):\n",
        "\n",
        "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "    loss_ = loss_object(real, pred)\n",
        "\n",
        "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "    loss_ *= mask\n",
        "\n",
        "    return tf.reduce_mean(loss_)"
      ],
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lhLNgjjGkXxT"
      },
      "source": [
        "physical_devices = tf.config.list_physical_devices('GPU')\n",
        "try:\n",
        "  tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
        "except:\n",
        "  # Invalid device or cannot modify virtual devices once initialized.\n",
        "  pass"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ADib6j0K93UK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 558
        },
        "outputId": "775aa24c-2c82-4ad8-e3c9-0fcbf3bad249"
      },
      "source": [
        "EPOCHS = 2\n",
        "import time\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "    start = time.time()\n",
        "    \n",
        "    #hidden = encoder.initialize_hidden_state()\n",
        "    total_loss = 0\n",
        "    \n",
        "    for (batch, (img, ques, ans)) in enumerate(dataset):\n",
        "        loss = 0\n",
        "        hidden = img\n",
        "        \n",
        "        with tf.GradientTape() as tape:\n",
        "            enc_output_img, enc_output_ques = encoder(ques, hidden)\n",
        "            \n",
        "            dec_hidden = enc_output_ques\n",
        "            \n",
        "            dec_input = [\"~\"] * BATCH_SIZE       \n",
        "            \n",
        "            # Teacher forcing - feeding the target as the next input\n",
        "            for t in range(1, ans.shape[1]):\n",
        "                # passing enc_output to the decoder\n",
        "                predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output_img)\n",
        "                \n",
        "                loss += loss_function(ans[:, t], predictions, loss_object)\n",
        "                \n",
        "                # using teacher forcing\n",
        "                temp_inp = ans[:, t]\n",
        "                temp_inp = tz.convert_ids_to_tokens(temp_inp)\n",
        "                #print(\"Temp input\", temp_inp)\n",
        "                dec_input = temp_inp\n",
        "                #dec_input = tf.expand_dims(ans[:, t], 1)\n",
        "        \n",
        "        total_loss += (loss / int(ans.shape[1]))\n",
        "        \n",
        "        variables = encoder.variables + decoder.variables\n",
        "        \n",
        "        gradients = tape.gradient(loss, variables)\n",
        "      \n",
        "        optimizer.apply_gradients(zip(gradients, variables), tf.train.get_or_create_global_step())\n",
        "\n",
        "        if batch % 100 == 0:\n",
        "            print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                                         batch,\n",
        "                                                         loss.numpy() / int(ans.shape[1])))\n",
        "    \n",
        "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                        total_loss/len(img_name_vector)))\n",
        "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ResourceExhaustedError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-47-f6d87ec61b29>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mans\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m                 \u001b[0;31m# passing enc_output to the decoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m                 \u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdec_hidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdec_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdec_hidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menc_output_img\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mans\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_object\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1028\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1029\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1030\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1031\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1032\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-43-e46c025e9a41>\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, x, hidden, enc_output)\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbert_preprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0;31m#print(\"Hallo Ahuhuh\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbert_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"pooled_output\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m         \u001b[0;31m#print(\"Ahuhuh\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1028\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1029\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1030\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1031\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1032\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow_hub/keras_layer.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, training)\u001b[0m\n\u001b[1;32m    237\u001b[0m       result = smart_cond.smart_cond(training,\n\u001b[1;32m    238\u001b[0m                                      \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 239\u001b[0;31m                                      lambda: f(training=False))\n\u001b[0m\u001b[1;32m    240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m     \u001b[0;31m# Unwrap dicts returned by signatures.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/smart_cond.py\u001b[0m in \u001b[0;36msmart_cond\u001b[0;34m(pred, true_fn, false_fn, name)\u001b[0m\n\u001b[1;32m     54\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mtrue_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfalse_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m     return control_flow_ops.cond(pred, true_fn=true_fn, false_fn=false_fn,\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow_hub/keras_layer.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    237\u001b[0m       result = smart_cond.smart_cond(training,\n\u001b[1;32m    238\u001b[0m                                      \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 239\u001b[0;31m                                      lambda: f(training=False))\n\u001b[0m\u001b[1;32m    240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m     \u001b[0;31m# Unwrap dicts returned by signatures.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/saved_model/load.py\u001b[0m in \u001b[0;36m_call_attribute\u001b[0;34m(instance, *args, **kwargs)\u001b[0m\n\u001b[1;32m    668\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    669\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_call_attribute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 670\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    671\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    887\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    922\u001b[0m       \u001b[0;31m# In this case we have not created variables on the first call. So we can\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    923\u001b[0m       \u001b[0;31m# run the first trace but we should fail if variables are created.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 924\u001b[0;31m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    925\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_variables\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    926\u001b[0m         raise ValueError(\"Creating variables on a non-first call to a function\"\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3022\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   3023\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 3024\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   3025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3026\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1967\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1968\u001b[0m       flat_outputs = forward_function.call(\n\u001b[0;32m-> 1969\u001b[0;31m           ctx, args_with_tangents, cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1970\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1971\u001b[0m       with default_graph._override_gradient_function(  # pylint: disable=protected-access\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    594\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    595\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 596\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    597\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mResourceExhaustedError\u001b[0m:  OOM when allocating tensor with shape[4096,2048] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node StatefulPartitionedCall/StatefulPartitionedCall/bert_encoder/StatefulPartitionedCall/transformer/layer_2/intermediate/einsum/Einsum}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n [Op:__forward_restored_function_body_105838]\n\nFunction call stack:\nrestored_function_body\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3aoKSlW_-ZcT"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nPAstB7Q9q40"
      },
      "source": [
        "vision_model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MOWMgTJ--ap8"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XztcrWz5-09g"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8yWeJbpb-kd2"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}