eng_to_ipa_seq2seq_method.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/OrderAndCh4oS/phonetic-translation-ml-seq2seq-attention/blob/main/eng_to_ipa_seq2seq_method.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tensorflow_text"
      ],
      "metadata": {
        "id": "phBEIRGWI-WW",
        "outputId": "5a9e17e0-5c21-43b6-a7b6-008aecb8e330",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: tensorflow_text in /usr/local/lib/python3.9/dist-packages (2.12.1)\n",
            "Requirement already satisfied: tensorflow<2.13,>=2.12.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow_text) (2.12.0)\n",
            "Requirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow_text) (0.13.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (23.0)\n",
            "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (0.4.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (2.2.0)\n",
            "Requirement already satisfied: keras<2.13,>=2.12.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (2.12.0)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (3.20.3)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (3.3.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (0.2.0)\n",
            "Requirement already satisfied: tensorboard<2.13,>=2.12 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (2.12.1)\n",
            "Requirement already satisfied: flatbuffers>=2.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (23.3.3)\n",
            "Requirement already satisfied: jax>=0.3.15 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (0.4.8)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (1.53.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (3.8.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (67.6.1)\n",
            "Requirement already satisfied: tensorflow-estimator<2.13,>=2.12.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (2.12.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (16.0.0)\n",
            "Requirement already satisfied: numpy<1.24,>=1.22 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (1.22.4)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (4.5.0)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (0.32.0)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (1.14.1)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (1.16.0)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.9/dist-packages (from tensorflow<2.13,>=2.12.0->tensorflow_text) (1.6.3)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.9/dist-packages (from astunparse>=1.6.0->tensorflow<2.13,>=2.12.0->tensorflow_text) (0.40.0)\n",
            "Requirement already satisfied: ml-dtypes>=0.0.3 in /usr/local/lib/python3.9/dist-packages (from jax>=0.3.15->tensorflow<2.13,>=2.12.0->tensorflow_text) (0.0.4)\n",
            "Requirement already satisfied: scipy>=1.7 in /usr/local/lib/python3.9/dist-packages (from jax>=0.3.15->tensorflow<2.13,>=2.12.0->tensorflow_text) (1.10.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.9/dist-packages (from tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (2.27.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.9/dist-packages (from tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (1.8.1)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.9/dist-packages (from tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (0.7.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.9/dist-packages (from tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (2.17.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.9/dist-packages (from tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (3.4.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.9/dist-packages (from tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (1.0.0)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (2.2.3)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (5.3.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.9/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.9/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.9/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.9/dist-packages (from markdown>=2.6.8->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (6.3.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (3.4)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (2.0.12)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.9/dist-packages (from werkzeug>=1.0.1->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (2.1.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (3.15.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.9/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.9/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.13,>=2.12->tensorflow<2.13,>=2.12.0->tensorflow_text) (3.2.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "collapsed": true,
        "pycharm": {
          "name": "#%%\n",
          "is_executing": true
        },
        "id": "cvSglKevD7Td"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import typing\n",
        "from typing import Any, Tuple\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import tensorflow_text as tf_text\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "import pathlib"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vAKupH6zEJzN",
        "outputId": "660a72cc-0369-4420-869f-69a9932e99c4"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "outputs": [],
      "source": [
        "np.set_printoptions(precision=3, suppress=True)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "t4cKjM_ND7Tf"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "outputs": [],
      "source": [
        "class ShapeChecker:\n",
        "    def __init__(self):\n",
        "        # Keep a cache of every axis-name seen\n",
        "        self.shapes = {}\n",
        "\n",
        "    def __call__(self, tensor, names, broadcast=False):\n",
        "        if not tf.executing_eagerly():\n",
        "            return\n",
        "\n",
        "        if isinstance(names, str):\n",
        "            names = (names,)\n",
        "\n",
        "        shape = tf.shape(tensor)\n",
        "        rank = tf.rank(tensor)\n",
        "\n",
        "        if rank != len(names):\n",
        "            raise ValueError(\n",
        "                f'Rank mismatch:\\n'\n",
        "                f'    found {rank}: {shape.numpy()}\\n'\n",
        "                f'    expected {len(names)}: {names}\\n'\n",
        "            )\n",
        "\n",
        "        for i, name in enumerate(names):\n",
        "            if isinstance(name, int):\n",
        "                old_dim = name\n",
        "            else:\n",
        "                old_dim = self.shapes.get(name, None)\n",
        "            new_dim = shape[i]\n",
        "\n",
        "            if broadcast and new_dim == 1:\n",
        "                continue\n",
        "\n",
        "            if old_dim is None:\n",
        "                # If the axis name is new, add its length to the cache.\n",
        "                self.shapes[name] = new_dim\n",
        "                continue\n",
        "\n",
        "            if new_dim != old_dim:\n",
        "                raise ValueError(\n",
        "                    f\"Shape mismatch for dimension: '{name}'\\n\"\n",
        "                    f\"    found: {new_dim}\\n\"\n",
        "                    f\"    expected: {old_dim}\\n\"\n",
        "                )"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "431TfGjfD7Tg"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ü b e r m e n s c h\n",
            "/ ʔ y ː b ɐ m ɛ n ʃ /\n"
          ]
        }
      ],
      "source": [
        "import re\n",
        "\n",
        "regex = r\"(,.*)\"\n",
        "\n",
        "def load_data(path):\n",
        "    data = pd.read_csv(path, delimiter='\\t').replace(np.nan, '')\n",
        "\n",
        "    pairs = np.array(data)\n",
        "    for x in pairs:\n",
        "        if not isinstance(x[0], str):\n",
        "            print(x[0])\n",
        "\n",
        "    inp = [' '.join(x[0]) for x in pairs]\n",
        "    targ = [' '.join(x[1]) for x in pairs]\n",
        "\n",
        "    return targ, inp\n",
        "\n",
        "\n",
        "# targ, inp = load_data('ipa_pairs_all.csv')\n",
        "targ, inp = load_data('en_UK.csv')\n",
        "print(inp[-1])\n",
        "print(targ[-1])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j0uB9VqwD7Tg",
        "outputId": "57f8a022-ff69-4a50-b8a4-8502324a8b16"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "outputs": [],
      "source": [
        "BUFFER_SIZE = len(inp)\n",
        "BATCH_SIZE = 64"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "K0lmwLXwD7Th"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(\n",
            "[b'r e c e d i n g' b's i m p l i f i e s' b'c l a r i f y i n g'\n",
            " b's e r v i c e m a n' b'f a r m e r s'], shape=(5,), dtype=string)\n",
            "\n",
            "tf.Tensor(\n",
            "[b'/ \\xc9\\xb9 \\xc9\\xaa s \\xcb\\x88 i \\xcb\\x90 d \\xc9\\xaa \\xc5\\x8b /'\n",
            " b'/ s \\xcb\\x88 \\xc9\\xaa m p l \\xc9\\xaa f \\xcb\\x8c a \\xc9\\xaa z /'\n",
            " b'/ k l \\xcb\\x88 \\xc3\\xa6 \\xc9\\xb9 \\xc9\\xaa f \\xcb\\x8c a \\xc9\\xaa \\xc9\\xaa \\xc5\\x8b /'\n",
            " b'/ s \\xcb\\x88 \\xc9\\x9c \\xcb\\x90 v \\xc9\\xaa s m \\xc9\\x99 n /'\n",
            " b'/ f \\xcb\\x88 \\xc9\\x91 \\xcb\\x90 m \\xc9\\x99 z /'], shape=(5,), dtype=string)\n",
            "b'l u \\xcb\\x90 t'\n"
          ]
        }
      ],
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\n",
        "dataset = dataset.batch(BATCH_SIZE)\n",
        "\n",
        "for example_input_batch, example_target_batch in dataset.take(1):\n",
        "    print(example_input_batch[:5])\n",
        "    print()\n",
        "    print(example_target_batch[:5])\n",
        "    break\n",
        "\n",
        "example_text = tf.constant('l u ː t')\n",
        "\n",
        "print(example_text.numpy())"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QZKXTVeMD7Th",
        "outputId": "1d27f49c-f0f6-4507-f4c6-a609e472875c"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "l u ː t\n",
            "[START] l u ː t [END]\n"
          ]
        }
      ],
      "source": [
        "def tf_lower_and_split_punct(text):\n",
        "    # Split accented characters.\n",
        "    # Keep space, a to z, and select punctuation.\n",
        "    # Add spaces around punctuation.\n",
        "    # Strip whitespace.\n",
        "    text = tf.strings.strip(text)\n",
        "\n",
        "    text = tf.strings.join(['[START]', text, '[END]'], separator=' ')\n",
        "    return text\n",
        "\n",
        "\n",
        "print(example_text.numpy().decode())\n",
        "print(tf_lower_and_split_punct(example_text).numpy().decode())"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dw6XpopAD7Ti",
        "outputId": "22979d2c-ab61-4920-d929-892e4496bd4d"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['', '[UNK]', '[START]', '[END]', 'e', 's', 'i', 'a', 'r', 'n']\n"
          ]
        }
      ],
      "source": [
        "max_vocab_size = 5000\n",
        "\n",
        "input_text_processor = tf.keras.layers.TextVectorization(\n",
        "    standardize=tf_lower_and_split_punct,\n",
        "    max_tokens=max_vocab_size\n",
        ")\n",
        "\n",
        "input_text_processor.adapt(inp)\n",
        "\n",
        "# Here are the first 10 words from the vocabulary:\n",
        "print(input_text_processor.get_vocabulary()[:10])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pNfQeZzGD7Ti",
        "outputId": "75807248-585d-4f07-8a7d-f87509f85a3e"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['', '[UNK]', '/', '[START]', '[END]', 'ˈ', 'ɪ', 'ə', 't', 's']\n"
          ]
        }
      ],
      "source": [
        "output_text_processor = tf.keras.layers.TextVectorization(\n",
        "    standardize=tf_lower_and_split_punct,\n",
        "    max_tokens=max_vocab_size\n",
        ")\n",
        "\n",
        "output_text_processor.adapt(targ)\n",
        "print(output_text_processor.get_vocabulary()[:10])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p60gDwA3D7Tj",
        "outputId": "7b0030ef-2b85-4cbe-e04d-062dc108723e"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(\n",
            "[[ 2  8  4 13  4 14  6  9 16  3]\n",
            " [ 2  5  6 18 17 12  6 22  6  4]\n",
            " [ 2 13 12  7  8  6 22 21  6  9]], shape=(3, 10), dtype=int64)\n"
          ]
        }
      ],
      "source": [
        "example_tokens = input_text_processor(example_input_batch)\n",
        "print(example_tokens[:3, :10])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gM6-bje2D7Tj",
        "outputId": "c8cc7a51-1c8f-43cc-ff15-c8fd19f8c627"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[START]receding[END]\n"
          ]
        }
      ],
      "source": [
        "input_vocab = np.array(input_text_processor.get_vocabulary())\n",
        "tokens = input_vocab[example_tokens[0].numpy()]\n",
        "print(''.join(tokens))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "78m9BqQ2D7Tk",
        "outputId": "60bf337f-a025-49ed-b72c-e576711a366b"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Mask')"
            ]
          },
          "metadata": {},
          "execution_count": 24
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGzCAYAAAAFROyYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2RUlEQVR4nO3de3zUxb3/8XfuCUk2IVwSIiSAIKAIalSI4hExEtBGqKiA1karaD2RCtHaUhWEoqi1oCiXqhQOtmjF8wPUikijYD0SRBSrUBEQAQ0J11wI5P79/cFhjysg2dkNO9m8no/HPh7mu/vOzC5h/DCZ+U6I4ziOAAAALBIa6A4AAAD8EAUKAACwDgUKAACwDgUKAACwDgUKAACwDgUKAACwDgUKAACwDgUKAACwDgUKAACwDgUKJEkhISG65557At0NAPDKqlWrFBISotdeey3QXYGfUaA0YyEhIY16rFq1KtBd9crAgQPVu3dvj2udO3d2v5/Q0FAlJibq3HPP1Z133qm1a9cGqKcAFixY4P67+cEHHxz3vOM46tSpk0JCQvSTn/wkAD1EcxUe6A7A3EsvveTx9cKFC7Vy5crjrvfq1et0dqvJnHfeebrvvvskSRUVFfr3v/+txYsX64UXXtD48eM1ffr0APcQaLmio6O1aNEiDRgwwOP66tWr9e233yoqKipAPUNzRYHSjP3sZz/z+LqwsFArV6487nqwOOOMM457b0888YRuuukmzZgxQ927d9fdd98doN4BLdvVV1+txYsXa+bMmQoP/7//tSxatEgZGRnat29fAHuH5ohf8QS5yspK3XffferUqZOioqLUo0cPPfXUU2rMIdZTp05VaGionn32Wfe15cuX67LLLlNsbKzi4+N1zTXXaOPGjR65W2+9VXFxcfruu+80fPhwxcXFqV27drr//vtVX1/v1/cXExOjl156SUlJSXr00Uc93tcrr7yijIwMxcfHy+Vy6dxzz9Uzzzzj1/YBHDV69Gjt379fK1eudF+rqanRa6+9pptuuum41z/11FO65JJL1KZNG8XExCgjI+OE60hWrlypAQMGKDExUXFxcerRo4d+97vf/Whfqqur9ZOf/EQJCQn68MMPfX9zCAgKlCDmOI6uvfZazZgxQ0OGDNH06dPVo0cP/frXv1Z+fv6PZh966CFNnDhRf/rTnzR27FhJR3+ldM011yguLk5PPPGEHn74YW3atEkDBgzQN99845Gvr69Xdna22rRpo6eeekqXX365/vjHP+r555/3+/uMi4vTT3/6U3333XfatGmTpKOD2ujRo9W6dWs98cQTevzxxzVw4ED9z//8j9/bB3B0nVhmZqZefvll97Xly5errKxMo0aNOu71zzzzjM4//3xNmTJFjz32mMLDw3XDDTfo73//u/s1Gzdu1E9+8hNVV1drypQp+uMf/6hrr732R/8eHzlyRDk5Ofrwww/1j3/8Q5dccol/3yhOHwdBIy8vz/n+H+nSpUsdSc7UqVM9Xnf99dc7ISEhztatW93XJDl5eXmO4zjOfffd54SGhjoLFixwP19RUeEkJiY6Y8aM8fhexcXFTkJCgsf13NxcR5IzZcoUj9eef/75TkZGxinfx+WXX+6cc845HtfS09Oda6655qSZGTNmOJKcZcuWOY7jOPfee6/jcrmcurq6U7YHwNz8+fMdSc66deuc5557zomPj3cOHz7sOI7j3HDDDc4VV1zhOM7xf4ePveaYmpoap3fv3s6gQYPc1479vd67d+9J23/vvfccSc7ixYudiooK5/LLL3fatm3rfPrpp358lwgEZlCC2FtvvaWwsDD96le/8rh+3333yXEcLV++3OO64zi655579Mwzz+gvf/mLcnNz3c+tXLlSpaWlGj16tPbt2+d+hIWFqV+/fnrvvfeOa/+Xv/ylx9eXXXaZvv76az++w/8TFxcn6ejiWUlKTExUZWWlx3QzgKZ144036siRI3rzzTdVUVGhN99884S/3pGO/nr2mIMHD6qsrEyXXXaZPvnkE/f1xMRESdKyZcvU0NDwo22XlZVp8ODB+vLLL7Vq1Sqdd955Pr8fBBaLZIPYjh07lJqaqvj4eI/rx3b17Nixw+P6woULdejQIc2ZM0ejR4/2eG7Lli2SpEGDBp2wLZfL5fF1dHS02rVr53GtdevWOnjwoPdvpBEOHTokSe73+p//+Z969dVXNXToUJ1xxhkaPHiwbrzxRg0ZMqRJ2gcgtWvXTllZWVq0aJEOHz6s+vp6XX/99Sd87ZtvvqmpU6dqw4YNqq6udl8PCQlx//fIkSP14osv6o477tBvf/tbXXnllbruuut0/fXXKzTU89/X48aNU1VVlT799FOdc845TfMGcVoxgwK3Sy+9VMnJyXruued04MABj+eO/evlpZde0sqVK497LFu2zOP1YWFhp63fkvTFF19Ikrp16yZJat++vTZs2KDXX39d1157rd577z0NHTrUY1YIgP/ddNNNWr58uebOnauhQ4e6Z0G+75///KeuvfZaRUdHa/bs2Xrrrbe0cuVK3XTTTR4L3WNiYvT+++/rH//4h2655Rb961//0siRI3XVVVcdt+B+2LBhchxHjz/++ClnW9A8UKAEsfT0dBUVFbl/7XHMl19+6X7++7p166Z33nlHRUVFGjJkiEfuzDPPlHT0f/xZWVnHPQYOHNi0b+ZHHDp0SEuWLFGnTp087vkSGRmpnJwczZ49W9u2bdNdd92lhQsXauvWrQHrKxDsfvrTnyo0NFSFhYUn/fXOf//3fys6OlorVqzQL37xCw0dOlRZWVknfG1oaKiuvPJKTZ8+XZs2bdKjjz6qd99997hfKw8fPlx//vOftWjRIuXl5fn9feH0o0AJYldffbXq6+v13HPPeVyfMWOGQkJCNHTo0OMyffr00VtvvaV///vfysnJ0ZEjRyRJ2dnZcrlceuyxx1RbW3tcbu/evU3zJk7hyJEjuuWWW3TgwAE9+OCD7unh/fv3e7wuNDRUffr0kSSP6WQA/hUXF6c5c+bokUceUU5OzglfExYWppCQEI9ZkG+++UZLly71eN0PZ3IludeWnOjv8c9//nPNnDlTc+fO1W9+8xvzNwErsAYliOXk5OiKK67Qgw8+qG+++UZ9+/bVO++8o2XLlmncuHHuWZEf6t+/v5YtW6arr75a119/vZYuXSqXy6U5c+bolltu0QUXXKBRo0apXbt22rlzp/7+97/r0ksvPa4Q8rfvvvtOf/nLXyQdnTXZtGmTFi9erOLiYt13332666673K+94447dODAAQ0aNEgdO3bUjh079Oyzz+q8884LmjvrArY61a9Sr7nmGk2fPl1DhgzRTTfdpD179mjWrFnq1q2b/vWvf7lfN2XKFL3//vu65pprlJ6erj179mj27Nnq2LHjcXesPeaee+5ReXm5HnzwQSUkJJzynimwWGA3EcGffrjN2HGObg8eP368k5qa6kRERDjdu3d3/vCHPzgNDQ0er9P3thkfs2zZMic8PNwZOXKkU19f7zjO0S192dnZTkJCghMdHe2ceeaZzq233up8/PHH7lxubq4TGxt7XP8mTZp0XP9O5GTbjCU5kpyQkBDH5XI555xzjjNmzBhn7dq1x32P1157zRk8eLDTvn17JzIy0klLS3PuuusuZ/fu3adsH0DjfX+b8Y/54TbjefPmOd27d3eioqKcnj17OvPnzz9ujCgoKHCGDRvmpKamOpGRkU5qaqozevRo56uvvnK/5vvbjL/vgQcecCQ5zz33nJ/eKU63EMdpxC1FAQAATiPWoAAAAOtQoAAAAOtQoAAAAOtQoAAAAOtQoAAAAOtQoAAAAOtYd6O2hoYGFRUVKT4+3uPQKACnj+M4qqioUGpq6nGHstmKsQMILH+PG9YVKEVFRerUqVOguwFA0q5du9SxY8dAd6NRGDsAO/hr3LCuQImPj5ckDUy+TeGhkV7nazu3M2571+BWxtm255UYZ8tWJRtn62OMo5KkNhvrT/2ik6iLMa+Q43YdMc4eeaDi1C86iYjnWhtnd99y/BlEjdX195XG2bpt242zxm2qVh/oLfffx+bgWF93fNJZrrjmMesDNJWfnnXuaW/T3+OGdQXKsanZ8NBIowLFCY82bjs02jwbHhtlnA2LMm9X5s1KksIjzAsUJ9L8fwLh4eY3MA6PrfGhXR9+PlqFmbcbVmecVUiEedbU//7xNKdflRzrqysuVK548z8rIBiEB8G4wT8zAACAdShQAACAdaz7FY+vtv7M+18LHdPmY/N2Q9aZr33pfK/5GoNt73YxzkpS7Dfm6zlCD1UbZ+u2bDPOtso2jkr62jjZ+R3zVn34BQ+A/5Wd2jfQXcBpxAwKAACwDgUKAACwDgUKAACwDgUKAACwDgUKAACwTtDt4knYaP6Wwo+Y3zzs22zzbPRD5neS7ajDxllfbc4z37nUY5Z5u77sAALQfK0o+sw4yw6g5ocZFAAAYB0KFAAAYB0KFAAAYB0KFAAAYB0KFAAAYJ2g28XT7lPzXS17z29lnE3+p/nx7mGVNcbZfRmJxllJ2j/AvN8J683r2/LnzHc9xf32bOOsL59X2/WlxtktuS7jbI9Ze42z7HgCjvJlB1BLY8uOJ2ZQAACAdbwuUL777jv97Gc/U5s2bRQTE6Nzzz1XH3/8f8cAO46jiRMnqkOHDoqJiVFWVpa2bNni104DaH4YOwB4w6sC5eDBg7r00ksVERGh5cuXa9OmTfrjH/+o1q1bu1/z5JNPaubMmZo7d67Wrl2r2NhYZWdnq6qqyu+dB9A8MHYA8JZXa1CeeOIJderUSfPnz3df69Kli/u/HcfR008/rYceekjDhg2TJC1cuFDJyclaunSpRo0a5aduA2hOGDsAeMurGZTXX39dF154oW644Qa1b99e559/vl544QX389u3b1dxcbGysrLc1xISEtSvXz+tWbPmhN+zurpa5eXlHg8AwYWxA4C3vJpB+frrrzVnzhzl5+frd7/7ndatW6df/epXioyMVG5uroqLiyVJycmeZ8skJye7n/uhadOmafLkyYbdP15VuyjjbHXrU7/mZA4PMN89FP5z8ynsmLn1xllJOqdzkXG27hnzD6xYKcbZiHbmu57KuxpH1X51tXH2zHGFxtk646Q9msPYAdjGlt00geLVDEpDQ4MuuOACPfbYYzr//PN15513asyYMZo7d65xByZMmKCysjL3Y9euXcbfC4CdGDsAeMurAqVDhw46+2zPe1D06tVLO3fulCSlpBz9V3FJSYnHa0pKStzP/VBUVJRcLpfHA0BwYewA4C2vCpRLL71Umzdv9rj21VdfKT09XdLRRW8pKSkqKChwP19eXq61a9cqMzPTD90F0BwxdgDwlldrUMaPH69LLrlEjz32mG688UZ99NFHev755/X8889LkkJCQjRu3DhNnTpV3bt3V5cuXfTwww8rNTVVw4cPb4r+A2gGGDsAeMurAuWiiy7SkiVLNGHCBE2ZMkVdunTR008/rZtvvtn9mgceeECVlZW68847VVpaqgEDBujtt99WdHS03zsPoHlg7ADgrRDHccwPRWkC5eXlSkhIUFaHuxQeGul1ftw//2Hc9rS7c42zFWne9/WY/QPMd4d0e9G3XTw7xpr/8Se/Yv4/jiNtzM8Aar+65NQvOgnOpmmcOqdWq7RMZWVlzWZtx7Gx4+BXXeWKN//5An6ope+maSx/jxucxQMAAKxDgQIAAKxDgQIAAKxDgQIAAKzj1S6e5mDK1hzjbIQP7Vammmejvza/Pf9uX28R4cOa0Wt//45x9p1fDjBvGAAQ9JhBAQAA1qFAAQAA1qFAAQAA1qFAAQAA1qFAAQAA1gm6XTxxvzW//fpNrywzzr7w2+uMsyUXmd+Wuyq1zjgrSR1XhBhn/7p7sHG2dasa4+zu4cnG2eiD5llusQ+0TCuKPgtIuy39FvvMoAAAAOtQoAAAAOtQoAAAAOtQoAAAAOtQoAAAAOsE3S6eHTmJxtn5Y4cZZ/s/sc44+/kvehpnS89OMM5KUvwXe4yzMUsCszMl1fwIIJ/4tl8KgMTOFDQeMygAAMA6FCgAAMA6FCgAAMA6FCgAAMA6FCgAAMA6QbeLJzLjoHE26g3zc2ne2dnDOFuflWicjStyjLOSVNG7rXE24YXDxtnKx84wzhb3jzTOphSanwFUF2t+ZlL8F/vM2+UcHwSRQJ1r09IEw24pZlAAAIB1KFAAAIB1KFAAAIB1KFAAAIB1gm6R7JEq8wWUlZ2jjbNJs+qNsxVpxlElfWR+q3rJtwWYNUvM243QbuNspwDd6j7Chyy3yUcwCYYFmLAfMygAAMA6FCgAAMA6FCgAAMA6FCgAAMA6FCgAAMA6QbeLJ2xjrHE2vNL8NugH8iqNs0mzjKMqGpJsHpaU6kOWW7ADLZMvt6tnBxAaixkUAABgHQoUAABgHQoUAABgHQoUAABgHQoUAABgnaDbxdPx3cPG2a0/Mz/Hp80brY2z4YfN+3y4g3mfJenAxe2Ns4lxUT61bWpfRqJxtvWX5p91REmFcZYdT8BRvuwAamla+o4nZlAAAIB1KFAAAIB1KFAAAIB1KFAAAIB1KFAAAIB1gm4Xz6//66/G2YkT7zDOJm4qNc5+fUOicXbMte8YZyWpYPHFxtnSsxOMs0kf7THPvvihcdYXdQFpFYA/tPQdMc0RMygAAMA6XhUojzzyiEJCQjwePXv2dD9fVVWlvLw8tWnTRnFxcRoxYoRKSkr83mkAzQtjBwBveT2Dcs4552j37t3uxwcffOB+bvz48XrjjTe0ePFirV69WkVFRbruuuv82mEAzRNjBwBveL0GJTw8XCkpKcddLysr07x587Ro0SINGjRIkjR//nz16tVLhYWF6t+/v++9BdBsMXYA8IbXMyhbtmxRamqqunbtqptvvlk7d+6UJK1fv161tbXKyspyv7Znz55KS0vTmjVrTvr9qqurVV5e7vEAEHwYOwB4w6sZlH79+mnBggXq0aOHdu/ercmTJ+uyyy7TF198oeLiYkVGRioxMdEjk5ycrOLi4pN+z2nTpmny5MlGnT+RP+TebJytOj/Eb/3wRtJGxzj71qqBPrVdnGN+lk9KYY1xtjY53jhb1bufcbaiY5hxtvVm8/e7t6/55xx90Diq9qsN13HUV0t+PD6oOYwdCG6cAdT0yivq1fos/30/rwqUoUOHuv+7T58+6tevn9LT0/Xqq68qJibGqAMTJkxQfn6+++vy8nJ16tTJ6HsBsBNjBwBv+bTNODExUWeddZa2bt2qlJQU1dTUqLS01OM1JSUlJ/y98zFRUVFyuVweDwDBjbEDwKn4VKAcOnRI27ZtU4cOHZSRkaGIiAgVFBS4n9+8ebN27typzMxMnzsKIHgwdgA4Fa9+xXP//fcrJydH6enpKioq0qRJkxQWFqbRo0crISFBt99+u/Lz85WUlCSXy6WxY8cqMzOTVfhAC8fYAcBbXhUo3377rUaPHq39+/erXbt2GjBggAoLC9WuXTtJ0owZMxQaGqoRI0aourpa2dnZmj17dpN0/GQO9mxlnE35oNQ4m//aYuPsA0+PMc4mfeTDCkpJnd7x40rI08RsxYLvWV+k+nYigTHT2/PXObV+7UdzGDvQeNw2HidydNz42m/fz6sC5ZVXXvnR56OjozVr1izNmjXLp04BCC6MHQC8xVk8AADAOhQoAADAOhQoAADAOhQoAADAOl4fFmi7g70bjLNt15u3++svrjfO1kebt9sQF2UelrTtafNtnD1m7TXO1m1pfruHABwVqNvGs3uoZWEGBQAAWIcCBQAAWIcCBQAAWIcCBQAAWIcCBQAAWCfodvG0WxcSkHaPVEUaZ7v+o9Q4uyMn0TgrSZEHzLMVvdsaZ49cnmycrfPhQJ12nx42zkaUVBhn2bUE+C5Qu4cCpaXvWmIGBQAAWIcCBQAAWIcCBQAAWIcCBQAAWIcCBQAAWCfodvEkbiozzu7LSDTOtvrAOKrKzvXG2dgi83Ylaf+AauPsPpmfA5T+Rqlxtj7WfMcUO3EAnE4tfSeOL5hBAQAA1qFAAQAA1qFAAQAA1qFAAQAA1gm6RbK+LKDsfccXxtnv7u9qnN17fivjbMoHpcZZSarNcYyzrs/Nf3yKByQaZ1tvrjHO7rjN/Bb73eYbR1lgC7RQvtyev6UvsGUGBQAAWIcCBQAAWIcCBQAAWIcCBQAAWIcCBQAAWCfodvEc7Gm+Iybs7o7G2a33hhlnYzcaR33aDSNJSbPMd8TUxfrUtLGovYeNs+lv1fmxJwCApsIMCgAAsA4FCgAAsA4FCgAAsA4FCgAAsA4FCgAAsE7Q7eIJP2J+towvur1Yb5yta2W+kyb8sG+7Unw5B6gu2rzduCLzP6fQQ9XG2YZPNxln2f8DtEwt/UycQGEGBQAAWIcCBQAAWIcCBQAAWIcCBQAAWIcCBQAAWCfodvGUXObDbpqYROPsYxNeMM5OnHiHcTbisPkZQJJ0eEClcTb92RDjbERJhXG2bss24yyAlomdOM0PMygAAMA6FCgAAMA6FCgAAMA6FCgAAMA6FCgAAMA6QbeLJ/mf5rtaDqWat3vXytvMw1eZn8VzRoeD5u1KSp7bzji79Q7z02l6PBNlnA3vfqZxlh1AQMu0ougz4yw7gALDpxmUxx9/XCEhIRo3bpz7WlVVlfLy8tSmTRvFxcVpxIgRKikp8bWfAIIE4waAxjAuUNatW6c//elP6tOnj8f18ePH64033tDixYu1evVqFRUV6brrrvO5owCaP8YNAI1lVKAcOnRIN998s1544QW1bt3afb2srEzz5s3T9OnTNWjQIGVkZGj+/Pn68MMPVVhY6LdOA2h+GDcAeMOoQMnLy9M111yjrKwsj+vr169XbW2tx/WePXsqLS1Na9asOeH3qq6uVnl5uccDQPDx57ghMXYAwc7rRbKvvPKKPvnkE61bt+6454qLixUZGanExESP68nJySouLj7h95s2bZomT57sbTdO6j8nvmacXTTqKuPsk3l/Ns4+8NNfGGe35LY3zkpSjy/2Gme7LzFfcNpgnPQti8Dw97gh+X/swOnBglM0llczKLt27dK9996rv/71r4qOjvZLByZMmKCysjL3Y9euXX75vgDs0BTjhsTYAQQ7rwqU9evXa8+ePbrgggsUHh6u8PBwrV69WjNnzlR4eLiSk5NVU1Oj0tJSj1xJSYlSUlJO+D2joqLkcrk8HgCCR1OMGxJjBxDsvPoVz5VXXqnPP//c49ptt92mnj176je/+Y06deqkiIgIFRQUaMSIEZKkzZs3a+fOncrMzPRfrwE0G4wbAEx4VaDEx8erd+/eHtdiY2PVpk0b9/Xbb79d+fn5SkpKksvl0tixY5WZman+/fv7r9cAmg3GDQAm/H4n2RkzZig0NFQjRoxQdXW1srOzNXv2bH83AyCIMG4A+KEQx3GcQHfi+8rLy5WQkKCsDncpPDTS63zRiC7GbT857gXj7LS7c42zO68O3IkDkQfMbybc7+ovjLN7bjG/xT63q296dU6tVmmZysrKms3ajmNjx8GvusoVb37kBYITu4eanr/HDQ4LBAAA1qFAAQAA1qFAAQAA1qFAAQAA1qFAAQAA1gnc9pEmkn/PYuPs9OtvMM5+c6/5ZqhuL1YZZyNKKoyzkm87Yoqm+NIyB7sBzRU7YnA6MIMCAACsQ4ECAACsQ4ECAACsQ4ECAACsQ4ECAACsE3S7eGZPud44m6gy42ybD6KMs2GVpcbZfz+YaJyVpM6LLjTOVqR5f1bSMfsHVBtnez1aapzlHB/AdyuKPgt0F04rdi0FBjMoAADAOhQoAADAOhQoAADAOhQoAADAOkG3SHbKlBeNs/lzxxhn62OMo7oi70vjbPVvLjBvWFL44TrjbPQNB4yzve4JMc6y0BWAt1jo2vwwgwIAAKxDgQIAAKxDgQIAAKxDgQIAAKxDgQIAAKwTdLt4VpafG5B2w46YZ99c0t842/HwYfOGJe24upVxNnmu+e39t06tMs52e+hM4yw7gICWyZfb87MDKDCYQQEAANahQAEAANahQAEAANahQAEAANahQAEAANYJul08G8b2Mc52rCw1zu7LSDTOxn9WY5z9dpD5LhxJOmOVedsHe0QaZy9J326c3aN2xlkAQPPADAoAALAOBQoAALAOBQoAALAOBQoAALAOBQoAALBO0O3i8cXXv40wztYerDPOHuxt/sfQY1aJcVby7Wya9u+Yt1v0rHlWKvclDMBHnE2D04EZFAAAYB0KFAAAYB0KFAAAYB0KFAAAYJ2gWyT79fXRxtm0eeYLXcMPm98yfsfV5rer3zk82TgrSdEHzfN1Mebttt5s/nlVpJnfYn//heZ/xj3+dMg4G3qo2jjry0JmoCmsKPos0F3AKQTDQmZmUAAAgHUoUAAAgHUoUAAAgHUoUAAAgHUoUAAAgHWCbhdP6mrHOOvL7pC6GPPsGavMd7R8N9C8XUlq95kPbd9ea5yN/9Z8t5Uvu4d82YmzJddl3u6svcZZAGiJmEEBAADW8apAmTNnjvr06SOXyyWXy6XMzEwtX77c/XxVVZXy8vLUpk0bxcXFacSIESop8e0wOwDNH2MHAG95VaB07NhRjz/+uNavX6+PP/5YgwYN0rBhw7Rx40ZJ0vjx4/XGG29o8eLFWr16tYqKinTdddc1SccBNB+MHQC85dUalJycHI+vH330Uc2ZM0eFhYXq2LGj5s2bp0WLFmnQoEGSpPnz56tXr14qLCxU//79/ddrAM0KYwcAbxmvQamvr9crr7yiyspKZWZmav369aqtrVVWVpb7NT179lRaWprWrFlz0u9TXV2t8vJyjweA4MXYAaAxvN7F8/nnnyszM1NVVVWKi4vTkiVLdPbZZ2vDhg2KjIxUYmKix+uTk5NVXFx80u83bdo0TZ482euOn0zJqCrjbOTHscbZeh92lqRP+co4+92OLuYNS4qZb76rpfPIwJwR48NHrQYfsmd+ap41PwEoeNg+duD0CIYzYnB6eD2D0qNHD23YsEFr167V3XffrdzcXG3atMm4AxMmTFBZWZn7sWvXLuPvBcBejB0AvOH1DEpkZKS6desmScrIyNC6dev0zDPPaOTIkaqpqVFpaanHv4RKSkqUkpJy0u8XFRWlqKgo73sOoFlh7ADgDZ/vg9LQ0KDq6mplZGQoIiJCBQUF7uc2b96snTt3KjMz09dmAAQZxg4AP8arGZQJEyZo6NChSktLU0VFhRYtWqRVq1ZpxYoVSkhI0O233678/HwlJSXJ5XJp7NixyszMZBU+0MIxdgDwllcFyp49e/Tzn/9cu3fvVkJCgvr06aMVK1boqquukiTNmDFDoaGhGjFihKqrq5Wdna3Zs2c3SccBNB+MHQC8FeI4jvnhNU2gvLxcCQkJyupwl8JDvT9nZuuM9sZtJ73eyjh7KDXEOFtzYaVx9pL07cZZSVr7Vm/jbPobpcbZfRmJxtm2683bDT1UbZyt2xKYXUuBUOfUapWWqaysTC6X+RlEp9OxsePgV13lig8LdHcASS1r15K/xw3O4gEAANahQAEAANahQAEAANahQAEAANahQAEAANbx+k6ytnOtND9PJ3FTqXG25CrzE2KiN5r3ece8s4yzklR1k/muFl+wEwdAc9GSduLYhBkUAABgHQoUAABgHQoUAABgHQoUAABgnaBbJJvyM/Nbv3/TuosPLZsv3PTllvEpc741zkpS+P1djbOBWqzaYJwE0JyxWLVlYQYFAABYhwIFAABYhwIFAABYhwIFAABYhwIFAABYJ+h28VQ/lGyczZ37jnF29geDjLM7chLNs2+ZZyWpow77lAcAoCkwgwIAAKxDgQIAAKxDgQIAAKxDgQIAAKxDgQIAAKwTdLt4fPHC64ONsx3X1Rtnj7Qxjupgb99OpqlqF2Wc/XZQK+Nsx3fjjbMRJRXGWV/OAAIQWCuKPgtIu5wBFBjMoAAAAOtQoAAAAOtQoAAAAOtQoAAAAOtQoAAAAOsE3S6eHWMd42ztwTrjbPmt5jtLjlRFGmfT5kUYZyWpIs287aqu1cbZiJfZiQPg9GEnTvPDDAoAALAOBQoAALAOBQoAALAOBQoAALBO0C2SPSt5j3F248FOxtma9a2Ns7U+LDb95jrfaszkf5ovKu71aKlxloWuALzFQteWhRkUAABgHQoUAABgHQoUAABgHQoUAABgHQoUAABgnaDbxbPvhc7G2Y6HG4yzJRcZRxW7Mco87KO6mIA1DQDASTGDAgAArEOBAgAArEOBAgAArEOBAgAArEOBAgAArBN0u3gOXHvYOHtJ+nbjbO2z5xhnD6WGGGfDBhw0zkpSzAKXcXbP5cnG2fgu5mcX1cWGGWdLLjLPpr9l/rMVUVJhnOXcIuCoFUWfBboLzUYwnFvEDAoAALCOVwXKtGnTdNFFFyk+Pl7t27fX8OHDtXnzZo/XVFVVKS8vT23atFFcXJxGjBihkpISv3YaQPPC2AHAW14VKKtXr1ZeXp4KCwu1cuVK1dbWavDgwaqsrHS/Zvz48XrjjTe0ePFirV69WkVFRbruuuv83nEAzQdjBwBvebUG5e233/b4esGCBWrfvr3Wr1+v//iP/1BZWZnmzZunRYsWadCgQZKk+fPnq1evXiosLFT//v3913MAzQZjBwBv+bQGpaysTJKUlJQkSVq/fr1qa2uVlZXlfk3Pnj2VlpamNWvWnPB7VFdXq7y83OMBILgxdgA4FeNdPA0NDRo3bpwuvfRS9e7dW5JUXFysyMhIJSYmerw2OTlZxcXFJ/w+06ZN0+TJk027cZxhZ31unP38Fz2Ns3tzHeNs9/8qM86WFiUaZ48yP3/Il3N8ovaa74iJ2V5tnl0SmB0xdQFp1U62jh3AyQTDjpjmyHgGJS8vT1988YVeeeUVnzowYcIElZWVuR+7du3y6fsBsBtjB4DGMJpBueeee/Tmm2/q/fffV8eOHd3XU1JSVFNTo9LSUo9/CZWUlCglJeWE3ysqKkpRUYE7zRfA6cPYAaCxvJpBcRxH99xzj5YsWaJ3331XXbp08Xg+IyNDERERKigocF/bvHmzdu7cqczMTP/0GECzw9gBwFtezaDk5eVp0aJFWrZsmeLj492/G05ISFBMTIwSEhJ0++23Kz8/X0lJSXK5XBo7dqwyMzNZhQ+0YIwdALzlVYEyZ84cSdLAgQM9rs+fP1+33nqrJGnGjBkKDQ3ViBEjVF1drezsbM2ePdsvnQXQPDF2APBWiOM45ttPmkB5ebkSEhKU1eEuhYdGep3fek+XU7/oJJLX1Rtno/ea7ywJq6wxzu7ISTTOSlLYEfNs663mn1fR5ebnD/WYtdc4y7k2jVPn1GqVlqmsrEwul/l5TafTsbHj4Fdd5Yo3P3MJ+CF28TSOv8cNzuIBAADWoUABAADWoUABAADWoUABAADWMb7VPTx9O6iVcTb9DfNFsrFFxlFJ0sHe5re6rzzTfH11dJH5IsaK3m3N202ON85GlFQYZ1mcCzRfK4o+C0i7LX1xLjMoAADAOhQoAADAOhQoAADAOhQoAADAOhQoAADAOkG3i8eX29UfaWO+syTqoHFU1e3MdwCFH/HtpIJ268xvOR/+833G2Yj/l2ScrUjz/giEY77NNs+6Ophn2//hPOMsu4eAlqml7x5iBgUAAFiHAgUAAFiHAgUAAFiHAgUAAFiHAgUAAFgn6HbxlIyqMs52fbzWOFsfa77DI6zS/CyeCXMWG2cl6ff3/8I4G/17l3E2osR821PEO+Y7U5JeNI4GTF2gOwAEAVt2pqDxmEEBAADWoUABAADWoUABAADWoUABAADWoUABAADWCbpdPNoWaxzdkWPebEqh+U6cgz3Nz+L57Ei6cVaSovdWG2c5IwbA6cROnJaFGRQAAGAdChQAAGAdChQAAGAdChQAAGCdoFskm7yu3jj77TDzm4rXfW7+UYYfcYyzb+UPNM5K0ndXm9+iv9t880WyAAD8GGZQAACAdShQAACAdShQAACAdShQAACAdShQAACAdYJuF0/JqCrjbPLr5recP9gtxDgbfdA4qr19zXfhSFJtkvnOpT2XJxtnWyfHG2e5xT7QMq0o+iwg7XKL/cBgBgUAAFiHAgUAAFiHAgUAAFiHAgUAAFiHAgUAAFgn6Hbx1O+OMc4mbiozzh64NsI42/HxWuPsvoxE46wkHakyr1Hbri81zvrS7/KrzXdbnbGqtXE2au9h42x9rPluK3YtAYEVqN1DgWLLriVmUAAAgHUoUAAAgHUoUAAAgHUoUAAAgHUoUAAAgHWCbhdP/LbA1FyulbHG2eIB5u2mvl1iHpaU9KL5Lo8GX9r91IesD+36wpf3a35Sk2R+WhLQNGzZ5YHgxgwKAACwjtcFyvvvv6+cnBylpqYqJCRES5cu9XjecRxNnDhRHTp0UExMjLKysrRlyxZ/9RdAM8S4AcBbXhcolZWV6tu3r2bNmnXC55988knNnDlTc+fO1dq1axUbG6vs7GxVVVX53FkAzRPjBgBveb0GZejQoRo6dOgJn3McR08//bQeeughDRs2TJK0cOFCJScna+nSpRo1apRvvQXQLDFuAPCWX9egbN++XcXFxcrKynJfS0hIUL9+/bRmzZoTZqqrq1VeXu7xANBymIwbEmMHEOz8uounuLhYkpScnOxxPTk52f3cD02bNk2TJ0/2Wx8OD6g0zpYeSDDOxuyvN85WdAwzzgLNncm4Ifl/7ABgl4Dv4pkwYYLKysrcj127dgW6SwCaAcYOILj5tUBJSUmRJJWUeN6bo6SkxP3cD0VFRcnlcnk8ALQcJuOGxNgBBDu/FihdunRRSkqKCgoK3NfKy8u1du1aZWZm+rMpAEGCcQPAiXi9BuXQoUPaunWr++vt27drw4YNSkpKUlpamsaNG6epU6eqe/fu6tKlix5++GGlpqZq+PDh/uw3gGaEcQOAt7wuUD7++GNdccUV7q/z8/MlSbm5uVqwYIEeeOABVVZW6s4771RpaakGDBigt99+W9HR0f7r9Y9If9b8puJhlWXG2eV/f9k42/X1McbZlA+ijLOS5Aw4zzi79/xWxtmUD0qNs6Vnmy9mTtxk/me8Jdf8Vwg9Zu01ztZtMT+OwBa2jxvwzoqizwLdBZxCMBxH4HWBMnDgQDmOc9LnQ0JCNGXKFE2ZMsWnjgEIHowbALwV8F08AAAAP0SBAgAArEOBAgAArEOBAgAArOPXW93bIGpqyalfdBJlz6QZZzN//UvjbLcdR4yzoYeqjbOS1PDpJuNs+w98aNc8KtengWn3TB/arfOhXcA2wbBDBPZjBgUAAFiHAgUAAFiHAgUAAFiHAgUAAFiHAgUAAFgn6HbxlFbHmGe7hRlnw6uMo0rcVGOc3T462bxhSSmFrY2zUXsPG2d9OU8n6aM9xtlgONcGCLSWdhYPu5YCgxkUAABgHQoUAABgHQoUAABgHQoUAABgHQoUAABgnaDbxVNRFWWcbfeZ+W6a8MPmp61Udo43zsYWGUclSQd7RBpn4+47YN7wQvNdPDuHm+9cqrkwzjh7xrwI4+x3A80/527zzc+XYtcS4Dt2LQUGMygAAMA6FCgAAMA6FCgAAMA6FCgAAMA6QbdIdnDaZuNsYexFxtkj7cwXUB65sdQ42+GREOOs5Nst52N+7zLORpSY367e1QwXfnZ+xzxrvvwawDG2LPxE4zGDAgAArEOBAgAArEOBAgAArEOBAgAArEOBAgAArBN0u3g2jO1jnI2trDDO+rIb5r6e/zDOzm83zDgrSQfOMd8FlLjJ/GiA754yv/W7a0E/4+yRNmHG2farueU80Fw1x9vVt/SdR8ygAAAA61CgAAAA61CgAAAA61CgAAAA61CgAAAA6wTdLp4dV7cyzoZVmWfbfl5vnJ095XrjbIxqjbOSNGxooXF276A442z1xLOMsxVp5jtx9g+oNs62X20cBQB4iRkUAABgHQoUAABgHQoUAABgHQoUAABgHQoUAABgnaDbxZP+1mHjbF0r84/jQF6lcfZIlfm5NGEbY42zkvTxby4wzl74xCfG2R3GSant+lLjbOsvzT9rAMDpwwwKAACwDgUKAACwDgUKAACwDgUKAACwDgUKAACwTtDt4qlIjzHOJm4qM84e2djaONt1calxdktug3HWV+/N6m+c3X+T+Zk4vR41zzZ8usk4W2ecBNCcZaf2DXQXWqQmm0GZNWuWOnfurOjoaPXr108fffRRUzUFIEgwbgA4pkkKlL/97W/Kz8/XpEmT9Mknn6hv377Kzs7Wnj17mqI5AEGAcQPA9zVJgTJ9+nSNGTNGt912m84++2zNnTtXrVq10p///OemaA5AEGDcAPB9fl+DUlNTo/Xr12vChAnua6GhocrKytKaNWuOe311dbWqq/9vTUFZ2dF1IHUNNUbt19dUGeUkqa7eh7UNVc2vXUmqqzNfWVFfY77+peGI+Xv25fOqc2qNsy1JnY5+To7jnJb2vB03pJOPHeWHArcuC8GJcaNx/D1u+L1A2bdvn+rr65WcnOxxPTk5WV9++eVxr582bZomT5583PVVJfPNOvCqWcxnn5lHvw5Qu5Jvt5z3yX+ZR3f5rxc4hf379yshIaHJ2/F23JBOPnakX/BNU3QRLZpPo3SL469xI+C7eCZMmKD8/Hz316WlpUpPT9fOnTtPy8DYnJWXl6tTp07atWuXXC5XoLtjPT6vxisrK1NaWpqSkpIC3ZWTYuwwx9+FxuOzajx/jxt+L1Datm2rsLAwlZSUeFwvKSlRSkrKca+PiopSVFTUcdcTEhL4YWgkl8vFZ+UFPq/GCw09PbdK8nbckBg7/IG/C43HZ9V4/ho3/D76REZGKiMjQwUFBe5rDQ0NKigoUGZmpr+bAxAEGDcA/FCT/IonPz9fubm5uvDCC3XxxRfr6aefVmVlpW677bamaA5AEGDcAPB9TVKgjBw5Unv37tXEiRNVXFys8847T2+//fZxC+BOJCoqSpMmTTrh1C088Vl5h8+r8QLxWfkybkj8+XqDz6rx+Kwaz9+fVYhzuvYRAgAANBKHBQIAAOtQoAAAAOtQoAAAAOtQoAAAAOtQoAAAAOtYV6DMmjVLnTt3VnR0tPr166ePPvoo0F2yziOPPKKQkBCPR8+ePQPdLSu8//77ysnJUWpqqkJCQrR06VKP5x3H0cSJE9WhQwfFxMQoKytLW7ZsCUxnLXCqz+vWW2897mdtyJAhgensj2DcaBzGjpNj7Gi80zVuWFWg/O1vf1N+fr4mTZqkTz75RH379lV2drb27NkT6K5Z55xzztHu3bvdjw8++CDQXbJCZWWl+vbtq1mzZp3w+SeffFIzZ87U3LlztXbtWsXGxio7O1tVPp4K3Vyd6vOSpCFDhnj8rL388sunsYenxrjhHcaOE2PsaLzTNm44Frn44oudvLw899f19fVOamqqM23atAD2yj6TJk1y+vbtG+huWE+Ss2TJEvfXDQ0NTkpKivOHP/zBfa20tNSJiopyXn755QD00C4//Lwcx3Fyc3OdYcOGBaQ/jcW40XiMHY3D2NF4TTluWDODUlNTo/Xr1ysrK8t9LTQ0VFlZWVqzZk0Ae2anLVu2KDU1VV27dtXNN9+snTt3BrpL1tu+fbuKi4s9fsYSEhLUr18/fsZ+xKpVq9S+fXv16NFDd999t/bv3x/oLrkxbniPscN7jB3e88e4YU2Bsm/fPtXX1x93W+vk5GQVFxcHqFd26tevnxYsWKC3335bc+bM0fbt23XZZZepoqIi0F2z2rGfI37GGm/IkCFauHChCgoK9MQTT2j16tUaOnSo6uvrA901SYwb3mLsMMPY4R1/jRtNchYPmtbQoUPd/92nTx/169dP6enpevXVV3X77bcHsGcINqNGjXL/97nnnqs+ffrozDPP1KpVq3TllVcGsGcwwdiB08Ff44Y1Myht27ZVWFiYSkpKPK6XlJQoJSUlQL1qHhITE3XWWWdp69atge6K1Y79HPEzZq5r165q27atNT9rjBu+YexoHMYO35iOG9YUKJGRkcrIyFBBQYH7WkNDgwoKCpSZmRnAntnv0KFD2rZtmzp06BDorlitS5cuSklJ8fgZKy8v19q1a/kZa6Rvv/1W+/fvt+ZnjXHDN4wdjcPY4RvTccOqX/Hk5+crNzdXF154oS6++GI9/fTTqqys1G233Rborlnl/vvvV05OjtLT01VUVKRJkyYpLCxMo0ePDnTXAu7QoUMeVfr27du1YcMGJSUlKS0tTePGjdPUqVPVvXt3denSRQ8//LBSU1M1fPjwwHU6gH7s80pKStLkyZM1YsQIpaSkaNu2bXrggQfUrVs3ZWdnB7DXnhg3Go+x4+QYOxrvtI0bPu8D8rNnn33WSUtLcyIjI52LL77YKSwsDHSXrDNy5EinQ4cOTmRkpHPGGWc4I0eOdLZu3RroblnhvffecyQd98jNzXUc5+h2wYcffthJTk52oqKinCuvvNLZvHlzYDsdQD/2eR0+fNgZPHiw065dOyciIsJJT093xowZ4xQXFwe628dh3Ggcxo6TY+xovNM1boQ4juP4UEgBAAD4nTVrUAAAAI6hQAEAANahQAEAANahQAEAANahQAEAANahQAEAANahQAEAANahQAEAANahQAEAANahQAEAANahQAEAANb5//cYi/zu31bRAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "plt.subplot(1, 2, 1)\n",
        "plt.pcolormesh(example_tokens)\n",
        "plt.title('Token IDs')\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.pcolormesh(example_tokens != 0)\n",
        "plt.title('Mask')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 470
        },
        "id": "-ppg1RJxD7Tk",
        "outputId": "ba1cf77d-6cee-4dff-f762-0a686c9588ac"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "outputs": [],
      "source": [
        "class Encoder(tf.keras.layers.Layer):\n",
        "    def __init__(self, input_vocab_size, embedding_dim, enc_units):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.enc_units = enc_units\n",
        "        self.input_vocab_size = input_vocab_size\n",
        "\n",
        "        # The embedding layer converts tokens to vectors\n",
        "        self.embedding = tf.keras.layers.Embedding(\n",
        "            self.input_vocab_size,\n",
        "            embedding_dim\n",
        "        )\n",
        "\n",
        "        # The GRU RNN layer processes those vectors sequentially.\n",
        "        self.gru = tf.keras.layers.GRU(\n",
        "            self.enc_units,\n",
        "            return_sequences=True,\n",
        "            return_state=True,\n",
        "            recurrent_initializer='glorot_uniform'\n",
        "        )\n",
        "\n",
        "    def call(self, tokens, state=None):\n",
        "        shape_checker = ShapeChecker()\n",
        "        shape_checker(tokens, ('batch', 's'))\n",
        "\n",
        "        # 2. The embedding layer looks up the embedding for each token.\n",
        "        vectors = self.embedding(tokens)\n",
        "        shape_checker(vectors, ('batch', 's', 'embed_dim'))\n",
        "\n",
        "        # 3. The GRU processes the embedding sequence.\n",
        "        #    output shape: (batch, s, enc_units)\n",
        "        #    state shape: (batch, enc_units)\n",
        "        output, state = self.gru(vectors, initial_state=state)\n",
        "        shape_checker(output, ('batch', 's', 'enc_units'))\n",
        "        shape_checker(state, ('batch', 'enc_units'))\n",
        "\n",
        "        # 4. Returns the new sequence and its state.\n",
        "        return output, state"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "cMVp6tHfD7Tk"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "outputs": [],
      "source": [
        "embedding_dim = 256\n",
        "units = 1024"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "X_8LxJFDD7Tk"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input batch, shape (batch): (64,)\n",
            "Input batch tokens, shape (batch, s): (64, 15)\n",
            "Encoder output, shape (batch, s, units): (64, 15, 1024)\n",
            "Encoder state, shape (batch, units): (64, 1024)\n"
          ]
        }
      ],
      "source": [
        "# Convert the input text to tokens.\n",
        "example_tokens = input_text_processor(example_input_batch)\n",
        "\n",
        "# Encode the input sequence.\n",
        "encoder = Encoder(input_text_processor.vocabulary_size(), embedding_dim, units)\n",
        "example_enc_output, example_enc_state = encoder(example_tokens)\n",
        "\n",
        "print(f'Input batch, shape (batch): {example_input_batch.shape}')\n",
        "print(f'Input batch tokens, shape (batch, s): {example_tokens.shape}')\n",
        "print(f'Encoder output, shape (batch, s, units): {example_enc_output.shape}')\n",
        "print(f'Encoder state, shape (batch, units): {example_enc_state.shape}')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xEk4T1d8D7Tl",
        "outputId": "839656db-88af-4839-933c-2110b36ffd93"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "outputs": [],
      "source": [
        "class BahdanauAttention(tf.keras.layers.Layer):\n",
        "    def __init__(self, units):\n",
        "        super().__init__()\n",
        "        # For Eqn. (4), the  Bahdanau attention\n",
        "        self.W1 = tf.keras.layers.Dense(units, use_bias=False)\n",
        "        self.W2 = tf.keras.layers.Dense(units, use_bias=False)\n",
        "\n",
        "        self.attention = tf.keras.layers.AdditiveAttention()\n",
        "\n",
        "    def call(self, query, value, mask):\n",
        "        shape_checker = ShapeChecker()\n",
        "        shape_checker(query, ('batch', 't', 'query_units'))\n",
        "        shape_checker(value, ('batch', 's', 'value_units'))\n",
        "        shape_checker(mask, ('batch', 's'))\n",
        "\n",
        "        # From Eqn. (4), `W1@ht`.\n",
        "        w1_query = self.W1(query)\n",
        "        shape_checker(w1_query, ('batch', 't', 'attn_units'))\n",
        "\n",
        "        # From Eqn. (4), `W2@hs`.\n",
        "        w2_key = self.W2(value)\n",
        "        shape_checker(w2_key, ('batch', 's', 'attn_units'))\n",
        "\n",
        "        query_mask = tf.ones(tf.shape(query)[:-1], dtype=bool)\n",
        "        value_mask = mask\n",
        "\n",
        "        context_vector, attention_weights = self.attention(\n",
        "            inputs=[w1_query, value, w2_key],\n",
        "            mask=[query_mask, value_mask],\n",
        "            return_attention_scores=True,\n",
        "        )\n",
        "        shape_checker(context_vector, ('batch', 't', 'value_units'))\n",
        "        shape_checker(attention_weights, ('batch', 't', 's'))\n",
        "\n",
        "        return context_vector, attention_weights"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "gkvPiwtvD7Tl"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(64, 15)\n"
          ]
        }
      ],
      "source": [
        "attention_layer = BahdanauAttention(units)\n",
        "print((example_tokens != 0).shape)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sMituEowD7Tl",
        "outputId": "88bedfa4-9356-4e38-f559-4e99c8477120"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Attention result shape: (batch_size, query_seq_length, units):           (64, 2, 1024)\n",
            "Attention weights shape: (batch_size, query_seq_length, value_seq_length): (64, 2, 15)\n"
          ]
        }
      ],
      "source": [
        "# Later, the decoder will generate this attention query\n",
        "example_attention_query = tf.random.normal(shape=[len(example_tokens), 2, 10])\n",
        "\n",
        "# Attend to the encoded tokens\n",
        "context_vector, attention_weights = attention_layer(\n",
        "    query=example_attention_query,\n",
        "    value=example_enc_output,\n",
        "    mask=(example_tokens != 0)\n",
        ")\n",
        "\n",
        "print(f'Attention result shape: (batch_size, query_seq_length, units):           {context_vector.shape}')\n",
        "print(f'Attention weights shape: (batch_size, query_seq_length, value_seq_length): {attention_weights.shape}')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y-_CtBIaD7Tl",
        "outputId": "a4642807-f10e-437e-ceb8-8fd3c7320e04"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Mask')"
            ]
          },
          "metadata": {},
          "execution_count": 31
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGzCAYAAAAFROyYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3aUlEQVR4nO3de3xU1b338e9kyI3cIBASqCRcNajFS9QYBVRIiRxEEBQQego+Vj2cSIvRY0+qiLEoioqUlkvr04MvT6EoKqCeI9RGbspVRI/VgqBQaEPCpZBwDSGznj88zOOQoJnZk+w1yefta14vs2evWb/ZCSu/rFm/tT3GGCMAAACLRLkdAAAAwLlIUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHVIUCBJ8ng8evzxx90OI+zGjx+vLl26hNw2MTExvAEBCKtVq1bJ4/HotddeczsUhBkJShjMmTNHHo9Hubm59T7/+eef6/HHH9fu3bvrbfvSSy81boD/67//+7+bZRLithMnTujxxx/XqlWr3A4FaHIvvfSSPB6PPB6P3n///TrPG2PUuXNneTwe3XLLLS5EiEhFghIGCxYsUJcuXbRp0ybt3LmzzvOff/65SkpKrEhQSkpK6n3u5MmTevTRR5skjqb04osvavv27Y3ax4kTJ1RSUkKCghYtLi5OCxcurHN89erV+tvf/qbY2FgXokIkI0FxaNeuXVq3bp1mzJihtLQ0LViwwO2QQhIXF6dWrVq5HUbYRUdHMzACTeCf/umftHjxYp05cybg+MKFC5WTk6OMjAyXIkOkIkFxaMGCBWrbtq0GDx6s22+/vU6C8tJLL+mOO+6QJN10003+qdBVq1apS5cu+uyzz7R69Wr/8RtvvNHf9siRI5o0aZI6d+6s2NhY9ejRQ88884x8Pp//nN27d8vj8ei5557Tb3/7W3Xv3l2xsbG6+uqrtXnzZv9548eP1+zZsyXJ35fH4/E/X98alK1bt2rQoEFKTk5WYmKiBgwYoA0bNtR5fx6PRx988IGKioqUlpamhIQE3XbbbTpw4MC3Xrs333xTHo9H//M//+M/9vrrr8vj8Wj48OEB5/bq1UujRo0KOPb73/9eOTk5io+PV2pqqkaPHq29e/cGnFPfGpRDhw7pn//5n5WcnKw2bdpo3Lhx+uSTT+TxeOqdzfr73/+uYcOGKTExUWlpaXrooYdUW1sr6evrn5aWJkkqKSnxX9ez17K8vFx33XWXLrjgAsXGxqpjx44aOnRovbNpQCS78847dejQIb377rv+Y6dPn9Zrr72mMWPG1Dn/ueee03XXXad27dopPj5eOTk59a4jeffdd9WnTx+1adNGiYmJuuiii/Tzn//8W2Oprq7WLbfcopSUFK1bt875m4Mrmt+fzE1swYIFGj58uGJiYnTnnXdq7ty52rx5s66++mpJUr9+/fSTn/xEs2bN0s9//nP16tVL0te/cGfOnKmJEycqMTFRjzzyiCQpPT1d0tcfG9xwww36+9//rvvuu0+ZmZlat26diouLtW/fPs2cOTMgjoULF+ro0aO677775PF4NH36dA0fPlxfffWVoqOjdd9996msrEzvvvuu/vM///M739dnn32mvn37Kjk5WQ8//LCio6P1m9/8RjfeeKNWr15dZ73NxIkT1bZtW02ZMkW7d+/WzJkzdf/99+uVV145bx99+vSRx+PRmjVr1Lt3b0nS2rVrFRUVFfBZ9oEDB7Rt2zbdf//9/mNPPvmkJk+erJEjR+rHP/6xDhw4oF/96lfq16+ftm7dqjZt2tTbp8/n05AhQ7Rp0yZNmDBB2dnZWrZsmcaNG1fv+bW1tSooKFBubq6ee+45/elPf9Lzzz+v7t27a8KECUpLS9PcuXM1YcIE3Xbbbf7E6uz7GTFihD777DNNnDhRXbp00f79+/Xuu+9qz549IS/eBWzUpUsX5eXl6Q9/+IMGDRokSXrnnXdUWVmp0aNHa9asWQHn//KXv9Stt96qsWPH6vTp01q0aJHuuOMOvf322xo8eLCkr8ehW265Rb1799YTTzyh2NhY7dy5Ux988MF54zh58qSGDh2qDz/8UH/605/8YzEikEHIPvzwQyPJvPvuu8YYY3w+n7ngggvMT3/604DzFi9ebCSZlStX1nmNSy65xNxwww11jv/iF78wCQkJ5osvvgg4/u///u/G6/WaPXv2GGOM2bVrl5Fk2rVrZ/7xj3/4z1u2bJmRZN566y3/scLCQnO+b7kkM2XKFP/Xw4YNMzExMebLL7/0HysrKzNJSUmmX79+/mPz5883kkx+fr7x+Xz+4w888IDxer3myJEj9fb3zfc/cuRI/9dXXnmlueOOO4wk85e//MUYY8wbb7xhJJlPPvnEGGPM7t27jdfrNU8++WTAa3366aemVatWAcfHjRtnsrKy/F+//vrrRpKZOXOm/1htba3p37+/kWTmz58f0FaSeeKJJwL6ueKKK0xOTo7/6wMHDtS5fsYYc/jwYSPJPPvss996DYBIdnYM2Lx5s/n1r39tkpKSzIkTJ4wxxtxxxx3mpptuMsYYk5WVZQYPHuxvd/acs06fPm0uvfRS079/f/+xF154wUgyBw4cOG//K1euNJLM4sWLzdGjR80NN9xg2rdvb7Zu3RrGdwk38BGPAwsWLFB6erpuuukmSV9/TDJq1CgtWrTI/xFAqBYvXqy+ffuqbdu2OnjwoP+Rn5+v2tparVmzJuD8UaNGqW3btv6v+/btK0n66quvgu67trZWf/zjHzVs2DB169bNf7xjx44aM2aM3n//fVVVVQW0uffeewM+Murbt69qa2v117/+9Vv76tu3r9auXStJOnr0qD755BPde++9at++vf/42rVr1aZNG1166aWSpDfeeEM+n08jR44MuDYZGRnq2bOnVq5ced7+li9frujoaN1zzz3+Y1FRUSosLDxvm3/5l3+pE3NDrmt8fLxiYmK0atUqHT58+DvPByLdyJEjdfLkSb399ts6evSo3n777Xo/3pG+/vdx1uHDh1VZWam+ffvqo48+8h8/OxO6bNmygI+261NZWamBAwdq27ZtWrVqlS6//HLH7wfuIkEJUW1trRYtWqSbbrpJu3bt0s6dO7Vz507l5uaqoqJCpaWljl5/x44dWr58udLS0gIe+fn5kqT9+/cHnJ+ZmRnw9dlkJZRfjAcOHNCJEyd00UUX1XmuV69e8vl8ddZ6hNp/3759tW/fPu3cuVPr1q2Tx+NRXl5eQOKydu1aXX/99YqK+vrHdceOHTLGqGfPnnWuz1/+8pc61+ab/vrXv6pjx45q3bp1wPEePXrUe35cXJx/jck331tDrmtsbKyeeeYZvfPOO0pPT1e/fv00ffp0lZeXf2dbIBKdHaMWLlyoN954Q7W1tbr99tvrPfftt9/Wtddeq7i4OKWmpvo/Lq2srPSfM2rUKF1//fX68Y9/rPT0dI0ePVqvvvpqvcnKpEmTtHnzZv3pT3/SJZdc0mjvEU2HNSgheu+997Rv3z4tWrRIixYtqvP8ggULNHDgwJBf3+fz6Qc/+IEefvjhep+/8MILA772er31nmeMCTmGYITaf58+fSRJa9as0VdffaUrr7xSCQkJ6tu3r2bNmqVjx45p69atevLJJ/1tfD6fPB6P3nnnnXr7Defmaud7Xw01adIkDRkyREuXLtWKFSs0efJkTZs2Te+9956uuOKKMEUJ2GPMmDG65557VF5erkGDBtW7Hmzt2rW69dZb1a9fP82ZM0cdO3ZUdHS05s+fH1CqHB8frzVr1mjlypX6r//6Ly1fvlyvvPKK+vfvrz/+8Y8B/z6HDh2qRYsW6emnn9bLL7/s/4MGkYsEJUQLFixQhw4d/JUx3/TGG29oyZIlmjdvnuLj4wM++jjX+Z7r3r27jh075p8xCYdvi+Ob0tLS1Lp163r3D9m2bZuioqLUuXPnsMSUmZmpzMxMrV27Vl999ZX/o6l+/fqpqKhIixcvVm1trfr16+dv0717dxlj1LVr1zqJ2nfJysrSypUrdeLEiYBZlPr2r2mo77qu3bt314MPPqgHH3xQO3bs0OWXX67nn39ev//970PuE7DVbbfdpvvuu08bNmw47yL5119/XXFxcVqxYkXANgDz58+vc25UVJQGDBigAQMGaMaMGXrqqaf0yCOPaOXKlQHj47BhwzRw4ECNHz9eSUlJmjt3bvjfHJoUKWYITp48qTfeeEO33HKLbr/99jqP+++/X0ePHtWbb74pSUpISJD0ddnwuRISEuo9PnLkSK1fv14rVqyo89yRI0fq7DXQEN8Wxzd5vV4NHDhQy5YtCyiHraio0MKFC9WnTx8lJycH3f/59O3bV++99542bdrkT1Auv/xyJSUl6emnn/aXIJ41fPhweb1elZSU1JmhMcbo0KFD5+2roKBANTU1evHFF/3HfD5fvYlmQ51NdM69ridOnNCpU6cCjnXv3l1JSUmqrq4OuT/AZomJiZo7d64ef/xxDRkypN5zvF6vPB5PwFq93bt3a+nSpQHn/eMf/6jT9uzakvr+Df3oRz/SrFmzNG/ePP3sZz8L/U3ACsyghODNN9/U0aNHdeutt9b7/LXXXuvftG3UqFG6/PLL5fV69cwzz6iyslKxsbHq37+/OnTooJycHM2dO1dTp05Vjx491KFDB/Xv31//9m//pjfffFO33HKLxo8fr5ycHB0/flyffvqpXnvtNe3evVvt27cPKu6zv+R/8pOfqKCgQF6vV6NHj6733KlTp/r3H/jXf/1XtWrVSr/5zW9UXV2t6dOnB3fBvkPfvn21YMECeTwe/0c+Xq9X1113nVasWKEbb7xRMTEx/vO7d++uqVOnqri4WLt379awYcOUlJSkXbt2acmSJbr33nv10EMP1dvXsGHDdM011+jBBx/Uzp07lZ2drTfffNM/EDZ0lumb4uPjdfHFF+uVV17RhRdeqNTUVF166aU6c+aMBgwYoJEjR+riiy9Wq1attGTJElVUVJz3ugPNwfnK9s8aPHiwZsyYoZtvvlljxozR/v37NXv2bPXo0SNgX6QnnnhCa9as0eDBg5WVlaX9+/drzpw5uuCCC/xjxbnuv/9+VVVV6ZFHHlFKSsp37pkCi7lZQhSphgwZYuLi4szx48fPe8748eNNdHS0OXjwoDHGmBdffNF069bNeL3egJLj8vJyM3jwYJOUlGQkBZQcHz161BQXF5sePXqYmJgY0759e3PdddeZ5557zpw+fdoY8//LjOsrZdU5pa9nzpwxEydONGlpacbj8QSUHJ97rjHGfPTRR6agoMAkJiaa1q1bm5tuusmsW7cu4Jxvlhh+09nSv/pKq8/12WefGUmmV69eAcenTp1qJJnJkyfX2+711183ffr0MQkJCSYhIcFkZ2ebwsJCs337dv8555YZG/N1WfCYMWNMUlKSSUlJMePHjzcffPCBkWQWLVoU0DYhIaFOv1OmTKlTrr1u3TqTk5NjYmJi/Nfy4MGDprCw0GRnZ5uEhASTkpJicnNzzauvvvqd1wSIFOcbA851bpnx7373O9OzZ08TGxtrsrOzzfz58+v82yotLTVDhw41nTp1MjExMaZTp07mzjvvDNh+4Ztlxt/08MMPG0nm17/+dZjeKZqax5gmWkUJWGzp0qW67bbb9P777+v66693OxwAaPFIUNDinDx5MmAPhtraWg0cOFAffvihysvLA54DALiDNShocSZOnKiTJ08qLy9P1dXVeuONN7Ru3To99dRTJCcAYAlmUNDiLFy4UM8//7x27typU6dOqUePHpowYULAvX4AAO4iQQEAANZhHxQAAGAdEhQAAGAd6xbJ+nw+lZWVKSkpKaRNswA4Z4zR0aNH1alTp4i5pwljB+CucI8b1iUoZWVlYbvPCwBn9u7dqwsuuMDtMBqEsQOwQ7jGDesSlKSkJElSl589pqjYuOBfwK0lv07+YHMSs5t/KLr1np2IwCXhmb/Y2OR9nlGN3td/+/89RoKzsf71oy5KToyMWR+gsdx24febvM9wjxvWJShnp2ajYuMUFRd8guJx6ReQcfDL2knMTvp1LAITFLd+Ppxo5Ylu+k7/9zpF0kclZ2NNToxScpLX5WgAdzWHcYM/MwAAgHVIUAAAgHWs+4jnrJwbtik6ISbodlEOPjvwubSgw+fgc5ooFz+zcBL3wesOhzESAC1BQafL3A4BTYgZFAAAYB0SFAAAYB0SFAAAYB0SFAAAYB0SFAAAYB1rq3h2Hm4vb3Vs0O2cVOI4qQBysi+NcVCI41blkeSwYuq/0kJumzr4i5DbAohcK8o+CbktFUCRhxkUAABgHRIUAABgHRIUAABgHRIUAABgHRIUAABgHWureI5+2F7e2LjgG7p1axonxTTu3U7HESe3ATIOUuPjj7V30HHoTV0rmHIQc+dfrAtfHEAEc1IB1NLYUvHEDAoAALBO0AnK3//+d/3whz9Uu3btFB8fr+9///v68MMP/c8bY/TYY4+pY8eOio+PV35+vnbs2BHWoAFEHsYOAMEIKkE5fPiwrr/+ekVHR+udd97R559/rueff15t27b1nzN9+nTNmjVL8+bN08aNG5WQkKCCggKdOnUq7MEDiAyMHQCCFdQalGeeeUadO3fW/Pnz/ce6du3q/39jjGbOnKlHH31UQ4cOlSS9/PLLSk9P19KlSzV69OgwhQ0gkjB2AAhWUDMob775pq666irdcccd6tChg6644gq9+OKL/ud37dql8vJy5efn+4+lpKQoNzdX69evr/c1q6urVVVVFfAA0LwwdgAIVlAzKF999ZXmzp2roqIi/fznP9fmzZv1k5/8RDExMRo3bpzKy8slSenp6QHt0tPT/c+da9q0aSopKalzvCbFqDYuhPIF925NEzon5TBOGZcumJO37FYljoN+uz9Y/y/ZlqIpxw6gubClmsYtQc2g+Hw+XXnllXrqqad0xRVX6N5779U999yjefPmhRxAcXGxKisr/Y+9e/eG/FoA7MTYASBYQSUoHTt21MUXXxxwrFevXtqzZ48kKSMjQ5JUUVERcE5FRYX/uXPFxsYqOTk54AGgeWHsABCsoBKU66+/Xtu3bw849sUXXygrK0vS14veMjIyVFpa6n++qqpKGzduVF5eXhjCBRCJGDsABCuoNSgPPPCArrvuOj311FMaOXKkNm3apN/+9rf67W9/K0nyeDyaNGmSpk6dqp49e6pr166aPHmyOnXqpGHDhjVG/AAiAGMHgGAFlaBcffXVWrJkiYqLi/XEE0+oa9eumjlzpsaOHes/5+GHH9bx48d177336siRI+rTp4+WL1+uuLgQtq0H0CwwdgAIlscYY9WdYKqqqpSSkqIu//GoolqHci+e0Ms07LoSTcPjpKrFreojB9/jbmO2hjGQ5uuMqdEqLVNlZWXErO04O3Yc/qKbkpO8boeDZqSlV9M0VLjHDe7FAwAArEOCAgAArEOCAgAArEOCAgAArBNUFU+TMp7QFkM6WbjpYPGlo8WmDkTswl4WMwMAvgUzKAAAwDokKAAAwDokKAAAwDokKAAAwDokKAAAwDrWVvGYA3EyTX0PDncKgJxxuN28o9Y+t0qXQm/65Qx37ozbvWi9K/0CcG5F2Seu9NvSt9hnBgUAAFiHBAUAAFiHBAUAAFiHBAUAAFiHBAUAAFjH2iqef/vBm4pPbNrwaoy3Sfs7y+vxhdw22lPrqG8n7/mV7AxHfQNoeVp6ZQoajhkUAABgHRIUAABgHRIUAABgHRIUAABgHRIUAABgHWureGYsG6qoEO7F4+TWNE7up+Pwljghc+0eQJL0ZOhNI/H75Na17vLIOnc6BhqBW/e1aWmaQ7UUMygAAMA6JCgAAMA6JCgAAMA6JCgAAMA61i6S3fzDF5WcFHz+VGucbBsf+uWoMWdCbuumW76X43YIACJMc1iACfsxgwIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKxjbRXPq8faK76Jw6sx3ibt76xa416eePtf9ofc9rVeHcIYCYBI4WS7eiqA0FDMoAAAAOuQoAAAAOuQoAAAAOuQoAAAAOuQoAAAAOtYW8Xz9B+HKSouLviGxkGnHgdtnXASs1NO3vMLDtq6+Z5d0L1ovdshAFZwUgHU0rT0iidmUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHWsreJJ7XFI3taxQbfzOShLiXJQWuJWv27yOKgAqjWhN04d/EXoHQNokVp6RUwkYgYFAABYJ6gE5fHHH5fH4wl4ZGdn+58/deqUCgsL1a5dOyUmJmrEiBGqqKgIe9AAIgtjB4BgBT2Dcskll2jfvn3+x/vvv+9/7oEHHtBbb72lxYsXa/Xq1SorK9Pw4cPDGjCAyMTYASAYQa9BadWqlTIyMuocr6ys1O9+9zstXLhQ/fv3lyTNnz9fvXr10oYNG3Tttdc6jxZAxGLsABCMoGdQduzYoU6dOqlbt24aO3as9uzZI0nasmWLampqlJ+f7z83OztbmZmZWr/+/Nt8V1dXq6qqKuABoPlh7AAQjKBmUHJzc/XSSy/poosu0r59+1RSUqK+ffvqz3/+s8rLyxUTE6M2bdoEtElPT1d5efl5X3PatGkqKSmpc/zEB+3ljW3ie/G4xcn9cCLx/TpU9rP2oTd2cr3cutYu3COqtvqUNHNZ2F6vKccOoD7cA6jxVR2tVdsLw/d6QSUogwYN8v9/7969lZubq6ysLL366quKj48PKYDi4mIVFRX5v66qqlLnzp1Dei0AdmLsABAsR2XGbdq00YUXXqidO3cqIyNDp0+f1pEjRwLOqaioqPdz57NiY2OVnJwc8ADQvDF2APgujhKUY8eO6csvv1THjh2Vk5Oj6OholZaW+p/fvn279uzZo7y8PMeBAmg+GDsAfJegPuJ56KGHNGTIEGVlZamsrExTpkyR1+vVnXfeqZSUFN19990qKipSamqqkpOTNXHiROXl5bEKH2jhGDsABCuoBOVvf/ub7rzzTh06dEhpaWnq06ePNmzYoLS0NEnSCy+8oKioKI0YMULV1dUqKCjQnDlzQgpsw4T/q+Qkb9DtfPKF1J+bakyta33f+r2rXOsb9jpjarQ9jK/XlGMHGh/bxqM+Z0yNpK/C9noeY4xVdSBVVVVKSUnR4S+6kaA0ARIU1OeMqdEqLVNlZWXErO1wOnag4UhQUJ9wjxvciwcAAFiHBAUAAFiHBAUAAFiHBAUAAFgn6JsFNpX79l6vmISYJu3T52BPcZ9xYT/yMMhYH/qiYifXa3/ekZDbAnCXW9vGszi3ZWEGBQAAWIcEBQAAWIcEBQAAWIcEBQAAWIcEBQAAWMfaKp6Nm7IVFRcXfEMnxTRWbfrfQG4WD3kcXLAXHATu1vfJQb/dH1wfvjiAFsqt6iG3tPSqJWZQAACAdUhQAACAdUhQAACAdUhQAACAdUhQAACAdayt4vG1rpXia90Oo+GcVJZE5m18Wpye921yOwQAEaalV+I4wQwKAACwDgkKAACwDgkKAACwDgkKAACwjrWLZKNOehVlvG6H0XCRuE2+U5F4WwEHMX85Iy/ktt2L2OoeaImcbM/f0hfYMoMCAACsQ4ICAACsQ4ICAACsQ4ICAACsQ4ICAACsY20VT8SJxIoWp31H4hb9LbHaCgAiEDMoAADAOiQoAADAOiQoAADAOiQoAADAOiQoAADAOtZW8Xg6nJSndfAlFx5P6GUaxjgoS3HQ1kRqZYmDuD0upcbdxmx1p2MAEaul3xPHLcygAAAA65CgAAAA65CgAAAA65CgAAAA65CgAAAA61hbxdM2+YS8rWvdDqPBfA5uTON1VHkUclPHnLxnJ1IHf+FKvwAiF5U4kYcZFAAAYB0SFAAAYB0SFAAAYB0SFAAAYB0SFAAAYB1rq3gOVyYoqiYu+IZOKmJ8oVeleJzcxsfRPW1cLONxcu8iBw4vuCLktt3Hci8eoCVaUfZJyG2pAHKHoxmUp59+Wh6PR5MmTfIfO3XqlAoLC9WuXTslJiZqxIgRqqiocBongGaCcQNAQ4ScoGzevFm/+c1v1Lt374DjDzzwgN566y0tXrxYq1evVllZmYYPH+44UACRj3EDQEOFlKAcO3ZMY8eO1Ysvvqi2bdv6j1dWVup3v/udZsyYof79+ysnJ0fz58/XunXrtGHDhrAFDSDyMG4ACEZICUphYaEGDx6s/Pz8gONbtmxRTU1NwPHs7GxlZmZq/fr19b5WdXW1qqqqAh4Amp9wjhsSYwfQ3AW9SHbRokX66KOPtHnz5jrPlZeXKyYmRm3atAk4np6ervLy8npfb9q0aSopKalz/KHL/6j4xODX8Lq1/XqtCX05j9fjC7ltlJwtknVyvV7JznDUN1qOcI8b0vnHDtiNBadoqKB+q+7du1c//elPtWDBAsXFhVBhU4/i4mJVVlb6H3v37g3L6wKwQ2OMGxJjB9DcBZWgbNmyRfv379eVV16pVq1aqVWrVlq9erVmzZqlVq1aKT09XadPn9aRI0cC2lVUVCgjo/6/tmNjY5WcnBzwANB8NMa4ITF2AM1dUJ+hDBgwQJ9++mnAsbvuukvZ2dn62c9+ps6dOys6OlqlpaUaMWKEJGn79u3as2eP8vLywhc1gIjBuAEgFEElKElJSbr00ksDjiUkJKhdu3b+43fffbeKioqUmpqq5ORkTZw4UXl5ebr22mvDFzWAiMG4ASAUYd9J9oUXXlBUVJRGjBih6upqFRQUaM6cOeHuBkAzwrgB4FweY5xstB5+VVVVSklJ0ZWvFcmbEBt0+1qXtl83Dvp1UsXTErUZvMPtEJq9M6ZGq7RMlZWVEbO24+zYcfiLbkpO8rodDixD9VDjC/e4wc0CAQCAdUhQAACAdUhQAACAdUhQAACAdUhQAACAdcJeZhwupb2XhrQSv8acCblPryf0fK3WRGYlzi3fy3E7BAARhooYNAVmUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHVIUAAAgHWsreK5Yu1YRbWOC76hx8GthVy6j4+bzAK3I2ha3cdudTsEIOKtKPvE7RCaFFVL7mAGBQAAWIcEBQAAWIcEBQAAWIcEBQAAWMfaRbKpKcflTQh+23rjYI2sT6Evko1S6B076ddNXgcLkp18n9oM3hF6YwAtEgtdIw8zKAAAwDokKAAAwDokKAAAwDokKAAAwDokKAAAwDrWVvH8ozJBUTXBb3XvpDrELZ7ILOJx7bYC/1h4Rchtu41hq3ugJXKyPT8VQO5gBgUAAFiHBAUAAFiHBAUAAFiHBAUAAFiHBAUAAFjH2ioeeUxIVSKeCLyvjZPKIzcrgIwv9M6dxB2JlVoAgOAwgwIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKxjbRXPlj4LlJwUfP5UY2obIZrvFu3xhtw2ykGeWGPOhNxWkm75Xo6j9gBaHu5Ng6bADAoAALAOCQoAALAOCQoAALAOCQoAALCOtYtkr/79PYqKiwu+oYMt1D2+0Ns6YZzE7HDbd/Oke327wcm1dkuXR9a5HQIQYEXZJ26HgO/QHBYyM4MCAACsQ4ICAACsQ4ICAACsQ4ICAACsQ4ICAACsY20VjxvcqqZx0tZpVYpblThuVi4BAOzHDAoAALBOUAnK3Llz1bt3byUnJys5OVl5eXl65513/M+fOnVKhYWFateunRITEzVixAhVVFSEPWgAkYWxA0CwgkpQLrjgAj399NPasmWLPvzwQ/Xv319Dhw7VZ599Jkl64IEH9NZbb2nx4sVavXq1ysrKNHz48EYJHEDkYOwAECyPMcbRJ/qpqal69tlndfvttystLU0LFy7U7bffLknatm2bevXqpfXr1+vaa69t0OtVVVUpJSVF3R57qsl3kpVL60icYA1K0/XrFjd2kj1jarRKy1RZWank5ORG6aOxxo7DX3RTcpK3UWIGIoUbO8mGe9wIeQ1KbW2tFi1apOPHjysvL09btmxRTU2N8vPz/edkZ2crMzNT69evP+/rVFdXq6qqKuABoPli7ADQEEFX8Xz66afKy8vTqVOnlJiYqCVLlujiiy/Wxx9/rJiYGLVp0ybg/PT0dJWXl5/39aZNm6aSkpI6x4uGLlN8YvBFRj4HUyi1JvQ1w14HN/KJcjB14+T9OvVKdoZrfSPyNNXYAbs1h3vEoGkE/Rv5oosu0scff6yNGzdqwoQJGjdunD7//POQAyguLlZlZaX/sXfv3pBfC4C9GDsABCPoKYqYmBj16NFDkpSTk6PNmzfrl7/8pUaNGqXTp0/ryJEjAX8JVVRUKCPj/H9px8bGKjY2NvjIAUQUxg4AwXC8D4rP51N1dbVycnIUHR2t0tJS/3Pbt2/Xnj17lJeX57QbAM0MYweAbxPUDEpxcbEGDRqkzMxMHT16VAsXLtSqVau0YsUKpaSk6O6771ZRUZFSU1OVnJysiRMnKi8vr8Gr8AE0T4wdAIIVVIKyf/9+/ehHP9K+ffuUkpKi3r17a8WKFfrBD34gSXrhhRcUFRWlESNGqLq6WgUFBZozZ06jBA4gcjB2AAiW431Qwu3sXgadf/mEouJD2AfFAUd7c4RexONs7xY0mZ73bXI7hCbTFPughBv7oMBGLalqyZp9UAAAABoLCQoAALAOCQoAALAOCQoAALAOCQoAALBO8De7aSLepNOKah1C/hSJt6t1wM0aLE8EXupuY7a6HQKACNOSKnFswgwKAACwDgkKAACwDgkKAACwDgkKAACwjrWLZK/J+qtiEmKCbudzsG+8z6UFtlEeq+420CT25x1xOwQAEYbFqi0LMygAAMA6JCgAAMA6JCgAAMA6JCgAAMA6JCgAAMA61lbxGHlCqshxUonjpJomSqG3dVJ55KRfp327VfUEAGj+mEEBAADWIUEBAADWIUEBAADWIUEBAADWIUEBAADWsbaKZ/PuLEW1jgu6nXFQ1OKhKKXpLOwSclMn3+PuY7eG3hiAq1aUfeJKv9wDyB3MoAAAAOuQoAAAAOuQoAAAAOuQoAAAAOuQoAAAAOtYW8XTNuW4vK3PNGmfTqp4nFSWOOHkXjqRKnXwF26HACDCUIkTeZhBAQAA1iFBAQAA1iFBAQAA1iFBAQAA1rF2kWyPtgcVnRATdDufCX3RaJTHnZWuTmJu5fE56tvJez5jQs9v9+cdCbktgJaJha4tCzMoAADAOiQoAADAOiQoAADAOiQoAADAOiQoAADAOtZW8YTKaVWLG5xsV3/a5+xbGBWB1wsA0PwxgwIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKxjbRXPltXZioqLczuMhgu9EEduFtI4uA2QI56pobd1K2YnnNzmKevRdeELBIhgK8o+cTuEiNEc7lvEDAoAALBOUAnKtGnTdPXVVyspKUkdOnTQsGHDtH379oBzTp06pcLCQrVr106JiYkaMWKEKioqwho0gMjC2AEgWEElKKtXr1ZhYaE2bNigd999VzU1NRo4cKCOHz/uP+eBBx7QW2+9pcWLF2v16tUqKyvT8OHDwx44gMjB2AEgWB5jTMifjh84cEAdOnTQ6tWr1a9fP1VWViotLU0LFy7U7bffLknatm2bevXqpfXr1+vaa6/9ztesqqpSSkqKuj32FGtQmoBra1AcrMlgDUrjO2NqtErLVFlZqeTk5LC/fmOOHYe/6KbkJG/YYwYiiRtrUMI9bjhag1JZWSlJSk1NlSRt2bJFNTU1ys/P95+TnZ2tzMxMrV+/vt7XqK6uVlVVVcADQPPG2AHgu4RcxePz+TRp0iRdf/31uvTSSyVJ5eXliomJUZs2bQLOTU9PV3l5eb2vM23aNJWUlNR9/ayTUuvg/+wMfT7IGY9bf9U7+dPcIeOLvKmM7mO3uh1Ci9fYYwcQbs2hIiYShTyDUlhYqD//+c9atGiRowCKi4tVWVnpf+zdu9fR6wGwG2MHgIYIaQbl/vvv19tvv601a9boggsu8B/PyMjQ6dOndeTIkYC/hCoqKpSRkVHva8XGxio2NjaUMABEGMYOAA0V1AyKMUb333+/lixZovfee09du3YNeD4nJ0fR0dEqLS31H9u+fbv27NmjvLy88EQMIOIwdgAIVlAzKIWFhVq4cKGWLVumpKQk/2fDKSkpio+PV0pKiu6++24VFRUpNTVVycnJmjhxovLy8hq0Ch9A88TYASBYQSUoc+fOlSTdeOONAcfnz5+v8ePHS5JeeOEFRUVFacSIEaqurlZBQYHmzJkTlmABRCbGDgDBCipBaciWKXFxcZo9e7Zmz54dclBOuFZN4xaHm4I4qXpydK0dVB9FYvVQSxcJYwcAu3AvHgAAYB0SFAAAYB0SFAAAYB0SFAAAYJ2Qt7pvbGZ/nExT3ywwEtdeOt3p3sF7dm+T/dB9OcOdPTW6F9V/PxkA9ltR9okr/bb0LfaZQQEAANYhQQEAANYhQQEAANYhQQEAANYhQQEAANaxtoon+ohH3tgQSkzcqsRxsmW8g7YmUlNMt0qAnPx8OIh57+TrHHQcus6/WOdKvwCca+nVQ5H66w0AADRjJCgAAMA6JCgAAMA6JCgAAMA6JCgAAMA61lbxJF19UN7WsW6HgUbic1BOkzr4izBGAqAlsKUyBQ3HDAoAALAOCQoAALAOCQoAALAOCQoAALAOCQoAALCOtVU8PdoeVHRCjNthNJjPhF6VEuXgZjxRDm9q46Saxsl7Pnjd4ZDbAmiZqMRpWZhBAQAA1iFBAQAA1iFBAQAA1iFBAQAA1rF2kawb3Fro2srjC7ntaZ835LaSs7gBAGgszKAAAADrkKAAAADrkKAAAADrkKAAAADrkKAAAADrWFvFs3l3lqJaxzVtpw4qWowv9AogT+hNZRwW4Tjp25GFXUJu6uQ9dx+7NfTGAFy1ouwTV/pli313MIMCAACsQ4ICAACsQ4ICAACsQ4ICAACsQ4ICAACsY20VT6sd8fLGNnEVT0u8LY2TKh63rpeDmPdOvi58cUSAzr9Y53YIQMRzq3rILbZULTGDAgAArEOCAgAArEOCAgAArEOCAgAArEOCAgAArGNtFc/m//N/lZzkDbpdjTkTcp9ejzv5Wo2pdaVfSbr1e1e51jeAyGRLlQeaN2ZQAACAdYJOUNasWaMhQ4aoU6dO8ng8Wrp0acDzxhg99thj6tixo+Lj45Wfn68dO3aEK14AEYhxA0Cwgk5Qjh8/rssuu0yzZ8+u9/np06dr1qxZmjdvnjZu3KiEhAQVFBTo1KlTjoMFEJkYNwAEK+g1KIMGDdKgQYPqfc4Yo5kzZ+rRRx/V0KFDJUkvv/yy0tPTtXTpUo0ePdpZtAAiEuMGgGCFdQ3Krl27VF5ervz8fP+xlJQU5ebmav369fW2qa6uVlVVVcADQMsRyrghMXYAzV1YE5Ty8nJJUnp6esDx9PR0/3PnmjZtmlJSUvyPzp07S5J8MvKF8J8TUS79B7RkoYwb0vnHDgDNg+u/HYuLi1VZWel/7N271+2QAEQAxg6geQtrgpKRkSFJqqioCDheUVHhf+5csbGxSk5ODngAaDlCGTckxg6guQtrgtK1a1dlZGSotLTUf6yqqkobN25UXl5eOLsC0EwwbgCoT9BVPMeOHdPOnTv9X+/atUsff/yxUlNTlZmZqUmTJmnq1Knq2bOnunbtqsmTJ6tTp04aNmxYOOMGEEEYNwAEK+gE5cMPP9RNN93k/7qoqEiSNG7cOL300kt6+OGHdfz4cd177706cuSI+vTpo+XLlysuLi6ofq7+/T2KCrKNJHlM0E3Cwnjc6dexJ0NvyrVuOCfXKuvRdeELxCVNNW6gaawo+8TtEPAdmsPtCDzGGJd+zdSvqqpKKSkp6vbYUyQoluNaN1ykJShnTI1WaZkqKysjZm3H2bHj8BfdQrqPF9CcuJGghHvccL2KBwAA4FwkKAAAwDokKAAAwDokKAAAwDpBV/E0lc0/fFHJScHnTzWmNuQ+oz2hL6xzsmV9jTkTclufnK1UvfV7VzlqD6DlaQ4VIrAfMygAAMA6JCgAAMA6JCgAAMA6JCgAAMA6JCgAAMA61lbx5Kz9oaJah3AfDid7ijvYQ92tGwZ4HG77bha41LdL36duY7aG3i8ASS3vXjxULbmDGRQAAGAdEhQAAGAdEhQAAGAdEhQAAGAdEhQAAGAda6t4Wu2Ilzc2hCoeB8UhLhWWSA4rcRxxUn3k1nv2hd507+TrHHQceTr/Yp3bIQARj6oldzCDAgAArEOCAgAArEOCAgAArEOCAgAArGPtItmkqw7K2zo26HY+B6svoxysGHWy7XutoxW2zngdrAx2K+7UwV+40i+AyGXLwk80HDMoAADAOiQoAADAOiQoAADAOiQoAADAOiQoAADAOtZW8fxjZztFxTXtVveORDno2OegGsbp+3Vzm/0QHX4hz52OHVzr7kXrwxcHgKBF4nb1Lb3yiBkUAABgHRIUAABgHRIUAABgHRIUAABgHRIUAABgHWureEIWgVUprlUAucmtaisAQERgBgUAAFiHBAUAAFiHBAUAAFiHBAUAAFiHBAUAAFin+VXxOOGkssS4VE3jtFu3qmkitPgIANA0mEEBAADWIUEBAADWIUEBAADWIUEBAADWIUEBAADWsbaKx5N2Sp7WoTR0UJbiViWOE07eryQTqffyCVH3sVvdDgFAhCnodJnbIbRIjTaDMnv2bHXp0kVxcXHKzc3Vpk2bGqsrAM0E4waAsxolQXnllVdUVFSkKVOm6KOPPtJll12mgoIC7d+/vzG6A9AMMG4A+KZGSVBmzJihe+65R3fddZcuvvhizZs3T61bt9Z//Md/NEZ3AJoBxg0A3xT2NSinT5/Wli1bVFxc7D8WFRWl/Px8rV+/vs751dXVqq6u9n9dWVkpSfKdrK5zboOwBiUoLW0NyhlT43YIEeGMvr5OxjTNVsPBjhvS+ceOqmO+xg0WLQ7jRsOEe9wIe4Jy8OBB1dbWKj09PeB4enq6tm3bVuf8adOmqaSkpM7xPfdPD3dogP7qdgAR5tChQ0pJSWn0foIdN6Tzjx1ZV+5ujBDRon3ldgARJVzjhutVPMXFxSoqKvJ/feTIEWVlZWnPnj1NMjBGsqqqKnXu3Fl79+5VcnKy2+FYj+vVcJWVlcrMzFRqaqrboZwXY0fo+LfQcFyrhgv3uBH2BKV9+/byer2qqKgIOF5RUaGMjIw658fGxio2NrbO8ZSUFH4YGig5OZlrFQSuV8NFRTXNVknBjhsSY0c48G+h4bhWDReucSPso09MTIxycnJUWlrqP+bz+VRaWqq8vLxwdwegGWDcAHCuRvmIp6ioSOPGjdNVV12la665RjNnztTx48d11113NUZ3AJoBxg0A39QoCcqoUaN04MABPfbYYyovL9fll1+u5cuX11kAV5/Y2FhNmTKl3qlbBOJaBYfr1XBuXCsn44bE9zcYXKuG41o1XLivlcc0VR0hAABAA3GzQAAAYB0SFAAAYB0SFAAAYB0SFAAAYB0SFAAAYB3rEpTZs2erS5cuiouLU25urjZt2uR2SNZ5/PHH5fF4Ah7Z2dluh2WFNWvWaMiQIerUqZM8Ho+WLl0a8LwxRo899pg6duyo+Ph45efna8eOHe4Ea4Hvul7jx4+v87N28803uxPst2DcaBjGjvNj7Gi4pho3rEpQXnnlFRUVFWnKlCn66KOPdNlll6mgoED79+93OzTrXHLJJdq3b5//8f7777sdkhWOHz+uyy67TLNnz673+enTp2vWrFmaN2+eNm7cqISEBBUUFOjUqVNNHKkdvut6SdLNN98c8LP2hz/8oQkj/G6MG8Fh7KgfY0fDNdm4YSxyzTXXmMLCQv/XtbW1plOnTmbatGkuRmWfKVOmmMsuu8ztMKwnySxZssT/tc/nMxkZGebZZ5/1Hzty5IiJjY01f/jDH1yI0C7nXi9jjBk3bpwZOnSoK/E0FONGwzF2NAxjR8M15rhhzQzK6dOntWXLFuXn5/uPRUVFKT8/X+vXr3cxMjvt2LFDnTp1Urdu3TR27Fjt2bPH7ZCst2vXLpWXlwf8jKWkpCg3N5efsW+xatUqdejQQRdddJEmTJigQ4cOuR2SH+NG8Bg7gsfYEbxwjBvWJCgHDx5UbW1tnW2t09PTVV5e7lJUdsrNzdVLL72k5cuXa+7cudq1a5f69u2ro0ePuh2a1c7+HPEz1nA333yzXn75ZZWWluqZZ57R6tWrNWjQINXW1rodmiTGjWAxdoSGsSM44Ro3GuVePGhcgwYN8v9/7969lZubq6ysLL366qu6++67XYwMzc3o0aP9///9739fvXv3Vvfu3bVq1SoNGDDAxcgQCsYONIVwjRvWzKC0b99eXq9XFRUVAccrKiqUkZHhUlSRoU2bNrrwwgu1c+dOt0Ox2tmfI37GQtetWze1b9/emp81xg1nGDsahrHDmVDHDWsSlJiYGOXk5Ki0tNR/zOfzqbS0VHl5eS5GZr9jx47pyy+/VMeOHd0OxWpdu3ZVRkZGwM9YVVWVNm7cyM9YA/3tb3/ToUOHrPlZY9xwhrGjYRg7nAl13LDqI56ioiKNGzdOV111la655hrNnDlTx48f11133eV2aFZ56KGHNGTIEGVlZamsrExTpkyR1+vVnXfe6XZorjt27FhAlr5r1y59/PHHSk1NVWZmpiZNmqSpU6eqZ8+e6tq1qyZPnqxOnTpp2LBh7gXtom+7XqmpqSopKdGIESOUkZGhL7/8Ug8//LB69OihgoICF6MOxLjRcIwd58fY0XBNNm44rgMKs1/96lcmMzPTxMTEmGuuucZs2LDB7ZCsM2rUKNOxY0cTExNjvve975lRo0aZnTt3uh2WFVauXGkk1XmMGzfOGPN1ueDkyZNNenq6iY2NNQMGDDDbt293N2gXfdv1OnHihBk4cKBJS0sz0dHRJisry9xzzz2mvLzc7bDrYNxoGMaO82PsaLimGjc8xhjjIJECAAAIO2vWoAAAAJxFggIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKxDggIAAKzz/wBa52tLcu3OxwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "plt.subplot(1, 2, 1)\n",
        "plt.pcolormesh(attention_weights[:, 0, :])\n",
        "plt.title('Attention weights')\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.pcolormesh(example_tokens != 0)\n",
        "plt.title('Mask')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 470
        },
        "id": "gPzETxQdD7Tm",
        "outputId": "6ab1b7e2-2eb0-4d4a-9907-18b09486a756"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(64, 2, 15)\n"
          ]
        }
      ],
      "source": [
        "print(attention_weights.shape)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UmAlIT4MD7Tm",
        "outputId": "ab1d960d-b89b-427b-ab80-78bafb170a98"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "outputs": [],
      "source": [
        "attention_slice = attention_weights[0, 0].numpy()\n",
        "attention_slice = attention_slice[attention_slice != 0]"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "7qWcuQV5D7Tm"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f89c5a8b790>]"
            ]
          },
          "metadata": {},
          "execution_count": 34
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1200x600 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA94AAAINCAYAAADIsKceAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGj0lEQVR4nO3df1yV9f3/8SegcEQF8xeIqafSFNNAQQjbwjUWNmtRzsy1ZMzZ+nzCH5193cSZVtaopY42Wcwta1sznZ/KWTmaY2orKRIks0xdtXTqAV0FiQkG7+8f3TztzKNxTrw5B3ncb7frlud9va/39XpfaG+eXOdchBljjAAAAAAAgBXhwS4AAAAAAIBzGcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACzqEuwC2kJLS4sOHTqknj17KiwsLNjlAAAgY4w++ugjJSQkKDycn3O3BdZ7AEAo8WetPyeC96FDhzRo0KBglwEAwGkOHDig888/P9hlnBNY7wEAoag1a/05Ebx79uwp6dMJx8TEBLkaAACk+vp6DRo0yLNG4YtjvQcAhBJ/1vpzInifertZTEwMCzEAIKTwlui2w3oPAAhFrVnr+dAZAAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsCig4F1cXCyn0ymHw6H09HRVVFScse8bb7yhyZMny+l0KiwsTEVFRV94TAAAAAAAOoou/h6wdu1auVwulZSUKD09XUVFRcrOztaePXvUv3//0/ofP35cF154oaZMmaI77rijTcbs7Jzznwvauf95/6SgnRsAAAAAOqIwY4zx54D09HSNGzdOK1askCS1tLRo0KBBmjVrlubPn3/WY51Op+bOnau5c+e22ZiSVF9fr9jYWNXV1SkmJsaf6XRIBG+0N/7OAf7rbGtTe+CaAgBCiT/rkl93vJuamlRZWamCggJPW3h4uLKyslReXh5QsTbGbAsEjcCE8nULZm1Sx/66AgAAAAicX8H76NGjam5uVlxcnFd7XFyc3nrrrYAKCGTMxsZGNTY2el7X19cHdG4glITyDy0AAAAABK5DPtW8sLBQsbGxnm3QoEHBLgkAAAAAAJ/8uuPdt29fRUREqKamxqu9pqZG8fHxARUQyJgFBQVyuVye1/X19YRvACGHjzcAAABA8jN4R0ZGKiUlRWVlZcrJyZH06YPQysrKlJ+fH1ABgYwZFRWlqKiogM4H4NzCW/QDE+o/FODrCgAAziV+/zoxl8ul3NxcpaamKi0tTUVFRWpoaFBeXp4kafr06Ro4cKAKCwslffrwtDfffNPz54MHD6q6ulo9evTQ0KFDWzUmAAAAAAAdld/Be+rUqTpy5IgWLVokt9ut5ORklZaWeh6Otn//foWHf/bR8UOHDmnMmDGe10uXLtXSpUuVmZmpLVu2tGpMAAAAAAA6Kr+DtyTl5+ef8W3gp8L0KU6nU635VeFnGxMAAAAAgI6qQz7VHAAAAACAjoLgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALOoS7ALaUkNDgyIiItpkrJamE20yTiAaGhrOup/afAvl2qTQro/aAhPKtUlnry+Ua5NC++va3uMAAICOL8wYY4JdxBdVX1+v2NjYYJcBAMBp6urqFBMTE+wyzgmn1nuuKQAgFPizLvFWcwAAAAAALDqn3mp+6NChNvsJeOKdpW0yTiB2L5l41v3U5lso1yaFdn3UFphQrk06e32hXJsU2l/X1qqvr1dCQkKbjAUAADq2cyp4d+/eXd27d2+TscIjHW0yTiA+bw7U5lso1yaFdn3UFphQrk06e32hXJsU2l/X1mpubm6TcQAAQMfHW80BAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAnUxxcbGcTqccDofS09NVUVFxxr5vvPGGJk+eLKfTqbCwMBUVFX2hMY0xuvrqqxUWFqb169e3wWwAAAh9BG8AADqRtWvXyuVyafHixaqqqlJSUpKys7NVW1vrs//x48d14YUX6v7771d8fPwXHrOoqEhhYWFtOicAAEIdwRsAgE5k+fLlmjlzpvLy8jRy5EiVlJQoOjpaq1at8tl/3LhxevDBB3XTTTcpKirqC41ZXV2tZcuWnfFcAACcqwjeAAB0Ek1NTaqsrFRWVpanLTw8XFlZWSovL7c65vHjx/Wtb31LxcXFZ7xz/t8aGxtVX1/vtQEA0BERvAEA6CSOHj2q5uZmxcXFebXHxcXJ7XZbHfOOO+7Q+PHjdd1117V67MLCQsXGxnq2QYMGBVQjAADBRvAGAABWbdiwQX/729/O+GC2MykoKFBdXZ1nO3DggJ0CAQCwjOANAEAn0bdvX0VERKimpsarvaamptVv/w5kzL/97W96++231atXL3Xp0kVdunSRJE2ePFkTJkw449hRUVGKiYnx2gAA6IgI3gAAdBKRkZFKSUlRWVmZp62lpUVlZWXKyMiwNub8+fO1c+dOVVdXezZJ+tnPfqZHH3008AkBANBBdAl2AQAAoP24XC7l5uYqNTVVaWlpKioqUkNDg/Ly8iRJ06dP18CBA1VYWCjp04envfnmm54/Hzx4UNXV1erRo4eGDh3aqjHj4+N93lEfPHiwLrjggvaYNgAAQUXwBgCgE5k6daqOHDmiRYsWye12Kzk5WaWlpZ6Ho+3fv1/h4Z+9Ie7QoUMaM2aM5/XSpUu1dOlSZWZmasuWLa0aEwCAzo7gDQBAJ5Ofn6/8/Hyf+06F6VOcTqeMMV9oTF9aMyYAAOcKPuMNAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgUUDBu7i4WE6nUw6HQ+np6aqoqDhr/3Xr1mnEiBFyOBwaPXq0Nm7c6LX/2LFjys/P1/nnn69u3bpp5MiRKikpCaQ0AAAAAABCit/Be+3atXK5XFq8eLGqqqqUlJSk7Oxs1dbW+uy/bds2TZs2TTNmzNCOHTuUk5OjnJwc7dq1y9PH5XKptLRUjz/+uHbv3q25c+cqPz9fGzZsCHxmAAAAAACEAL+D9/LlyzVz5kzl5eV57kxHR0dr1apVPvs/9NBDmjhxoubNm6fExEQtWbJEY8eO1YoVKzx9tm3bptzcXE2YMEFOp1O33nqrkpKSPvdOOgAAAAAAoc6v4N3U1KTKykplZWV9NkB4uLKyslReXu7zmPLycq/+kpSdne3Vf/z48dqwYYMOHjwoY4w2b96svXv36qqrrvI5ZmNjo+rr6702AAAAAABCkV/B++jRo2publZcXJxXe1xcnNxut89j3G735/b/xS9+oZEjR+r8889XZGSkJk6cqOLiYl1xxRU+xywsLFRsbKxnGzRokD/TAAAAAACg3YTEU81/8Ytf6OWXX9aGDRtUWVmpZcuW6fbbb9df//pXn/0LCgpUV1fn2Q4cONDOFQMAAAAA0Dpd/Onct29fRUREqKamxqu9pqZG8fHxPo+Jj48/a/+PP/5YCxYs0NNPP61JkyZJki699FJVV1dr6dKlp71NXZKioqIUFRXlT+kAAAAAAASFX3e8IyMjlZKSorKyMk9bS0uLysrKlJGR4fOYjIwMr/6StGnTJk//kydP6uTJkwoP9y4lIiJCLS0t/pQHAAAAAEDI8euOt/Tpr/7Kzc1Vamqq0tLSVFRUpIaGBuXl5UmSpk+froEDB6qwsFCSNGfOHGVmZmrZsmWaNGmS1qxZo+3bt2vlypWSpJiYGGVmZmrevHnq1q2bhgwZoq1bt+p3v/udli9f3oZTBQAAAACg/fkdvKdOnaojR45o0aJFcrvdSk5OVmlpqecBavv37/e6ez1+/HitXr1aCxcu1IIFCzRs2DCtX79eo0aN8vRZs2aNCgoKdPPNN+v999/XkCFDdN999+m2225rgykCAAAAABA8fgdvScrPz1d+fr7PfVu2bDmtbcqUKZoyZcoZx4uPj9ejjz4aSCkAAAAAAIS0kHiqOQAAAAAA5yqCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAIBOpri4WE6nUw6HQ+np6aqoqDhj3zfeeEOTJ0+W0+lUWFiYioqK/B7z/fff16xZszR8+HB169ZNgwcP1uzZs1VXV9fWUwMAICQRvAEA6ETWrl0rl8ulxYsXq6qqSklJScrOzlZtba3P/sePH9eFF16o+++/X/Hx8QGNeejQIR06dEhLly7Vrl279Nhjj6m0tFQzZsywNk8AAEIJwRsAgE5k+fLlmjlzpvLy8jRy5EiVlJQoOjpaq1at8tl/3LhxevDBB3XTTTcpKioqoDFHjRqlJ598Utdee60uuugiXXnllbrvvvv0zDPP6JNPPrE2VwAAQgXBGwCATqKpqUmVlZXKysrytIWHhysrK0vl5eXtOmZdXZ1iYmLUpUuXM/ZpbGxUfX291wYAQEdE8AYAoJM4evSompubFRcX59UeFxcnt9vdbmMePXpUS5Ys0a233nrWsQsLCxUbG+vZBg0aFFCNAAAEG8EbAAC0m/r6ek2aNEkjR47UXXfddda+BQUFqqur82wHDhxonyIBAGhjZ35/FwAAOKf07dtXERERqqmp8Wqvqak544PT2nLMjz76SBMnTlTPnj319NNPq2vXrmcdOyoq6oyfKwcAoCPhjjcAAJ1EZGSkUlJSVFZW5mlraWlRWVmZMjIyrI5ZX1+vq666SpGRkdqwYYMcDkfgEwEAoIPhjjcAAJ2Iy+VSbm6uUlNTlZaWpqKiIjU0NCgvL0+SNH36dA0cOFCFhYWSPn142ptvvun588GDB1VdXa0ePXpo6NChrRrzVOg+fvy4Hn/8ca8HpfXr108RERHtfRkAAGhXBG8AADqRqVOn6siRI1q0aJHcbreSk5NVWlrqeTja/v37FR7+2RviDh06pDFjxnheL126VEuXLlVmZqa2bNnSqjGrqqr0yiuvSJInrJ/y7rvvyul0WpwxAADBR/AGAKCTyc/PV35+vs99p8L0KU6nU8aYLzTmhAkTWjUGAADnKj7jDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYFFAwbu4uFhOp1MOh0Pp6emqqKg4a/9169ZpxIgRcjgcGj16tDZu3Hhan927d+sb3/iGYmNj1b17d40bN0779+8PpDwAAAAAAEKG38F77dq1crlcWrx4saqqqpSUlKTs7GzV1tb67L9t2zZNmzZNM2bM0I4dO5STk6OcnBzt2rXL0+ftt9/Wl770JY0YMUJbtmzRzp07deedd8rhcAQ+MwAAAAAAQoDfwXv58uWaOXOm8vLyNHLkSJWUlCg6OlqrVq3y2f+hhx7SxIkTNW/ePCUmJmrJkiUaO3asVqxY4enz4x//WF//+tf105/+VGPGjNFFF12kb3zjG+rfv3/gMwMAAAAAIAT4FbybmppUWVmprKyszwYID1dWVpbKy8t9HlNeXu7VX5Kys7M9/VtaWvTcc8/p4osvVnZ2tvr376/09HStX7/+jHU0Njaqvr7eawMAAAAAIBT5FbyPHj2q5uZmxcXFebXHxcXJ7Xb7PMbtdp+1f21trY4dO6b7779fEydO1F/+8hddf/31uuGGG7R161afYxYWFio2NtazDRo0yJ9pAAAAAADQboL+VPOWlhZJ0nXXXac77rhDycnJmj9/vq655hqVlJT4PKagoEB1dXWe7cCBA+1ZMgAAAAAArdbFn859+/ZVRESEampqvNpramoUHx/v85j4+Piz9u/bt6+6dOmikSNHevVJTEzUiy++6HPMqKgoRUVF+VM6AAAAAABB4dcd78jISKWkpKisrMzT1tLSorKyMmVkZPg8JiMjw6u/JG3atMnTPzIyUuPGjdOePXu8+uzdu1dDhgzxpzwAAAAAAEKOX3e8Jcnlcik3N1epqalKS0tTUVGRGhoalJeXJ0maPn26Bg4cqMLCQknSnDlzlJmZqWXLlmnSpElas2aNtm/frpUrV3rGnDdvnqZOnaorrrhCX/nKV1RaWqpnnnlGW7ZsaZtZAgAAAAAQJH4H76lTp+rIkSNatGiR3G63kpOTVVpa6nmA2v79+xUe/tmN9PHjx2v16tVauHChFixYoGHDhmn9+vUaNWqUp8/111+vkpISFRYWavbs2Ro+fLiefPJJfelLX2qDKQIAAAAAEDx+B29Jys/PV35+vs99vu5ST5kyRVOmTDnrmN/97nf13e9+N5ByAAAAAAAIWUF/qjkAAAAAAOcygjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALOoS7AIAAADgzTn/uaCe/5/3Twrq+QHgXMMdbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARV2CXQAAAACA4HLOfy5o5/7n/ZOCdm6gvXDHGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIinmgMAAOCcwdO5AYQi7ngDAAAAAGARwRsAAAAAAIt4qzkAAOiUeEtyYIJ53aSOfe0AdF7c8QYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACziqeYAAABAO+BJ+kDnxR1vAAA6meLiYjmdTjkcDqWnp6uiouKMfd944w1NnjxZTqdTYWFhKioqCmjMEydO6Pbbb1efPn3Uo0cPTZ48WTU1NW05LQAAQhbBGwCATmTt2rVyuVxavHixqqqqlJSUpOzsbNXW1vrsf/z4cV144YW6//77FR8fH/CYd9xxh5555hmtW7dOW7du1aFDh3TDDTdYmSMAAKGG4A0AQCeyfPlyzZw5U3l5eRo5cqRKSkoUHR2tVatW+ew/btw4Pfjgg7rpppsUFRUV0Jh1dXV65JFHtHz5cl155ZVKSUnRo48+qm3btunll1+2NlcAAEIFwRsAgE6iqalJlZWVysrK8rSFh4crKytL5eXl1sasrKzUyZMnvfqMGDFCgwcPDvi8AAB0JDxcDQCATuLo0aNqbm5WXFycV3tcXJzeeusta2O63W5FRkaqV69ep/Vxu91nHLuxsVGNjY2e1/X19QHVCABAsHHHGwAAhKTCwkLFxsZ6tkGDBgW7JAAAAkLwBgCgk+jbt68iIiJOe5p4TU3NGR+c1hZjxsfHq6mpSR9++KFf5y0oKFBdXZ1nO3DgQEA1AgAQbARvAAA6icjISKWkpKisrMzT1tLSorKyMmVkZFgbMyUlRV27dvXqs2fPHu3fv/+s542KilJMTIzXBgBAR8RnvAEA6ERcLpdyc3OVmpqqtLQ0FRUVqaGhQXl5eZKk6dOna+DAgSosLJT06cPT3nzzTc+fDx48qOrqavXo0UNDhw5t1ZixsbGaMWOGXC6XevfurZiYGM2aNUsZGRm67LLLgnAVAABoXwRvAAA6kalTp+rIkSNatGiR3G63kpOTVVpa6nk42v79+xUe/tkb4g4dOqQxY8Z4Xi9dulRLly5VZmamtmzZ0qoxJelnP/uZwsPDNXnyZDU2Nio7O1u//OUv22fSAAAEGcEbAIBOJj8/X/n5+T73nQrTpzidThljvtCYkuRwOFRcXKzi4mK/agUA4FzAZ7wBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAi7oEuwAAAAAA6Iic858L6vn/ef+koJ4frccdbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwKKAgndxcbGcTqccDofS09NVUVFx1v7r1q3TiBEj5HA4NHr0aG3cuPGMfW+77TaFhYWpqKgokNIAAAAAAAgpfv86sbVr18rlcqmkpETp6ekqKipSdna29uzZo/79+5/Wf9u2bZo2bZoKCwt1zTXXaPXq1crJyVFVVZVGjRrl1ffpp5/Wyy+/rISEhMBnBAAAAOCcwa/swrnA7+C9fPlyzZw5U3l5eZKkkpISPffcc1q1apXmz59/Wv+HHnpIEydO1Lx58yRJS5Ys0aZNm7RixQqVlJR4+h08eFCzZs3S888/r0mT+MsNAAAAAOeqYP5AJRg/TPEreDc1NamyslIFBQWetvDwcGVlZam8vNznMeXl5XK5XF5t2dnZWr9+ved1S0uLbrnlFs2bN0+XXHLJ59bR2NioxsZGz+v6+np/pgEAAAAA57zOFm5DmV+f8T569Kiam5sVFxfn1R4XFye32+3zGLfb/bn9H3jgAXXp0kWzZ89uVR2FhYWKjY31bIMGDfJnGgAAAAAAtJugP9W8srJSDz30kB577DGFhYW16piCggLV1dV5tgMHDliuEgAAAACAwPgVvPv27auIiAjV1NR4tdfU1Cg+Pt7nMfHx8Wft//e//121tbUaPHiwunTpoi5duui9997TD37wAzmdTp9jRkVFKSYmxmsDAAAAACAU+RW8IyMjlZKSorKyMk9bS0uLysrKlJGR4fOYjIwMr/6StGnTJk//W265RTt37lR1dbVnS0hI0Lx58/T888/7Ox8AAAAAAEKK3081d7lcys3NVWpqqtLS0lRUVKSGhgbPU86nT5+ugQMHqrCwUJI0Z84cZWZmatmyZZo0aZLWrFmj7du3a+XKlZKkPn36qE+fPl7n6Nq1q+Lj4zV8+PAvOj8AAAAAAILK7+A9depUHTlyRIsWLZLb7VZycrJKS0s9D1Dbv3+/wsM/u5E+fvx4rV69WgsXLtSCBQs0bNgwrV+//rTf4Q0AAAAAwLnI7+AtSfn5+crPz/e5b8uWLae1TZkyRVOmTGn1+P/85z8DKQsAAAAAgJAT9KeaAwAAAABwLiN4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AQCdTXFwsp9Mph8Oh9PR0VVRUnLX/unXrNGLECDkcDo0ePVobN2702l9TU6PvfOc7SkhIUHR0tCZOnKh9+/Z59XG73brlllsUHx+v7t27a+zYsXryySfbfG4AAIQigjcAAJ3I2rVr5XK5tHjxYlVVVSkpKUnZ2dmqra312X/btm2aNm2aZsyYoR07dignJ0c5OTnatWuXJMkYo5ycHL3zzjv605/+pB07dmjIkCHKyspSQ0ODZ5zp06drz5492rBhg15//XXdcMMNuvHGG7Vjx452mTcAAMFE8AYAoBNZvny5Zs6cqby8PI0cOVIlJSWKjo7WqlWrfPZ/6KGHNHHiRM2bN0+JiYlasmSJxo4dqxUrVkiS9u3bp5dfflkPP/ywxo0bp+HDh+vhhx/Wxx9/rCeeeMIzzrZt2zRr1iylpaXpwgsv1MKFC9WrVy9VVla2y7wBAAgmgjcAAJ1EU1OTKisrlZWV5WkLDw9XVlaWysvLfR5TXl7u1V+SsrOzPf0bGxslSQ6Hw2vMqKgovfjii5628ePHa+3atXr//ffV0tKiNWvW6MSJE5owYcIZ621sbFR9fb3XBgBAR0TwBgCgkzh69Kiam5sVFxfn1R4XFye32+3zGLfbfdb+I0aM0ODBg1VQUKAPPvhATU1NeuCBB/Svf/1Lhw8f9hzzxz/+USdPnlSfPn0UFRWl73//+3r66ac1dOjQM9ZbWFio2NhYzzZo0KBApw4AQFARvAEAQMC6du2qp556Snv37lXv3r0VHR2tzZs36+qrr1Z4+GffZtx555368MMP9de//lXbt2+Xy+XSjTfeqNdff/2MYxcUFKiurs6zHThwoD2mBABAm+sS7AIAAED76Nu3ryIiIlRTU+PVXlNTo/j4eJ/HxMfHf27/lJQUVVdXq66uTk1NTerXr5/S09OVmpoqSXr77be1YsUK7dq1S5dccokkKSkpSX//+99VXFyskpISn+eOiopSVFRUwPMFACBUcMcbAIBOIjIyUikpKSorK/O0tbS0qKysTBkZGT6PycjI8OovSZs2bfLZPzY2Vv369dO+ffu0fft2XXfddZKk48ePS5LXHXBJioiIUEtLyxeaEwAAHQF3vAEA6ERcLpdyc3OVmpqqtLQ0FRUVqaGhQXl5eZI+/bVfAwcOVGFhoSRpzpw5yszM1LJlyzRp0iStWbNG27dv18qVKz1jrlu3Tv369dPgwYP1+uuva86cOcrJydFVV10l6dPPgQ8dOlTf//73tXTpUvXp00fr16/Xpk2b9Oyzz7b/RQAAoJ0RvAEA6ESmTp2qI0eOaNGiRXK73UpOTlZpaannAWr79+/3ujM9fvx4rV69WgsXLtSCBQs0bNgwrV+/XqNGjfL0OXz4sFwul2pqajRgwABNnz5dd955p2d/165dtXHjRs2fP1/XXnutjh07pqFDh+q3v/2tvv71r7ff5AEACBKCNwAAnUx+fr7y8/N97tuyZctpbVOmTNGUKVPOON7s2bM1e/bss55z2LBhevLJJ/2qEwCAcwWf8QYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWBRS8i4uL5XQ65XA4lJ6eroqKirP2X7dunUaMGCGHw6HRo0dr48aNnn0nT57Uj370I40ePVrdu3dXQkKCpk+frkOHDgVSGgAAAAAAIcXv4L127Vq5XC4tXrxYVVVVSkpKUnZ2tmpra33237Ztm6ZNm6YZM2Zox44dysnJUU5Ojnbt2iVJOn78uKqqqnTnnXeqqqpKTz31lPbs2aNvfOMbX2xmAAAAAACEAL+D9/LlyzVz5kzl5eVp5MiRKikpUXR0tFatWuWz/0MPPaSJEydq3rx5SkxM1JIlSzR27FitWLFCkhQbG6tNmzbpxhtv1PDhw3XZZZdpxYoVqqys1P79+7/Y7AAAAAAACDK/gndTU5MqKyuVlZX12QDh4crKylJ5ebnPY8rLy736S1J2dvYZ+0tSXV2dwsLC1KtXL5/7GxsbVV9f77UBAAAAABCK/AreR48eVXNzs+Li4rza4+Li5Ha7fR7jdrv96n/ixAn96Ec/0rRp0xQTE+OzT2FhoWJjYz3boEGD/JkGAAAAAADtJqSean7y5EndeOONMsbo4YcfPmO/goIC1dXVebYDBw60Y5UAAAAAALReF3869+3bVxEREaqpqfFqr6mpUXx8vM9j4uPjW9X/VOh+77339Le//e2Md7slKSoqSlFRUf6UDgAAAABAUPh1xzsyMlIpKSkqKyvztLW0tKisrEwZGRk+j8nIyPDqL0mbNm3y6n8qdO/bt09//etf1adPH3/KAgAAAAAgZPl1x1uSXC6XcnNzlZqaqrS0NBUVFamhoUF5eXmSpOnTp2vgwIEqLCyUJM2ZM0eZmZlatmyZJk2apDVr1mj79u1auXKlpE9D9ze/+U1VVVXp2WefVXNzs+fz371791ZkZGRbzRUAAAAAgHbnd/CeOnWqjhw5okWLFsntdis5OVmlpaWeB6jt379f4eGf3UgfP368Vq9erYULF2rBggUaNmyY1q9fr1GjRkmSDh48qA0bNkiSkpOTvc61efNmTZgwIcCpAQAAAAAQfH4Hb0nKz89Xfn6+z31btmw5rW3KlCmaMmWKz/5Op1PGmEDKAAAAAAAg5IXUU80BAAAAADjXELwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgAAAADAIoI3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsIngDAAAAAGARwRsAAAAAAIsI3gAAAAAAWETwBgCgkykuLpbT6ZTD4VB6eroqKirO2n/dunUaMWKEHA6HRo8erY0bN3rtr6mp0Xe+8x0lJCQoOjpaEydO1L59+04bp7y8XFdeeaW6d++umJgYXXHFFfr444/bdG4AAIQigjcAAJ3I2rVr5XK5tHjxYlVVVSkpKUnZ2dmqra312X/btm2aNm2aZsyYoR07dignJ0c5OTnatWuXJMkYo5ycHL3zzjv605/+pB07dmjIkCHKyspSQ0ODZ5zy8nJNnDhRV111lSoqKvTqq68qPz9f4eF8KwIAOPex2gEA0IksX75cM2fOVF5enkaOHKmSkhJFR0dr1apVPvs/9NBDmjhxoubNm6fExEQtWbJEY8eO1YoVKyRJ+/bt08svv6yHH35Y48aN0/Dhw/Xwww/r448/1hNPPOEZ54477tDs2bM1f/58XXLJJRo+fLhuvPFGRUVFtcu8AQAIJoI3AACdRFNTkyorK5WVleVpCw8PV1ZWlsrLy30eU15e7tVfkrKzsz39GxsbJUkOh8NrzKioKL344ouSpNraWr3yyivq37+/xo8fr7i4OGVmZnr2n0ljY6Pq6+u9NgAAOiKCNwAAncTRo0fV3NysuLg4r/a4uDi53W6fx7jd7rP2HzFihAYPHqyCggJ98MEHampq0gMPPKB//etfOnz4sCTpnXfekSTdddddmjlzpkpLSzV27Fh99atf9flZ8FMKCwsVGxvr2QYNGhTw3AEACCaCNwAACFjXrl311FNPae/everdu7eio6O1efNmXX311Z7Pb7e0tEiSvv/97ysvL09jxozRz372Mw0fPvyMb3GXpIKCAtXV1Xm2AwcOtMucAABoa12CXQAAAGgfffv2VUREhGpqarzaa2pqFB8f7/OY+Pj4z+2fkpKi6upq1dXVqampSf369VN6erpSU1MlSQMGDJAkjRw50mucxMRE7d+//4z1RkVF8RlwAMA5gTveAAB0EpGRkUpJSVFZWZmnraWlRWVlZcrIyPB5TEZGhld/Sdq0aZPP/rGxserXr5/27dun7du367rrrpMkOZ1OJSQkaM+ePV799+7dqyFDhnzRaQEAEPK44w0AQCficrmUm5ur1NRUpaWlqaioSA0NDcrLy5MkTZ8+XQMHDlRhYaEkac6cOcrMzNSyZcs0adIkrVmzRtu3b9fKlSs9Y65bt079+vXT4MGD9frrr2vOnDnKycnRVVddJUkKCwvTvHnztHjxYiUlJSk5OVm//e1v9dZbb+n//u//2v8iAADQzgjeAAB0IlOnTtWRI0e0aNEiud1uJScnq7S01PMAtf3793v9bu3x48dr9erVWrhwoRYsWKBhw4Zp/fr1GjVqlKfP4cOH5XK5VFNTowEDBmj69Om68847vc47d+5cnThxQnfccYfef/99JSUladOmTbrooovaZ+IAAAQRwRsAgE4mPz9f+fn5Pvdt2bLltLYpU6ZoypQpZxxv9uzZmj179ueed/78+Zo/f36r6wQA4FzBZ7wBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACwieAMAAAAAYBHBGwAAAAAAiwjeAAAAAABYRPAGAAAAAMAigjcAAAAAABYRvAEAAAAAsIjgDQAAAACARQRvAAAAAAAsCih4FxcXy+l0yuFwKD09XRUVFWftv27dOo0YMUIOh0OjR4/Wxo0bvfYbY7Ro0SINGDBA3bp1U1ZWlvbt2xdIaQAAAAAAhBS/g/fatWvlcrm0ePFiVVVVKSkpSdnZ2aqtrfXZf9u2bZo2bZpmzJihHTt2KCcnRzk5Odq1a5enz09/+lP9/Oc/V0lJiV555RV1795d2dnZOnHiROAzAwAAAAAgBPgdvJcvX66ZM2cqLy9PI0eOVElJiaKjo7Vq1Sqf/R966CFNnDhR8+bNU2JiopYsWaKxY8dqxYoVkj69211UVKSFCxfquuuu06WXXqrf/e53OnTokNavX/+FJgcAAAAAQLB18adzU1OTKisrVVBQ4GkLDw9XVlaWysvLfR5TXl4ul8vl1Zadne0J1e+++67cbreysrI8+2NjY5Wenq7y8nLddNNNp43Z2NioxsZGz+u6ujpJUn19vT/TOauWxuNtNpa/Pm8e1OZbKNcmhXZ91BaYUK5NOnt9oVybFNpfV3/HMca0yXj47Fq21dcolP+ehfK/0VCuTeLreibUFjj+PQSmI9fm7zitWuuNHw4ePGgkmW3btnm1z5s3z6Slpfk8pmvXrmb16tVebcXFxaZ///7GGGNeeuklI8kcOnTIq8+UKVPMjTfe6HPMxYsXG0lsbGxsbGwhvx04cMCfpRZnceDAgaB/PdnY2NjY2P57a81a79cd71BRUFDgdRe9paVF77//vvr06aOwsLAgVvbpTz0GDRqkAwcOKCYmJqi1dDRcu8Bw3QLHtQsM1611jDH66KOPlJCQEOxSzhkJCQk6cOCAevbsyXrfQXHdAse1CwzXLTBct9bxZ633K3j37dtXERERqqmp8WqvqalRfHy8z2Pi4+PP2v/Uf2tqajRgwACvPsnJyT7HjIqKUlRUlFdbr169/JmKdTExMfwlDRDXLjBct8Bx7QLDdft8sbGxwS7hnBIeHq7zzz8/2GV44d9BYLhugePaBYbrFhiu2+dr7Vrv18PVIiMjlZKSorKyMk9bS0uLysrKlJGR4fOYjIwMr/6StGnTJk//Cy64QPHx8V596uvr9corr5xxTAAAAAAAOgq/32rucrmUm5ur1NRUpaWlqaioSA0NDcrLy5MkTZ8+XQMHDlRhYaEkac6cOcrMzNSyZcs0adIkrVmzRtu3b9fKlSslSWFhYZo7d67uvfdeDRs2TBdccIHuvPNOJSQkKCcnp+1mCgAAAABAEPgdvKdOnaojR45o0aJFcrvdSk5OVmlpqeLi4iRJ+/fvV3j4ZzfSx48fr9WrV2vhwoVasGCBhg0bpvXr12vUqFGePj/84Q/V0NCgW2+9VR9++KG+9KUvqbS0VA6How2m2L6ioqK0ePHi094Kj8/HtQsM1y1wXLvAcN0A/h0EiusWOK5dYLhugeG6tb0wY/g9JwAAAAAA2OLXZ7wBAAAAAIB/CN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG821hxcbGcTqccDofS09NVUVER7JJCWmFhocaNG6eePXuqf//+ysnJ0Z49e4JdVod0//33e349H87u4MGD+va3v60+ffqoW7duGj16tLZv3x7sskJec3Oz7rzzTl1wwQXq1q2bLrroIi1ZskQ8oxOdDWu9/1jv2wZrfeux1geGtd4egncbWrt2rVwulxYvXqyqqiolJSUpOztbtbW1wS4tZG3dulW33367Xn75ZW3atEknT57UVVddpYaGhmCX1qG8+uqr+tWvfqVLL7002KWEvA8++ECXX365unbtqj//+c968803tWzZMp133nnBLi3kPfDAA3r44Ye1YsUK7d69Ww888IB++tOf6he/+EWwSwPaDWt9YFjvvzjW+tZjrQ8ca709/DqxNpSenq5x48ZpxYoVkqSWlhYNGjRIs2bN0vz584NcXcdw5MgR9e/fX1u3btUVV1wR7HI6hGPHjmns2LH65S9/qXvvvVfJyckqKioKdlkha/78+XrppZf097//PdildDjXXHON4uLi9Mgjj3jaJk+erG7duunxxx8PYmVA+2Gtbxus9/5hrfcPa33gWOvt4Y53G2lqalJlZaWysrI8beHh4crKylJ5eXkQK+tY6urqJEm9e/cOciUdx+23365JkyZ5/d3DmW3YsEGpqamaMmWK+vfvrzFjxujXv/51sMvqEMaPH6+ysjLt3btXkvTaa6/pxRdf1NVXXx3kyoD2wVrfdljv/cNa7x/W+sCx1tvTJdgFnCuOHj2q5uZmxcXFebXHxcXprbfeClJVHUtLS4vmzp2ryy+/XKNGjQp2OR3CmjVrVFVVpVdffTXYpXQY77zzjh5++GG5XC4tWLBAr776qmbPnq3IyEjl5uYGu7yQNn/+fNXX12vEiBGKiIhQc3Oz7rvvPt18883BLg1oF6z1bYP13j+s9f5jrQ8ca709BG+EjNtvv127du3Siy++GOxSOoQDBw5ozpw52rRpkxwOR7DL6TBaWlqUmpqqn/zkJ5KkMWPGaNeuXSopKWEx/hx//OMf9Yc//EGrV6/WJZdcourqas2dO1cJCQlcOwCtxnrfeqz1gWGtDxxrvT0E7zbSt29fRUREqKamxqu9pqZG8fHxQaqq48jPz9ezzz6rF154Qeeff36wy+kQKisrVVtbq7Fjx3rampub9cILL2jFihVqbGxUREREECsMTQMGDNDIkSO92hITE/Xkk08GqaKOY968eZo/f75uuukmSdLo0aP13nvvqbCwkMUYnQJr/RfHeu8f1vrAsNYHjrXeHj7j3UYiIyOVkpKisrIyT1tLS4vKysqUkZERxMpCmzFG+fn5evrpp/W3v/1NF1xwQbBL6jC++tWv6vXXX1d1dbVnS01N1c0336zq6moW4jO4/PLLT/sVNnv37tWQIUOCVFHHcfz4cYWHey8bERERamlpCVJFQPtirQ8c631gWOsDw1ofONZ6e7jj3YZcLpdyc3OVmpqqtLQ0FRUVqaGhQXl5ecEuLWTdfvvtWr16tf70pz+pZ8+ecrvdkqTY2Fh169YtyNWFtp49e5722bju3burT58+fGbuLO644w6NHz9eP/nJT3TjjTeqoqJCK1eu1MqVK4NdWsi79tprdd9992nw4MG65JJLtGPHDi1fvlzf/e53g10a0G5Y6wPDeh8Y1vrAsNYHjrXeIoM29Ytf/MIMHjzYREZGmrS0NPPyyy8Hu6SQJsnn9uijjwa7tA4pMzPTzJkzJ9hlhLxnnnnGjBo1ykRFRZkRI0aYlStXBrukDqG+vt7MmTPHDB482DgcDnPhhReaH//4x6axsTHYpQHtirXef6z3bYe1vnVY6wPDWm8Pv8cbAAAAAACL+Iw3AAAAAAAWEbwBAAAAALCI4A0AAAAAgEUEbwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AbOMRMmTNDcuXODXYZfHnvsMfXq1cuvY77zne8oJyfHSj0AAJwNay1a65///KfCwsJUXV0d7FIQZARvwA/l5eWKiIjQpEmTTtt31113KTk5+bT2sLAwrV+/vs1r2bJli8LCwvThhx96tT/11FNasmRJm5/PpqlTp2rv3r1tPq7T6VRRUVGbjwsAsIe11g7WWiC4CN6AHx555BHNmjVLL7zwgg4dOhTscnzq3bu3evbsGewy/NKtWzf1798/2GUAAEIAa60drLVAcBG8gVY6duyY1q5dq//5n//RpEmT9Nhjj3n2PfbYY7r77rv12muvKSwsTGFhYXrsscfkdDolSddff73CwsI8ryXpT3/6k8aOHSuHw6ELL7xQd999tz755BPP/rCwMP3mN7/R9ddfr+joaA0bNkwbNmyQ9Onblr7yla9Iks477zyFhYXpO9/5jqTT3/72wQcfaPr06TrvvPMUHR2tq6++Wvv27fOqvVevXnr++eeVmJioHj16aOLEiTp8+PAZr0VqaqqWLl3qeZ2Tk6OuXbvq2LFjkqR//etfCgsL0z/+8Q9JUmNjo/7f//t/GjhwoLp376709HRt2bLltBr+07333qv+/furZ8+e+t73vqf58+f7vMuxdOlSDRgwQH369NHtt9+ukydPeq7De++9pzvuuMPzNZGk9957T9dee63OO+88de/eXZdccok2btx4xrkCANoPa+1nOvNa63Q6PeP953bK66+/riuvvFLdunVTnz59dOutt3quiyS1tLTonnvu0fnnn6+oqCglJyertLTUs//U27//+Mc/6stf/rK6deumcePGae/evXr11VeVmpqqHj166Oqrr9aRI0e8avvNb36jxMREORwOjRgxQr/85S+99ldUVGjMmDFyOBxKTU3Vjh07Wj1vnOMMgFZ55JFHTGpqqjHGmGeeecZcdNFFpqWlxRhjzPHjx80PfvADc8kll5jDhw+bw4cPm+PHj5va2lojyTz66KPm8OHDpra21hhjzAsvvGBiYmLMY489Zt5++23zl7/8xTidTnPXXXd5zifJnH/++Wb16tVm3759Zvbs2aZHjx7m3//+t/nkk0/Mk08+aSSZPXv2mMOHD5sPP/zQGGNMZmammTNnjmecb3zjGyYxMdG88MILprq62mRnZ5uhQ4eapqYmY4wxjz76qOnatavJysoyr776qqmsrDSJiYnmW9/61hmvhcvlMpMmTTLGGNPS0mJ69+5t+vbta/785z8bY4x5/PHHzcCBAz39v/e975nx48ebF154wfzjH/8wDz74oImKijJ79+711BAbG+vp//jjjxuHw2FWrVpl9uzZY+6++24TExNjkpKSPH1yc3NNTEyMue2228zu3bvNM888Y6Kjo83KlSuNMcb8+9//Nueff7655557PF8TY4yZNGmS+drXvmZ27txp3n77bfPMM8+YrVu3tvJvAQDAJtbaz3Tmtba2ttYz3r/+9S9z2WWXmS9/+cvGGGOOHTtmBgwYYG644Qbz+uuvm7KyMnPBBReY3Nxcz/HLly83MTEx5oknnjBvvfWW+eEPf2i6du3quRbvvvuukWRGjBhhSktLzZtvvmkuu+wyk5KSYiZMmGBefPFFU1VVZYYOHWpuu+02r2s2YMAA8+STT5p33nnHPPnkk6Z3797mscceM8YY89FHH5l+/fqZb33rW2bXrl3mmWeeMRdeeKGRZHbs2NHq+ePcRPAGWmn8+PGmqKjIGGPMyZMnTd++fc3mzZs9+xcvXuy1WJ0iyTz99NNebV/96lfNT37yE6+23//+92bAgAFexy1cuNDz+tixY0aSZ8HdvHmzkWQ++OADr3H+85uBvXv3GknmpZde8uw/evSo6datm/njH/9ojPl0IZZk/vGPf3j6FBcXm7i4uDNeiw0bNpjY2FjzySefmOrqahMfH2/mzJljfvSjHxljPl38T30z8d5775mIiAhz8ODB065BQUGBp4b//GYgPT3d3H777V79L7/88tO+GRgyZIj55JNPPG1TpkwxU6dO9bweMmSI+dnPfuY1zujRo72+6QIAhA7W2s+w1n5q9uzZZsiQIZ4fqKxcudKcd9555tixY54+zz33nAkPDzdut9sYY0xCQoK57777vMYZN26c+d///V9jzGfB+ze/+Y1n/xNPPGEkmbKyMk9bYWGhGT58uOf1RRddZFavXu017pIlS0xGRoYxxphf/epXpk+fPubjjz/27H/44YcJ3jDGGMNbzYFW2LNnjyoqKjRt2jRJUpcuXTR16lQ98sgjAY332muv6Z577lGPHj0828yZM3X48GEdP37c0+/SSy/1/Ll79+6KiYlRbW1tq8+ze/dudenSRenp6Z62Pn36aPjw4dq9e7enLTo6WhdddJHn9YABA856ni9/+cv66KOPtGPHDm3dulWZmZmaMGGC5y1tW7du1YQJEyR9+naw5uZmXXzxxV7z3bp1q95++22f4+/Zs0dpaWlebf/9WpIuueQSRUREtLpuSZo9e7buvfdeXX755Vq8eLF27tx51v4AgPbBWuuNtVZauXKlHnnkEW3YsEH9+vWT9On1TkpKUvfu3T39Lr/8crW0tGjPnj2qr6/XoUOHdPnll3uNdfnll3t9PSTvr31cXJwkafTo0V5tp+ba0NCgt99+WzNmzPC6xvfee6/nGu/evVuXXnqpHA6HZ4yMjIyA5o5zT5dgFwB0BI888og++eQTJSQkeNqMMYqKitKKFSsUGxvr13jHjh3T3XffrRtuuOG0ff/5P+uuXbt67QsLC1NLS4uf1X8+X+cxxpyxf69evZSUlKQtW7aovLxcX/va13TFFVd4npi6b98+ZWZmSvp0rhEREaqsrPRauCWpR48ebV73512f733ve8rOztZzzz2nv/zlLyosLNSyZcs0a9asL1QLAOCLYa311tnX2s2bN2vWrFl64oknvAJyW/rPuZ36DPl/t52a66nPkP/617/2+iGLpNOuOeALd7yBz/HJJ5/od7/7nZYtW6bq6mrP9tprrykhIUFPPPGEJCkyMlLNzc2nHd+1a9fT2seOHas9e/Zo6NChp23h4a37ZxkZGSlJPs95SmJioj755BO98sornrZ///vf2rNnj0aOHNmq85xJZmamNm/erBdeeEETJkxQ7969lZiYqPvuu08DBgzQxRdfLEkaM2aMmpubVVtbe9pc4+PjfY49fPhwvfrqq15t//26Nc70NRk0aJBuu+02PfXUU/rBD36gX//6136PDQBoO6y1vnXWtfYf//iHvvnNb2rBggWn/eAkMTFRr732mhoaGjxtL730ksLDwzV8+HDFxMQoISFBL730ktdxL7300hf6esTFxSkhIUHvvPPOadf4ggsu8NS2c+dOnThxwnPcyy+/HPA5cW4heAOf49lnn9UHH3ygGTNmaNSoUV7b5MmTPW+Bczqdevfdd1VdXa2jR4+qsbHR015WVia3260PPvhAkrRo0SL97ne/091336033nhDu3fv1po1a7Rw4cJW1zVkyBCFhYXp2Wef1ZEjR7ye5nnKsGHDdN1112nmzJl68cUX9dprr+nb3/62Bg4cqOuuu+4LXZcJEybo+eefV5cuXTRixAhP2x/+8AfPT+Al6eKLL9bNN9+s6dOn66mnntK7776riooKFRYW6rnnnvM59qxZs/TII4/ot7/9rfbt26d7771XO3fu9HqiaWs4nU698MILOnjwoI4ePSpJmjt3rp5//nm9++67qqqq0ubNm5WYmBjgVQAAtAXWWt8641r78ccf69prr9WYMWN06623yu12ezZJuvnmm+VwOJSbm6tdu3Z57ozfcsstnreLz5s3Tw888IDWrl2rPXv2aP78+aqurtacOXP8mtt/u/vuu1VYWKif//zn2rt3r15//XU9+uijWr58uSTpW9/6lsLCwjRz5ky9+eab2rhxo9eT6dHJBfcj5kDou+aaa8zXv/51n/teeeUVI8m89tpr5sSJE2by5MmmV69enqerGvPpw1GGDh1qunTpYoYMGeI5trS01IwfP95069bNxMTEmLS0NM9TQo3x/aCY2NhYz7jGGHPPPfeY+Ph4ExYW5nma538/afX99983t9xyi4mNjTXdunUz2dnZnqd6GnP6w1aMMebpp582n/e/h3//+98mLCzM6wErp44rKSnx6tvU1GQWLVpknE6n6dq1qxkwYIC5/vrrzc6dO89Ywz333GP69u1revToYb773e+a2bNnm8suu8yzPzc311x33XVex8yZM8dkZmZ6XpeXl5tLL73UREVFeeaTn59vLrroIhMVFWX69etnbrnlFnP06NGzzhUAYBdrrW/n6lo7ZMgQs3jxYp9zPvXgM1/bKTt37jRf+cpXjMPhML179zYzZ840H330kWd/c3Ozueuuu8zAgQNN165dTVJSkueBef95jv984JmvB+n5umZ/+MMfTHJysomMjDTnnXeeueKKK8xTTz3ldT2SkpJMZGSkSU5O9jwZn4erIcyYs3y4BABCxNe+9jXFx8fr97//fbBLAQDgnNQea+3x48fVp08f/fnPf/Y8HA7oDHi4GoCQc/z4cZWUlCg7O1sRERF64okn9Ne//lWbNm0KdmkAAJwTgrXWbt68WVdeeSWhG50Od7wBhJxTn+/asWOHTpw4oeHDh2vhwoU+n0wLAAD8x1oLtC+CNwAAAAAAFvFUcwAAAAAALCJ4AwAAAABgEcEbAAAAAACLCN4AAAAAAFhE8AYAAAAAwCKCNwAAAAAAFhG8AQAAAACwiOANAAAAAIBFBG8AAAAAACz6/1QDCNMBT5gtAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "plt.suptitle('Attention weights for one sequence')\n",
        "\n",
        "plt.figure(figsize=(12, 6))\n",
        "a1 = plt.subplot(1, 2, 1)\n",
        "plt.bar(range(len(attention_slice)), attention_slice)\n",
        "# freeze the xlim\n",
        "plt.xlim(plt.xlim())\n",
        "plt.xlabel('Attention weights')\n",
        "\n",
        "a2 = plt.subplot(1, 2, 2)\n",
        "plt.bar(range(len(attention_slice)), attention_slice)\n",
        "plt.xlabel('Attention weights, zoomed')\n",
        "\n",
        "# zoom in\n",
        "top = max(a1.get_ylim())\n",
        "zoom = 0.85 * top\n",
        "a2.set_ylim([0.90 * top, top])\n",
        "a1.plot(a1.get_xlim(), [zoom, zoom], color='k')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 578
        },
        "id": "MdsKjpLwD7Tm",
        "outputId": "a69a4523-4c25-4d2c-cf49-ad7749669a04"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "outputs": [],
      "source": [
        "class Decoder(tf.keras.layers.Layer):\n",
        "    def __init__(self, output_vocab_size, embedding_dim, dec_units):\n",
        "        super(Decoder, self).__init__()\n",
        "        self.dec_units = dec_units\n",
        "        self.output_vocab_size = output_vocab_size\n",
        "        self.embedding_dim = embedding_dim\n",
        "\n",
        "        # For Step 1. The embedding layer converts token IDs to vectors\n",
        "        self.embedding = tf.keras.layers.Embedding(\n",
        "            self.output_vocab_size,\n",
        "            embedding_dim\n",
        "        )\n",
        "\n",
        "        # For Step 2. The RNN keeps track of what's been generated so far.\n",
        "        self.gru = tf.keras.layers.GRU(\n",
        "            self.dec_units,\n",
        "            return_sequences=True,\n",
        "            return_state=True,\n",
        "            recurrent_initializer='glorot_uniform'\n",
        "        )\n",
        "\n",
        "        # For step 3. The RNN output will be the query for the attention layer.\n",
        "        self.attention = BahdanauAttention(self.dec_units)\n",
        "\n",
        "        # For step 4. Eqn. (3): converting `ct` to `at`\n",
        "        self.Wc = tf.keras.layers.Dense(\n",
        "            dec_units,\n",
        "            activation=tf.math.tanh,\n",
        "            use_bias=False\n",
        "        )\n",
        "\n",
        "        # For step 5. This fully connected layer produces the logits for each\n",
        "        # output token.\n",
        "        self.fc = tf.keras.layers.Dense(self.output_vocab_size)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "1hvnwqThD7Tn"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "outputs": [],
      "source": [
        "class DecoderInput(typing.NamedTuple):\n",
        "    new_tokens: Any\n",
        "    enc_output: Any\n",
        "    mask: Any\n",
        "\n",
        "\n",
        "class DecoderOutput(typing.NamedTuple):\n",
        "    logits: Any\n",
        "    attention_weights: Any"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "lh0usODAD7Tn"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "outputs": [],
      "source": [
        "def call(\n",
        "        self,\n",
        "        inputs: DecoderInput,\n",
        "        state=None\n",
        ") -> Tuple[DecoderOutput, tf.Tensor]:\n",
        "    shape_checker = ShapeChecker()\n",
        "    shape_checker(inputs.new_tokens, ('batch', 't'))\n",
        "    shape_checker(inputs.enc_output, ('batch', 's', 'enc_units'))\n",
        "    shape_checker(inputs.mask, ('batch', 's'))\n",
        "\n",
        "    if state is not None:\n",
        "        shape_checker(state, ('batch', 'dec_units'))\n",
        "\n",
        "    # Step 1. Lookup the embeddings\n",
        "    vectors = self.embedding(inputs.new_tokens)\n",
        "    shape_checker(vectors, ('batch', 't', 'embedding_dim'))\n",
        "\n",
        "    # Step 2. Process one step with the RNN\n",
        "    rnn_output, state = self.gru(vectors, initial_state=state)\n",
        "\n",
        "    shape_checker(rnn_output, ('batch', 't', 'dec_units'))\n",
        "    shape_checker(state, ('batch', 'dec_units'))\n",
        "\n",
        "    # Step 3. Use the RNN output as the query for the attention over the\n",
        "    # encoder output.\n",
        "    context_vector, attention_weights = self.attention(\n",
        "        query=rnn_output,\n",
        "        value=inputs.enc_output,\n",
        "        mask=inputs.mask\n",
        "    )\n",
        "    shape_checker(context_vector, ('batch', 't', 'dec_units'))\n",
        "    shape_checker(attention_weights, ('batch', 't', 's'))\n",
        "\n",
        "    # Step 4. Eqn. (3): Join the context_vector and rnn_output\n",
        "    #     [ct; ht] shape: (batch t, value_units + query_units)\n",
        "    context_and_rnn_output = tf.concat([context_vector, rnn_output], axis=-1)\n",
        "\n",
        "    # Step 4. Eqn. (3): `at = tanh(Wc@[ct; ht])`\n",
        "    attention_vector = self.Wc(context_and_rnn_output)\n",
        "    shape_checker(attention_vector, ('batch', 't', 'dec_units'))\n",
        "\n",
        "    # Step 5. Generate logit predictions:\n",
        "    logits = self.fc(attention_vector)\n",
        "    shape_checker(logits, ('batch', 't', 'output_vocab_size'))\n",
        "\n",
        "    return DecoderOutput(logits, attention_weights), state"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "ssiD6Z_nD7Tn"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "outputs": [],
      "source": [
        "Decoder.call = call"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "iN7UGoTdD7Tn"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "logits shape: (batch_size, t, output_vocab_size) (64, 1, 99)\n",
            "state shape: (batch_size, dec_units) (64, 1024)\n"
          ]
        }
      ],
      "source": [
        "decoder = Decoder(\n",
        "    output_text_processor.vocabulary_size(),\n",
        "    embedding_dim,\n",
        "    units\n",
        ")\n",
        "\n",
        "# Convert the target sequence, and collect the \"[START]\" tokens\n",
        "example_output_tokens = output_text_processor(example_target_batch)\n",
        "\n",
        "start_index = output_text_processor.get_vocabulary().index('[START]')\n",
        "first_token = tf.constant([[start_index]] * example_output_tokens.shape[0])\n",
        "\n",
        "# Run the decoder\n",
        "dec_result, dec_state = decoder(\n",
        "    inputs=DecoderInput(\n",
        "        new_tokens=first_token,\n",
        "        enc_output=example_enc_output,\n",
        "        mask=(example_tokens != 0)\n",
        "    ),\n",
        "    state=example_enc_state\n",
        ")\n",
        "\n",
        "print(f'logits shape: (batch_size, t, output_vocab_size) {dec_result.logits.shape}')\n",
        "print(f'state shape: (batch_size, dec_units) {dec_state.shape}')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ynldYX6D7Tn",
        "outputId": "5d837914-87ea-4c69-ed8e-6b2bdbe29ff8"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['̚']\n",
            " ['t']\n",
            " ['ŋ']\n",
            " ['a']\n",
            " ['̆']]\n"
          ]
        }
      ],
      "source": [
        "sampled_token = tf.random.categorical(dec_result.logits[:, 0, :], num_samples=1)\n",
        "vocab = np.array(output_text_processor.get_vocabulary())\n",
        "first_word = vocab[sampled_token.numpy()]\n",
        "print(first_word[:5])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qeQ42mD-D7To",
        "outputId": "431c81c3-fd97-4a58-ff41-cf9a2ed255b0"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "outputs": [],
      "source": [
        "dec_result, dec_state = decoder(\n",
        "    DecoderInput(\n",
        "        sampled_token,\n",
        "        example_enc_output,\n",
        "        mask=(example_tokens != 0)\n",
        "    ),\n",
        "    state=dec_state\n",
        ")"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "eC2oLHYxD7To"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['ɪ']\n",
            " ['ʷ']\n",
            " ['m']\n",
            " ['ˀ']\n",
            " ['ɛ']]\n"
          ]
        }
      ],
      "source": [
        "sampled_token = tf.random.categorical(dec_result.logits[:, 0, :], num_samples=1)\n",
        "first_word = vocab[sampled_token.numpy()]\n",
        "print(first_word[:5])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ke7CZWiqD7To",
        "outputId": "e1b58755-d970-4efa-b392-9f9d98380d09"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "outputs": [],
      "source": [
        "class MaskedLoss(tf.keras.losses.Loss):\n",
        "    def __init__(self):\n",
        "        self.name = 'masked_loss'\n",
        "        self.loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "            from_logits=True, reduction='none')\n",
        "\n",
        "    def __call__(self, y_true, y_pred):\n",
        "        shape_checker = ShapeChecker()\n",
        "        shape_checker(y_true, ('batch', 't'))\n",
        "        shape_checker(y_pred, ('batch', 't', 'logits'))\n",
        "\n",
        "        # Calculate the loss for each item in the batch.\n",
        "        loss = self.loss(y_true, y_pred)\n",
        "        shape_checker(loss, ('batch', 't'))\n",
        "\n",
        "        # Mask off the losses on padding.\n",
        "        mask = tf.cast(y_true != 0, tf.float32)\n",
        "        shape_checker(mask, ('batch', 't'))\n",
        "        loss *= mask\n",
        "\n",
        "        # Return the total.\n",
        "        return tf.reduce_sum(loss)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "GXPPcuYgD7To"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "outputs": [],
      "source": [
        "class TrainTranslator(tf.keras.Model):\n",
        "    def __init__(\n",
        "            self,\n",
        "            embedding_dim,\n",
        "            units,\n",
        "            input_text_processor,\n",
        "            output_text_processor,\n",
        "            use_tf_function=True\n",
        "    ):\n",
        "        super().__init__()\n",
        "        # Build the encoder and decoder\n",
        "        encoder = Encoder(\n",
        "            input_text_processor.vocabulary_size(),\n",
        "            embedding_dim,\n",
        "            units\n",
        "        )\n",
        "        decoder = Decoder(\n",
        "            output_text_processor.vocabulary_size(),\n",
        "            embedding_dim,\n",
        "            units\n",
        "        )\n",
        "\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.input_text_processor = input_text_processor\n",
        "        self.output_text_processor = output_text_processor\n",
        "        self.use_tf_function = use_tf_function\n",
        "        self.shape_checker = ShapeChecker()\n",
        "\n",
        "    def train_step(self, inputs):\n",
        "        self.shape_checker = ShapeChecker()\n",
        "        if self.use_tf_function:\n",
        "            return self._tf_train_step(inputs)\n",
        "        else:\n",
        "            return self._train_step(inputs)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "pLczzYWcD7To"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "outputs": [],
      "source": [
        "def _preprocess(self, input_text, target_text):\n",
        "    self.shape_checker(input_text, ('batch',))\n",
        "    self.shape_checker(target_text, ('batch',))\n",
        "\n",
        "    # Convert the text to token IDs\n",
        "    input_tokens = self.input_text_processor(input_text)\n",
        "    target_tokens = self.output_text_processor(target_text)\n",
        "    self.shape_checker(input_tokens, ('batch', 's'))\n",
        "    self.shape_checker(target_tokens, ('batch', 't'))\n",
        "\n",
        "    # Convert IDs to masks.\n",
        "    input_mask = input_tokens != 0\n",
        "    self.shape_checker(input_mask, ('batch', 's'))\n",
        "\n",
        "    target_mask = target_tokens != 0\n",
        "    self.shape_checker(target_mask, ('batch', 't'))\n",
        "\n",
        "    return input_tokens, input_mask, target_tokens, target_mask"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "xjCoUfUlD7To"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "outputs": [],
      "source": [
        "TrainTranslator._preprocess = _preprocess"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "cBgkV9CWD7To"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "outputs": [],
      "source": [
        "def _train_step(self, inputs):\n",
        "    input_text, target_text = inputs\n",
        "\n",
        "    (\n",
        "        input_tokens,\n",
        "        input_mask,\n",
        "        target_tokens,\n",
        "        target_mask\n",
        "    ) = self._preprocess(input_text, target_text)\n",
        "\n",
        "    max_target_length = tf.shape(target_tokens)[1]\n",
        "\n",
        "    with tf.GradientTape() as tape:\n",
        "        # Encode the input\n",
        "        enc_output, enc_state = self.encoder(input_tokens)\n",
        "        self.shape_checker(enc_output, ('batch', 's', 'enc_units'))\n",
        "        self.shape_checker(enc_state, ('batch', 'enc_units'))\n",
        "\n",
        "        # Initialize the decoder's state to the encoder's final state.\n",
        "        # This only works if the encoder and decoder have the same number of\n",
        "        # units.\n",
        "        dec_state = enc_state\n",
        "        loss = tf.constant(0.0)\n",
        "\n",
        "        for t in tf.range(max_target_length - 1):\n",
        "            # Pass in two tokens from the target sequence:\n",
        "            # 1. The current input to the decoder.\n",
        "            # 2. The target for the decoder's next prediction.\n",
        "            new_tokens = target_tokens[:, t:t + 2]\n",
        "            step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n",
        "                                                   enc_output, dec_state)\n",
        "            loss = loss + step_loss\n",
        "\n",
        "        # Average the loss over all non padding tokens.\n",
        "        average_loss = loss / tf.reduce_sum(tf.cast(target_mask, tf.float32))\n",
        "\n",
        "    # Apply an optimization step\n",
        "    variables = self.trainable_variables\n",
        "    gradients = tape.gradient(average_loss, variables)\n",
        "    self.optimizer.apply_gradients(zip(gradients, variables))\n",
        "\n",
        "    # Return a dict mapping metric names to current value\n",
        "    return {'batch_loss': average_loss}\n"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "ldgIpPsbD7Tp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "outputs": [],
      "source": [
        "TrainTranslator._train_step = _train_step"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "Yf9Qgum8D7Tp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "outputs": [],
      "source": [
        "def _loop_step(self, new_tokens, input_mask, enc_output, dec_state):\n",
        "    input_token, target_token = new_tokens[:, 0:1], new_tokens[:, 1:2]\n",
        "\n",
        "    # Run the decoder one step.\n",
        "    decoder_input = DecoderInput(\n",
        "        new_tokens=input_token,\n",
        "        enc_output=enc_output,\n",
        "        mask=input_mask\n",
        "    )\n",
        "\n",
        "    dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n",
        "    self.shape_checker(dec_result.logits, ('batch', 't1', 'logits'))\n",
        "    self.shape_checker(dec_result.attention_weights, ('batch', 't1', 's'))\n",
        "    self.shape_checker(dec_state, ('batch', 'dec_units'))\n",
        "\n",
        "    # `self.loss` returns the total for non-padded tokens\n",
        "    y = target_token\n",
        "    y_pred = dec_result.logits\n",
        "    step_loss = self.loss(y, y_pred)\n",
        "\n",
        "    return step_loss, dec_state"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "TW56EguSD7Tp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "outputs": [],
      "source": [
        "TrainTranslator._loop_step = _loop_step"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "Zx3McK4fD7Tp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "outputs": [],
      "source": [
        "translator = TrainTranslator(\n",
        "    embedding_dim,\n",
        "    units,\n",
        "    input_text_processor=input_text_processor,\n",
        "    output_text_processor=output_text_processor,\n",
        "    use_tf_function=False\n",
        ")\n",
        "\n",
        "# Configure the loss and optimizer\n",
        "translator.compile(\n",
        "    optimizer=tf.optimizers.Adam(),\n",
        "    loss=MaskedLoss(),\n",
        ")"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "yxXonrKPD7Tp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4.59511985013459"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ],
      "source": [
        "np.log(output_text_processor.vocabulary_size())"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I2wo7eKvD7Tp",
        "outputId": "7ebc5742-46f8-42f3-ffd4-5aaec50818da"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.2409573>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.1319914>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=3.9159713>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=3.4024065>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=5.834148>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.979774>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=3.184472>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=3.3096178>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=3.333712>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=3.291463>}\n",
            "\n",
            "CPU times: user 1min 31s, sys: 2.29 s, total: 1min 33s\n",
            "Wall time: 1min 14s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "for n in range(10):\n",
        "    print(translator.train_step([example_input_batch, example_target_batch]))\n",
        "print()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZarPakND7Tp",
        "outputId": "e0aecbe8-1167-4777-8f7b-f1adba4d24b9"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "outputs": [],
      "source": [
        "@tf.function(input_signature=[[tf.TensorSpec(dtype=tf.string, shape=[None]),\n",
        "                               tf.TensorSpec(dtype=tf.string, shape=[None])]])\n",
        "def _tf_train_step(self, inputs):\n",
        "    return self._train_step(inputs)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "WwUClfw0D7Tq"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "outputs": [],
      "source": [
        "TrainTranslator._tf_train_step = _tf_train_step"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "Mzl8HOOBD7Tq"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "outputs": [],
      "source": [
        "translator.use_tf_function = True"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "-sl4iLV3D7Tq"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=3.209661>}"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ],
      "source": [
        "translator.train_step([example_input_batch, example_target_batch])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5rubZTfsD7Tq",
        "outputId": "597ee907-6d70-40b0-cd4c-0132c2f97c8a"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=3.1068137>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.9815407>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.8349876>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.6940465>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.630425>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.640282>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.5321374>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.4578242>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.4177098>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=2.3865001>}\n",
            "\n",
            "CPU times: user 1min 12s, sys: 656 ms, total: 1min 13s\n",
            "Wall time: 45.1 s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "for n in range(10):\n",
        "    print(translator.train_step([example_input_batch, example_target_batch]))\n",
        "print()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rYBIBxRnD7Tq",
        "outputId": "7134bc37-4f34-4821-c5d7-8f452eab6ab1"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "....................................................................................................\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f89c3b862b0>]"
            ]
          },
          "metadata": {},
          "execution_count": 59
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBZUlEQVR4nO3deVhU9eIG8PfMwg6DgGwKgorigoDIqqUlZWamVmbuuZtamt262WbdbtnvdtvdNZfcs9wy0wzNDQRBUVFREQUEBgSEYZF1zu8PcorrEiBwZnk/zzPPk2fOYV6+z73O6znf8z2CKIoiiIiIiCQikzoAERERmTaWESIiIpIUywgRERFJimWEiIiIJMUyQkRERJJiGSEiIiJJsYwQERGRpFhGiIiISFIKqQPUh1arRVZWFmxtbSEIgtRxiIiIqB5EUURxcTHc3d0hk937/IdBlJGsrCx4eHhIHYOIiIgaISMjA23btr3n+wZRRmxtbQHU/jJ2dnYSpyEiIqL60Gg08PDw0H2P34tBlJHbl2bs7OxYRoiIiAzM302x4ARWIiIikhTLCBEREUmKZYSIiIgkxTJCREREkmIZISIiIkmxjBAREZGkWEaIiIhIUiwjREREJCmWESIiIpIUywgRERFJimWEiIiIJMUyQkRERJIy6TJy+NINTF4bj1uVNVJHISIiMlkmW0ZKK6oxZ0sifruQgwlr4lBaUS11JCIiIpNksmXE2lyBZWODYGOuwPHUAoxbFQdNeZXUsYiIiEyOyZYRAAj2csD6yaGws1AgIe0mxq6MRWFZpdSxiIiITIpJlxEACPCwx8YpYWhlpcTp60UYuSIW+SUVUsciIiIyGSZfRgCgexsVNk8Nh5ONOS5kazB8WQyu5ZVKHYuIiMgksIz8obOrLbZMC4O7ygKpN0oxbPExxF0tkDoWERGR0WMZ+YsOrW2wY2Zv9Girws2yKoxeeRw/JlyXOhYREZFRYxn5H852FtgyNRwDu7uiqkbEa1tP47/7LkKrFaWORkREZJRYRu7C0kyORaN6Yka/DgCAhQdTMGvTSS6ORkRE1AxYRu5BJhPwxhO++O9wfyjlAvacVWPE8hjkaMqljkZERGRUWEb+xnNBbbFxShgcrM1w5noRnl54FGevF0kdi4iIyGiwjNRDsJcDdszoDR9nG+RoKjB8WTT2JmVLHYuIiMgosIzUk6ejFbbNiEC/zq1RXqXFSxtO4vsTGVLHIiIiMngsIw1ga6HEt+ODMSrUE6IIvPHjGaw+dlXqWERERAaNZaSB5DIBHw3tjikPeQMAPvjpPBYdTJE4FRERkeFiGWkEQRDw1pNdMLu/DwDg030X8Z+9yRBFrkVCRETUUCwjjSQIAl59rBPmDfQFACz+/Qre33WOi6MRERE1EMvIA5rWtwM+HNodALA2Jg3/2Hoa1TVaiVMREREZDpaRJjA2rB2+HBEAuUzAtlOZeGnDSZRXcbVWIiKi+mAZaSJDA9tg2ZggmClk2H8+BxPXnEBJRbXUsYiIiPQey0gTiuzqgjUTgmFtJkf0lXyMXnEcuVw+noiI6L5YRppYRAcnbJwShlZWSpy+XoSnFx7D6YxCqWMRERHpLZaRZuDvYY/tM3qjo7MN1JpyPL8sBjsTM6WORUREpJdYRpqJl5M1ts+IQH9fZ1RUazF7cyIW/HIBNbz1l4iIqA6WkWZka6HE8nG9MKNfBwDAskOpeHF1HPJLKiRORkREpD9YRpqZXCbgjSd88fXIQFgoZThyOQ+Dvj6KhLQCqaMRERHpBZaRFvK0vzt2zuyD9q2todaUY8Sy41h5JJVLyBMRkcljGWlBnV1tsWtWHwz2d0e1VsS/f76A6esTUFxeJXU0IiIiybCMtDAbcwW+fiEAHw7pBjO5DPvO5WDIomNIyS2ROhoREZEkWEYkIAgCxoZ7Yev0cLipLJB6oxRDFx3DvnNqqaMRERG1OJYRCfl72OOnl/sg1NsBJRXVmLYuAf/dd5G3/xIRkUlhGZGYk4051k8OxcTe3gCAhQdTMG1dPMoq+VwbIiIyDSwjekApl+G9wV3x5YgAmCtk+O1CLkYuP44bxVyPhIiIjB/LiB4ZGtgGG6eE6p5r88ySY7hygxNbiYjIuLGM6Jmgdg748aUIeDpYIaPgFp5dEo34a1wgjYiIjBfLiB5q39oG22ZEwN/DHoVlVRi1Mha/nM2WOhYREVGzYBnRU0425tg8JQyRXVxQWa3FjI0nsTb6mtSxiIiImhzLiB6zNJNj6ZieGB3qCVEE5u86hwW/XICWt/4SEZERYRnRcwq5DP8e2h2vD+gMoPbJv3O/T0RltVbiZERERE2DZcQACIKAmY90xKfP9YBCJmBHYhZGrTiO9PwyqaMRERE9MJYRAzK8lwe+fTEY1mZyxKfdxBNfHcaG2DQ++ZeIiAway4iB6dupNfbOeRih3g4oq6zB29uTMG5VHLKLbkkdjYiIqFFYRgyQh4MVNk0Jw7tPdYW5QoYjl/Pw+BeHsfj3FJRWcBl5IiIyLIJoAOf4NRoNVCoVioqKYGdnJ3UcvZKSW4LXtp7G6YxCAICDtRmmPdweY8PbwcpMIW04IiIyafX9/mYZMQI1WhE7EzPxddRlXPtjUquTjRnmRHbC6FBPCIIgcUIiIjJFLCMmqLpGi+2nMvH1gcvIKKidQ/Jsz7b4aFh3WCjlEqcjIiJTU9/vb84ZMSIKuQzDe3ngwGv98NaTvpAJwI8nr+OF5ceRoymXOh4REdFdsYwYIaVchqkPd8B3E0OhslQiMaMQg785ilPpN6WORkREdAeWESPWx8cJO2f2ho+zDXKLKzBi+XE+cI+IiPROg8rIggULEBwcDFtbWzg7O2Po0KG4ePHi3x63detW+Pr6wsLCAn5+ftizZ0+jA1PDeDlZY9uMCN0D92ZtOsVCQkREeqVBZeTQoUOYOXMmjh8/jv3796OqqgqPP/44SktL73lMdHQ0Ro4ciUmTJuHUqVMYOnQohg4diqSkpAcOT/Vja6HEsrFBGBbYBjVakYWEiIj0ygPdTXPjxg04Ozvj0KFDePjhh++6z4gRI1BaWordu3frtoWFhSEgIABLly6t1+fwbpqmUaMV8Y+tp7H9VCbkMgELRwZioJ+b1LGIiMhItcjdNEVFRQAABweHe+4TExODyMjIOtsGDBiAmJiYB/loagS5TMB/h/vXOUOyh2dIiIhIYo0uI1qtFnPmzEHv3r3RvXv3e+6nVqvh4uJSZ5uLiwvUavU9j6moqIBGo6nzoqZxRyHZeBLLD1/hw/aIiEgyjS4jM2fORFJSEjZv3tyUeQDUTpRVqVS6l4eHR5N/him7XUhGhnhAKwIf70nGq1sSUV5VI3U0IiIyQY0qI7NmzcLu3btx8OBBtG3b9r77urq6Iicnp862nJwcuLq63vOYefPmoaioSPfKyMhoTEy6D7lMwMfD/PDB090glwnYkZiF4UtjkFXIp/8SEVHLalAZEUURs2bNwvbt23HgwAF4e3v/7THh4eGIioqqs23//v0IDw+/5zHm5uaws7Or86KmJwgCxkd4Yf2kUDhYm+FsZhGeXngUcVcLpI5GREQmpEFlZObMmVi/fj02btwIW1tbqNVqqNVq3Lr157+mx40bh3nz5un+PHv2bOzduxefffYZkpOT8f777yM+Ph6zZs1qut+CHkh4B0fsnNkbXdzskFdSiZErjuPbo1c5j4SIiFpEg8rIkiVLUFRUhH79+sHNzU332rJli26f9PR0ZGf/eYdGREQENm7ciOXLl8Pf3x8//PADduzYcd9Jr9TyPBys8ONL4RgS4I4arYgPd5/HK5sTUVpRLXU0IiIycnxqL9UhiiLWRl/Dv3++gGqtiE4uNlg6JgjtW9tIHY2IiAwMn9pLjSIIAl7s7Y3NU8PgbGuOSzklGLLwGA5fuiF1NCIiMlIsI3RXvbwcsPuVPgj2aoXiimpMWHMCm+LSpY5FRERGiGWE7snZ1gLrJ4fimT8WSJu37SwW/HIBWq3eX9kjIiIDwjJC92WukOOz5/3xamQnAMCyQ6mYufEkF0gjIqImwzJCf0sQBMyO9MEXI/xhJpfhlyQ1Riw/jhvFFVJHIyIiI8AyQvU2LLAt1k8Ohb2VEqczCjFs8TFczimWOhYRERk4lhFqkBBvB2x7KQJejla4fvMWnlkSjWMpeVLHIiIiA8YyQg3WvrUNts3oXXunTXk1xq+Kw/cn+PwgIiJqHJYRahQHazOsnxyKIQHuqNaKeOPHM1h0MEXqWEREZIBYRqjRzBVyfDkiALMe6QgA+HTfRXy6L5nPtCEiogZhGaEHIggC/jGgM+YN9AUALDp4Bf/afZ6FhIiI6o1lhJrEtL4d8OGQbgCA1ceuYd62s6jh4mhERFQPLCPUZMaGe+G/w/0hE4DNJzIwe/MpVFRzcTQiIro/lhFqUs8FtcU3I3tCIROw+0w2xn0bh6KyKqljERGRHmMZoSY3qIcb1kwIgY25ArFXC/Ds0mhcv1kmdSwiItJTLCPULPr4OGHr9HC42lkgJbcEwxZHIymzSOpYRESkh1hGqNl0cbPD9pkR8HW1xY3iCjy/LIartRIR0R1YRqhZuaks8f30cPTu6IiyyhpMWHMCBy/mSh2LiIj0CMsINTs7CyVWvRiMyC4uqKzWYup38fj1nFrqWEREpCdYRqhFmCvkWDKmJwb5uaGqRsSMDSex+0yW1LGIiEgPsIxQi1HKZfjqhQA8E9gG1VoRr2w6hW0nr0sdi4iIJMYyQi1KIZfh0+H+eCHYA1oR+MfW09jHSzZERCaNZYRanFwm4ONhfhjRq7aQvLzpFGJT86WORUREEmEZIUnIZAI+GtYdj3WtndQ6+bt4XMjWSB2LiIgkwDJCklHIZfhmZCCCvVqhuLwa41fFIaOAK7USEZkalhGSlIVSjpXjgtHZxRa5xRUYtyoOeSUVUsciIqIWxDJCklNZKfHdpBC0sbfE1bxSjFx+HLnF5VLHIiKiFsIyQnrBxc4C6yeHwtXOApdzS/DC8uPI0bCQEBGZApYR0hveTtbYMi0MbewtkXqjFCOWxSCr8JbUsYiIqJmxjJBeaedojc1Tw+DhYIlr+WUYsTyGk1qJiIwcywjpHQ8HK2yZGg4vRytkFNzCC8uPs5AQERkxlhHSS+72ltgyLRztW1sjs5CFhIjImLGMkN5ysbPA5ilhLCREREaOZYT0mvPtQuL0ZyG5fpOFhIjImLCMkN5ztrPApqlh8GYhISIySiwjZBBc7CywaUoYvBytcP3mLYxeGYtcrkNCRGQUWEbIYLiqas+QeDhYIi2/DGO+jUVBaaXUsYiI6AGxjJBBcVNZYsOkMLjYmeNSTgnGr4qDprxK6lhERPQAWEbI4Hg6WmHD5FA4WJvhbGYRJq05gVuVNVLHIiKiRmIZIYPU0dkW300Mga2FAieu3cTUdfGorNZKHYuIiBqBZYQMVvc2KqyZEAxLpRxHLufhjR9OQ6sVpY5FREQNxDJCBi2onQOWjg2CQiZgR2IW/rPvotSRiIiogVhGyOD17dQaC57xAwAsPXQFa6OvSRuIiIgahGWEjMLwXh74x+OdAADv/3QOe5OyJU5ERET1xTJCRmPmIx0xKtQTogi8sjkRJ64VSB2JiIjqgWWEjIYgCPjX090Q2cUZldVaTFh9AqfSb0odi4iI/gbLCBkVhVyGb0b2RKi3A0oqqjFuVRzOXC+UOhYREd0HywgZHUszOVa9GIwQLwcUl1djzMpYJGUWSR2LiIjugWWEjJK1uQKrJgQjqF0raMqrMXplLM5lsZAQEekjlhEyWjbmCqyZEIwAD3sU3arCmJWxSFZrpI5FRET/g2WEjJqthRLfTQqBf1sVbpbVFpKU3BKpYxER0V+wjJDRs7NQ4ruJoejqZoe8kkqMXnkcafmlUsciIqI/sIyQSVBZKbF+cig6udggR1OBUSticf1mmdSxiIgILCNkQhyszbBhchjat7ZGZuEtjFoRC3VReYN/Dh/GR0TUtARRFPX+b1aNRgOVSoWioiLY2dlJHYcMnLqoHM8vi0F6QRncVRZYNLonAj1b3bGfKIo4cjkPp9ILkZZfimv5pUjLL0NxRTWWjumJR31dJEhPRGQ46vv9zTJCJun6zTKM/TYOV/NKoZQLePvJLhgf4QVBEAAAyWoNPth1HjGp+Xc9/iEfJ6ybFNqSkYmIDA7LCNHf0JRX4Z8/nMEvSWoAwCA/N7w50BfLD6diQ2watCJgrpBhUA83dGhtAy9Ha1iayTBxTTzkMgEn3o6Eg7WZxL8FEZH+qu/3t6IFMxHpFTsLJRaP7onVx67h4z0X8PPZbPx89s+n/T7p54p5A7vAw8GqznHd3O1wLkuDfefUGBni2dKxiYiMDiewkkkTBAET+3jj++nhcFdZAAA6u9hi45RQLB4ddEcRAYBBPdwAAHv+UlyIiKjxeGaECEBPz1b4Zc7DSMosQqi3AxTye/f0QX5u+M/ei4i+ko/8kgo42pi3YFIiIuPDMyNEf1BZKtG7o9N9iwgAtHO0Rvc2dqjRith3LqeF0hERGS+WEaJGGOTnDoCXaoiImgLLCFEjDPKrnTcSfSUP+SUVEqchIjJsLCNEjeDpaAW/NipoRfBSDRHRA2pwGTl8+DAGDx4Md3d3CIKAHTt23Hf/33//HYIg3PFSq9WNzUykF5784+zIz2ezJE5CRGTYGlxGSktL4e/vj0WLFjXouIsXLyI7O1v3cnZ2buhHE+mV25dqYq7kI+8vl2pEUUR20S0YwHqCRER6ocG39g4cOBADBw5s8Ac5OzvD3t6+wccR6StPRyv0aKvCmetF2HdOjce6uuDHhExsjc9Aal4pXurXAf98wlfqmEREeq/F5owEBATAzc0Njz32GI4dO3bffSsqKqDRaOq8iPTR7Us1n+67iPAFB/B/e5ORmlcKAFhxOBUpuSVSxiMiMgjNXkbc3NywdOlS/Pjjj/jxxx/h4eGBfv364eTJk/c8ZsGCBVCpVLqXh4dHc8ckapTbl2oKy6pQoxXR09Menzzjh36dW6NaK+LjPRckTkhEpP8e6EF5giBg+/btGDp0aIOO69u3Lzw9PbFu3bq7vl9RUYGKij+vwWs0Gnh4ePBBeaSXtpxIR0bBLQwJcIePiy0A4MqNEgz44jCqtSK+mxiChzu1ljglEVHL0+sH5YWEhODo0aP3fN/c3Bzm5lximwzDiOA7H5bXobUNxoa3w+pj1/Dvn89jT4eH/nZlVyIiUyXJ346JiYlwc3OT4qOJWszs/j6wt1LiUk4JNp/IkDoOEZHeavCZkZKSEqSkpOj+fPXqVSQmJsLBwQGenp6YN28eMjMz8d133wEAvvzyS3h7e6Nbt24oLy/HypUrceDAAfz6669N91sQ6SF7KzO8GtkJ83edw+f7L2GwvztUlkqpYxER6Z0GnxmJj49HYGAgAgMDAQBz585FYGAg3nvvPQBAdnY20tPTdftXVlbitddeg5+fH/r27YvTp0/jt99+Q//+/ZvoVyDSX6NCPdGhtTUKSivxTdRlqeMQEemlB5rA2lLqOwGGSB8dvJiLCatPAADcVBYIatcKvdq1Qi8vB3R1s4NMJkickIioedT3+5tlhKgFvL/rHNYdT0ONtu7/3QI87PGf53qg0x934RARGROWESI9U1ZZjcSMQiRcu4n4tJuIu1qAW1U1UMoFvPyoD6b37QAzBe+4ISLjwTJCpOeyi27hne1JiErOBQD4utriP8/1QI+29tIGIyJqIvX9/uY/w4gk4qayxMrxvfDVCwFoZaVEsroYzyyOxpHLN6SORkTUolhGiCQkCAKGBLTBb3P7IrKLM6q1Il7edAoZBWVSRyMiajEsI0R6wNHGHAtH9YR/WxUKy6owbV0CblXWSB2LiKhFsIwQ6QkLpRxLxgTB0doM57M1mLftDAxgShcR0QNjGSHSI+72llg4qifkMgE7ErOw+tg1qSMRETU7lhEiPRPewRFvPdkFAPDRngvYczYbVTVaiVMRETUfSZ7aS0T3N7G3F85cL8TOxCzM2HASdhYK9O3sjMguzujbqTXsrcykjkhE1GRYRoj0kCAI+OSZHrCzUOLns9koKK3ET6ez8NPpLJgpZPj3kO54PthD6phERE2Ci54R6bkarYjEjJuIupCL/edzcDm3BAAwu78P5kT6QBD4bBsi0k9cgZXICImiiM9+vYSFB1MAAMOD2uLjZ/yglHP6FxHpH67ASmSEBEHAPwZ0xsfD/CATgK0J1zFpbTxKKqqljkZE1GgsI0QGaFSoJ1aO7wVLpRyHL93A6BXHWUiIyGCxjBAZqEd9XbBlWhgcrM1w+noRpq9LQEU1V20lIsPDMkJkwHq0tceaCcGwNpPjaEoeXvv+NLRavZ8GRkRUB8sIkYHr0dYeS8cGQSkXsPtMNv61+zyXkScig8IyQmQEHvJpjc+eDwAArIm+hsW/X5E2EBFRA7CMEBmJp/3dMX9wVwDAp/suYtKaE9iblI3Kai4lT0T6jSuwEhmRCb29UVhWha+iLiMqORdRybloZaXEkIA2GBPmiY7OtlJHJCK6Axc9IzJCKbkl+CHhOradvI7c4goAgJlchs9H+OOpHu4SpyMiU8EVWIkI1TVaHEnJw7dHruJoSh4A4J1BXTD5ofYSJyMiU8AVWIkICrkMj3R2xtqJIXgxwgsA8O+fL+DD3ed5CzAR6Q2WESITIJcJmD+4K+YN9AUAfHv0Kl7efArlVVwkjYikxzJCZCIEQcC0vh3w1QsBUMoF/HwmG2NWxiKvpELqaERk4lhGiEzMkIA2WDshBLYWCsSn3cSQhceQrNZIHYuITBjLCJEJiujohO0zesPL0QqZhbfw7OJoHEjOkToWEZkolhEiE9XR2QY7ZvZGeHtHlFbWYNLaeCw9dAXVNVwkjYhaFssIkQmztzLDd5NCMDLEA6IIfPJLMp746gh+Pafm822IqMWwjBCZOKVcho+H+eGjYd1hb6VESm4Jpq5LwHNLY3DiWoHU8YjIBHDRMyLS0ZRXYdmhK/j26FWUV9VerhkV6okPh3SHXCZInI6IDA0XPSOiBrOzUOL1Ab449PojGBXqCZkAbIxNx8ubTqKimmuSEFHzYBkhoju42Fng42F+WDSqJ5RyAXvOqjF5bTzKKquljkZERohlhIjuaaCfG74dHwxLpRxHLudh7LdxKCqrkjoWERkZlhEiuq+HO7XG+skhsLNQICHtJkYsj0FGQZnUsYjIiLCMENHfCmrngC3TwuFkY45kdTGe/OoIdiZmSh2LiIwEywgR1UsXNzvsmBmBXu1aobiiGrM3J+K170+jpILzSIjowbCMEFG9tW1lhc1Tw/BKfx/IBODHk9fx1NdHcDqjUOpoRGTAWEaIqEEUchnmPtYJm6eGw11lgWv5ZXhmSTT+u+8ib/8lokZhGSGiRgnxdsAvsx/GUz3cUKMVsfBgCp7+5hjOXi+SOhoRGRiWESJqNJWVEgtH9cSS0T3haG2GiznFGLr4GD779SIfuEdE9cYyQkQPbKCfG3599WEM+uMsyTcHUjB7cyILCRHVC8sIETUJRxtzLBrVE1+9EAClXMDPZ7NZSIioXlhGiKhJDQlog6VjguoUkioWEiK6D5YRImpy/bu4YOmYIJjJZfj5bDbmsJAQ0X2wjBBRs+jfxQVLxvTUFZLJa+Nx/SaXkSeiO7GMEFGz6d/FBUvH1haSQ5duoP9nh/Dlb5dQXsX1SIjoTywjRNSsHvV1wU8v90FYewdUVGvx5W+X0f+zQ9iblA1RFKWOR0R6gGWEiJpdZ1dbbJoShoWjAuGmskBm4S1MX38S7+5MQo2WhYTI1LGMEFGLEAQBT/VwR9RrfTHrkY4QBGD98XTM3nwKldWc3EpkylhGiKhFWZkp8I8BnfH1C4FQygXsPpONyd/Fo6yST/8lMlUsI0QkicH+7lg5PhiWSjkOX7qB0StjUVhWKXWs+yqvquHkW6JmwDJCRJLp26k1NkwJhcpSiVPphXh2STSS1RqpY91VZbUWA786gkFfH+GqskRNjGWEiCTV07MVtk4Ph4udOa7cKMXTC49hbfQ1vbvT5lJOMa7mleLKjVJcyy+VOg6RUWEZISLJdXKxxc+vPIRHOrdGZbUW83edw6S18cgrqZA6mk5SZpHuvy+qSyRMQmR8WEaISC842Zhj1YvBeH9wV5gpZDiQnIsnvjyC46n5UkcDAJz9axnJKZYwCZHxYRkhIr0hCAJe7O2NXbN6o5OLDfJKKjD221jsTMyUOhqSsv6cy3JRT+e1EBkqlhEi0ju+rnbYNasPnvRzRVWNiNmbE7HoYIpk80iqarS4kP1nAbmUw8s0RE2JZYSI9JKFUo6FI3ti6sPtAQCf7ruIt7afleROlis3SlBZrYWZvPavzGv5pbzFl6gJsYwQkd6SyQS89WQXfPB0NwgCsCkuA5PWxkNTXtWiOZIya8+KBHjao5WVEqIIpOTy7AhRU2EZISK9Nz7CC8vGBMFCWfv036GLjuHKjZYrA7fvpPFro0InF1sAQLKak1iJmgrLCBEZhMe7uWLrtAi4qSyQeqMUQxcew8Hk3Bb57NtlpHsbO3R2rS0jl3hHDVGTaXAZOXz4MAYPHgx3d3cIgoAdO3b87TG///47evbsCXNzc3Ts2BFr1qxpRFQiMnV+bVXYNasPgr1aobiiGhPXnmj2ia01WhHn/5i86tdGpSsjF3lmhKjJNLiMlJaWwt/fH4sWLarX/levXsWgQYPwyCOPIDExEXPmzMHkyZOxb9++BoclImpta44Nk8MwOtQTolg7sfW1raebbWLr1bxSlFXWwFIph7eTDTq7sIwQNTVFQw8YOHAgBg4cWO/9ly5dCm9vb3z22WcAgC5duuDo0aP44osvMGDAgIZ+PBERzBQyfDTMD13d7fDeznPYdjITZRU1+GpkAMwV8ib9rHNZtZdourrbQS4T0OmPMyNqTTmKyqqgslI26ecRmaJmnzMSExODyMjIOtsGDBiAmJiY5v5oIjJyo0PbYemYIJjJZdh7To0p3yXgVmXT3nKrmy/ibgcAsLNQwl1lAQC4lMuzI0RNodnLiFqthouLS51tLi4u0Gg0uHXr1l2PqaiogEajqfMiIrqbx7q6YNWLwbBUynH40g2MXxWH4ia89ff2bb3d2qh0226fHeEdNURNQy/vplmwYAFUKpXu5eHhIXUkItJjfXycsG5SCGzNFYi7VoDRK2ORVXj3f+w0hCiKSMq6fWbkzzKiu6OGZYSoSTR7GXF1dUVOTk6dbTk5ObCzs4OlpeVdj5k3bx6Kiop0r4yMjOaOSUQGrpeXAzZNDUMrKyXOXC/CE18exu4zWQ/0M9MLylBcXg0zhQw+Lja67bpJrLy9l6hJNHsZCQ8PR1RUVJ1t+/fvR3h4+D2PMTc3h52dXZ0XEdHf6d5Ghe0zesPfwx6a8mrM2ngKr31/GiUV1Y36ebcv0XRxtYVS/udfl7cXPruUUyzZ83KIjEmDy0hJSQkSExORmJgIoPbW3cTERKSnpwOoPasxbtw43f7Tp09Hamoq3njjDSQnJ2Px4sX4/vvv8eqrrzbNb0BE9BdeTtb4YXo4Xn60I2QC8OPJ63jyqyNIzChs8M+6fYnmr/NFAKCjsw1kAlBYVoXc4oqmiE1k0hpcRuLj4xEYGIjAwEAAwNy5cxEYGIj33nsPAJCdna0rJgDg7e2Nn3/+Gfv374e/vz8+++wzrFy5krf1ElGzUcpleO3xztgyLRxt7C2RXlCGkcuPI/pKXoN+zp930tQtIxZKObycrAFwvRGipiCIBnCOUaPRQKVSoaioiJdsiKhBNOVVmLXxFA5fugFzhQwrxvXCw51a/+1xoigi6N+/oaC0Ertm9UaPtvZ13n9pfQJ+SVLjnUFdMPmh9s2Unsiw1ff7Wy/vpiEiaip2FkosHxuE/r7OqKjWYvLaeBxIzvnb47KKylFQWgmFTNDdPfNXnXl7L1GTYRkhIqNnoZRjyZggPNHNFZU1Wkxbl4C9Ser7HnP7Ek0nF9u7rura2YUPzCNqKiwjRGQSzBQyfDMqEE/1cENVjYiZG0/i3R1JUBeV37FvWn4pNsTWzn3r3ubup5Y7/eXpvVpt7dVurVbE3qTsB76lmMjUNPjZNEREhkopl+HLEQGwVMqxNeE61h1Pw5b4DIwK8cSMfh1QdKsKi3+/gp2JmfijX2Bgd7e7/iwvR2uYKWQor9Ii42bteiTzd51DQtpNAICzrQVCvB1a6lcjMmicwEpEJinmSj6+2H8JcdcKANSeOamq0eL234h9O7XGrEc7Itjr3oVi0NdHcC5Lg2CvVohPu4m//m36pJ8rFo8Oas5fgUjv1ff7m2dGiMgkhXdwRFj7MERfycfn+y/pzmgM6OaCWY/4wK+t6m9+Qu28kXNZGpy4VnvskAB3DAtsgxdXn8C+cznILLyFNvZ3X2maiP7EMkJEJksQBPTu6ISIDo44l6WBjblCt35IfYR1cMS2U5nwdbXFB093Q2h7RwBAeHtHxKTmY/3xNPzzCd/mik9kNFhGiMjkCYKA7m3+/kzI/xoe1BbBXg7waGUJxV+Wi3+xtxdiUvOxKS4ds/v7wEJ55904RPQn3k1DRNRIgiDA28m6ThEBgMguLmhjb4nCsirsTMyUKB2R4WAZISJqYnKZgPER7QAAq49d48P0iP4GywgRUTMY0csTlko5ktXFiL1aIHUcIr3GMkJE1AxUVkoM69kGALDm2DXd9spqLX6/mItdp7N0i6URmTpOYCUiaiYTIrywMTYdv55X48eE64i+ko/959XQlFcDAGJT8/Hvod0hCILESYmkxTJCRNRMfFxs0aejE46m5OG1rad1251szJBfWokNsemQCQL+NaQbCwmZNJYRIqJmNPORjohJzYeTjRkGdnfDwO6u6OXlgO2nMvH6D6ex7nga5DIB8wd3ZSEhk8UyQkTUjMI7OOLM/MdhqZRDJvuzbDwX1BZarYg3fjyDNdHXIAjAe0+xkJBp4gRWIqJmZm2uqFNEbns+2AOfPOMHoPYW4C9+u9zS0Yj0AssIEZGEXgjxxEfDugMAlvyegqzCWxInImp5LCNERBIbHdoOod4OqKoRsfTQFanjELU4lhEiIj0wu78PAGBzXAbUReUSpyFqWSwjRER6ILyDI4K9WqGyRsuzI2RyWEaIiPSAIAiY3b8TAGBTXDpyNTw7QqaDZYSISE/07uiIoHatUFGtxdJDqVLHIWoxLCNERHpCEAS88sfckQ2xacgt5tkRMg0sI0REeuRhHycEeNijolqLFYf/PDtSVaOFuqicD9cjo8QVWImI9IggCJgd6YMJq0/gu5g0xF0tQHZROW6UVEAUgU4uNlg/ORTOthZSRyVqMjwzQkSkZ/p1aq07O3L6ehFyi2uLCABcyinBmJWxKCitlDYkURMSRFHU+3N+Go0GKpUKRUVFsLOzkzoOEVGzy9GU4/eLuXCwNoebygKuKguUlFdjxPIY5Ggq0M3dDhunhEFlqZQ6KtE91ff7m2WEiMiApOSWYMSyGOSXViLQ0x7rJoXCxpxX3Ek/1ff7m5dpiIgMSEfn2jkj9lZKnEovxMQ1J3CrskbqWEQPhGWEiMjAdHGzw3cTQ2BrrkDc1QK8/sNpGMBJbqJ7YhkhIjJAPdraY+X4XlDIBOw+k40VR7hIGhkulhEiIgMV2t4R8wd3BQB88ksyjly+IXEiosZhGSEiMmBjwtpheFBbaEXg5U2nkFFQJnUkogZjGSEiMmCCIODDod3h31aFwrIqTF2XwAmtZHBYRoiIDJyFUo4lY4LgZGOGC9kavLzpJNRFfK4NGQ6WESIiI+Bub4lFo3pCIRPw24VcPPzpQby/6xxyNCwlpP9YRoiIjERoe0dsnBKGXu1aobJaizXR1/DQf2pLyU0uH096jCuwEhEZGVEUcSwlH1/8dgkJaTcBAK52Fvh6ZCBCvB0kTkemhCuwEhGZKEEQ0MfHCT9MD8e6SSFo72QNtaYcI1ccx6KDKdBq9f7foGRiWEaIiIyUIAh4yKc1fnq5D4YFtkGNVsSn+y5i/Oo43CiukDoekQ7LCBGRkbM2V+Dz5/3x6XM9YKmU48jlPDz1zRFkFd6SOhoRAJYRIiKTIAgChvfywK5ZvdG+tTVyNBV4aX0Cyqu4JglJj2WEiMiE+LjYYu2EENhbKXH6ehHe2ZHEh+yR5FhGiIhMjIeDFRaO7AmZAPyQcB3rj6dJHYlMHMsIEZEJ6uPjhDcH+gIAPvjpPOKuFkiciEwZywgRkYma8lB7DPZ3R7VWxIwNCZzQSpJhGSEiMlGCIOD/nvWDr6st8koqMejrI9hyIp3rkFCLYxkhIjJhVmYKrBjXC76utrhZVoV//ngWzy2NxrmsIqmjkQlhGSEiMnEeDlb46eU+eGdQF1ibyXEyvRCDvzmK93edQ9GtKqnjkQlgGSEiIijlMkx+qD2iXuuHp3q4QSsCa6Kv4dH//s5LN9Ts+KA8IiK6w9HLeZi/KwlXbpQCAHq0VeH9p7uhp2criZORIeGD8oiIqNH6+Dhh75yH8c6gLrAxV+DM9SI8szgan+5LljoaGSGWESIiuqvbl24O/KMvhge1BQAsOngFW+MzJE5GxoZlhIiI7svZ1gKfDvfH7P4+AIC3dyThzPVCaUORUWEZISKiepnd3weRXZxRWa3FtHUJyCupkDoSGQmWESIiqheZTMDnIwLQ3ska2UXlmLnhJKpqtFLHIiPAMkJERPVmZ6HE8nFBsDFXIPZqAT76+YLUkcgIsIwQEVGDdHS2xWfP+wOoXYtk6nfxSL1RInEqMmQsI0RE1GADurli3kBfyATg1/M5ePyLw5i/Mwn5nEdCjcAyQkREjTKtbwfsm/MwHvV1RrVWxNqYNPT79HesOXYVBrCeJukRlhEiImo0HxdbrHoxGBsnh6Kbux2KK6rx/k/n8crmRJRVVksdjwwEywgRET2wiI5O+GlWH8wf3BUKmYCfTmfhmcXRSM8vkzoaGQCWESIiahIymYAJvb2xcUoYnGzMkKwuxuCFR3H40g2po5Gea1QZWbRoEby8vGBhYYHQ0FDExcXdc981a9ZAEIQ6LwsLi0YHJiIi/Rbi7YCfXu4Dfw97FN2qwour47A5Ll3qWKTHGlxGtmzZgrlz52L+/Pk4efIk/P39MWDAAOTm5t7zGDs7O2RnZ+teaWlpDxSaiIj0m5vKElumhmF4UFtoReDNbWexMZaFhO6uwWXk888/x5QpUzBhwgR07doVS5cuhZWVFVatWnXPYwRBgKurq+7l4uLyQKGJiEj/WSjl+M9zPTCxtzcA4K3tZ7H+OP8xSndqUBmprKxEQkICIiMj//wBMhkiIyMRExNzz+NKSkrQrl07eHh4YMiQITh37tx9P6eiogIajabOi4iIDI8gCHj3qS6Y3Ke2kLyzIwnrWEjofzSojOTl5aGmpuaOMxsuLi5Qq9V3PaZz585YtWoVdu7cifXr10Or1SIiIgLXr1+/5+csWLAAKpVK9/Lw8GhITCIi0iOCIODtQV0w9eH2AIB3dyRh5ZFUrkVCOs1+N014eDjGjRuHgIAA9O3bF9u2bUPr1q2xbNmyex4zb948FBUV6V4ZGRnNHZOIiJqRIAiYN9AX0/rWFpJ//3wBY7+NQ1p+qcTJSB80qIw4OTlBLpcjJyenzvacnBy4urrW62colUoEBgYiJSXlnvuYm5vDzs6uzouIiAybIAh48wlfvDOoC8wVMhxNycOALw9j6aErfPqviWtQGTEzM0NQUBCioqJ027RaLaKiohAeHl6vn1FTU4OzZ8/Czc2tYUmJiMjgCYKAyQ+1x745DyOigyPKq7T45JdkPL3wGC7nFEsdjyTS4Ms0c+fOxYoVK7B27VpcuHABL730EkpLSzFhwgQAwLhx4zBv3jzd/v/617/w66+/IjU1FSdPnsSYMWOQlpaGyZMnN91vQUREBsXLyRobJofi0+d6QGWpxIVsDZ5dEo3Y1Hypo5EEFA09YMSIEbhx4wbee+89qNVqBAQEYO/evbpJrenp6ZDJ/uw4N2/exJQpU6BWq9GqVSsEBQUhOjoaXbt2bbrfgoiIDI4gCBjeywP9Ojtj+voEJKTdxNhv4/DlCwF40o9nz02JIBrAdGaNRgOVSoWioiLOHyEiMkLlVTV4ZdMp/Ho+B4IAzH+qK178Y30SMlz1/f7ms2mIiEhyFko5lowJwtiwdhBF4P2fzmPBngvQavX+38vUBFhGiIhIL8hlAv41pBteH9AZALDscCpe/T4RFdU1Eiej5sYyQkREekMQBMx8pCM+f94fCpmAnYlZeHHVCRTdqpI6GjUjlhEiItI7z/Rsi9UTgmFjrkBMaj6eXxqDrMJbUseiZsIyQkREeukhn9bYMi0MzrbmuJhTjGcWR+Ps9SKpY1EzYBkhIiK91c1dhe0ze8PH2QZqTTmGLj6G/+xNRnkV55EYE5YRIiLSa23sLfHD9Ag86eeKGq2Ixb9fwZNfH0H8tQKpo1ETYRkhIiK9p7JSYvHoICwd0xOtbc2ReqMUw5fF4P1d53iWxAiwjBARkcF4orsbfnu1L54LagtRBNZEX8OzS6KRnl8mdTR6ACwjRERkUFRWSvx3uD9WTwiGg7UZzmVp8NQ3RxB1IefvDya9xDJCREQG6ZHOztj9ch8EeNhDU16NSWvj8em+ZNRw1VaDwzJCREQGy93eEt9PC8eLEV4AgEUHr2DcqljcLK2UNhg1CMsIEREZNDOFDO8/3Q1fjwyElZkcx1LyMXjhUZzP0kgdjeqJZYSIiIzC0/7u2D6jN9o5WuH6zVt4Zskx7DqdJXUsqgeWESIiMhqdXW2xa2YfPNypNcqrtHhl0yks2HMBldVaqaPRfbCMEBGRUVFZKbH6xWC81K8DgNqn/z759RFEX8mTOBndC8sIEREZHblMwD+f8MXi0T3haG2GlNwSjFoRi1c2nUKuplzqePQ/WEaIiMhoPennhgOv9cPYsHYQBGDX6Sw8+tkhbIhNkzoa/QXLCBERGTWVlRIfDu2OXTP7wN/DHiUV1Xh7exLe33WOa5LoCZYRIiIyCX5tVdj+UgTeeKIzgNql5Keti0dpRbXEyYhlhIiITIZMJmBGv45YNKonzBQy/HYhF88vi0EO55FIimWEiIhMzqAebtg0JQyOfzzbZuiiYziQnANR5GUbKbCMEBGRSQpq1wrbZ/RGh9bWyC4qx8Q18Ri+NAaxqflSRzM5LCNERGSyPB2tsH1mb0zr2x7mChni025ixPLjGLcqDueyiqSOZzIE0QDOSWk0GqhUKhQVFcHOzk7qOEREZIRyNOX4OuoytpzIQLVWhFwmYHZ/H8zo1wEKOf/t3hj1/f5mGSEiIvqLtPxSfPJLMn5JUgMAenra48sRgfB0tJI4meGp7/c3qx4REdFftHO0xuLRPfHliADYmitwMr0QA786jK3xGZzg2kxYRoiIiP6HIAgYGtgGe2Y/hBAvB5RW1uD1H85gynfxUBfxNuCmxjJCRER0Dx4OVtg0NQyvD+gMpVzAbxdy8djnh7ApLp1nSZoQywgREdF9yGUCZj7SEbtffgj+HvYorqjGvG1nMWpFLNLyS6WOZxQ4gZWIiKiearQiVh+7iv/+ehHlVVqYyWUYGeKBGY90hIudhdTx9A7vpiEiImomafmleGdHEo5czgMAmClkGBPaDtP7tYezLUvJbSwjREREzSz6Sh6+2H8JJ67dBABYKGWY3rcDZvTrCDMFZ0KwjBAREbUAURRxNCUPn++/hFPphQCAzi62+L/neiDAw17SbFJjGSEiImpBoijipzPZ+GDXOeSXVkImABN7e+O1xzvD0kwudTxJcNEzIiKiFiQIAp72d8f+uX0xLLANtCKw8uhVRH5+CGuOXUVpRbXUEfUWz4wQERE1g4PJuXhr+1lk/7FImq2FAiNDPDE+wgtt7C0lTtcyeJmGiIhIYmWV1fgx4TpWHbuGq3m1a5LIZQKGBLjj1chO8HAw7ufdsIwQERHpCa1WxMGLuVh17CqOpeQDAMzkMowO88SsRzrC0cZc4oTNg2WEiIhID525Xoj/7L2Ioym1a5RYm8kx+aH2mNjbGyorpcTpmhbLCBERkR47ejkP/7c3GWcziwAAVmZyjAzxxMQ+3kYzp4RlhIiISM9ptSL2JGVj4YEUJKuLAQAKWe1dOZMfao+u7ob9nccyQkREZCBEUcShSzew7FAqYlLzddt7tWuFseHtMLC7m0Gu6MoyQkREZIBOZxRi+ZFU7EtSo1pb+xXtZGOOkSEeGBXqCTeV4VzCYRkhIiIyYDmacmyKS8fG2HTkFlcAqL0tOLKLM8aGeSGigyNkMkHilPfHMkJERGQEqmq0+PVcDr6LuYbYqwW67e2drDEyxBNDAt319knBLCNERERG5lJOMTYcT8OPJzNR8sfy8nKZgId8nPBsz7Z4rKsLLJT68xwclhEiIiIjVVJRjZ2Jmfgh4bruScEAYGuuwBPdXfF0gDvC2ztCIZd20ivLCBERkQlIvVGC7acyse1kJjILb+m2O9mYYZCfG57yd0dPz1aQSzC/hGWEiIjIhGi1Ik5cK8Cu01nYczYbN8uqdO85WpuhfxdnPN7VFX18nFrsUg7LCBERkYmqqtHiaEoefkrMwv4LOSgur9a9Z6mUo3dHJzzq64xHfZ3hqmq+ya8sI0RERISqGi3irhbg13Nq7D+fg6yi8jrvd3Wzw6O+zni+lwc8HZv2KcIsI0RERFSHKIo4l6XBweRcHLiYi8SMQtxuARunhCKig1OTfl59v78VTfqpREREpLcEQUD3Nip0b6PCy/19kF9SgUOXbuDo5TwEezlIl4tnRoiIiKg51Pf72/CeukNERERGhWWEiIiIJMUyQkRERJJiGSEiIiJJsYwQERGRpFhGiIiISFIsI0RERCQplhEiIiKSVKPKyKJFi+Dl5QULCwuEhoYiLi7uvvtv3boVvr6+sLCwgJ+fH/bs2dOosERERGR8GlxGtmzZgrlz52L+/Pk4efIk/P39MWDAAOTm5t51/+joaIwcORKTJk3CqVOnMHToUAwdOhRJSUkPHJ6IiIgMX4OXgw8NDUVwcDAWLlwIANBqtfDw8MDLL7+MN9988479R4wYgdLSUuzevVu3LSwsDAEBAVi6dGm9PpPLwRMRERmeZlkOvrKyEgkJCYiMjPzzB8hkiIyMRExMzF2PiYmJqbM/AAwYMOCe+xMREZFpadBTe/Py8lBTUwMXF5c6211cXJCcnHzXY9Rq9V33V6vV9/yciooKVFRU6P6s0WgaEpOIiIgMSIPKSEtZsGABPvjggzu2s5QQEREZjtvf2383I6RBZcTJyQlyuRw5OTl1tufk5MDV1fWux7i6ujZofwCYN28e5s6dq/tzZmYmunbtCg8Pj4bEJSIiIj1QXFwMlUp1z/cbVEbMzMwQFBSEqKgoDB06FEDtBNaoqCjMmjXrrseEh4cjKioKc+bM0W3bv38/wsPD7/k55ubmMDc31/3ZxsYGGRkZsLW1hSAIDYl8XxqNBh4eHsjIyODE2GbGsW45HOuWxfFuORzrltNUYy2KIoqLi+Hu7n7f/Rp8mWbu3LkYP348evXqhZCQEHz55ZcoLS3FhAkTAADjxo1DmzZtsGDBAgDA7Nmz0bdvX3z22WcYNGgQNm/ejPj4eCxfvrzenymTydC2bduGRq03Ozs7/g+7hXCsWw7HumVxvFsOx7rlNMVY3++MyG0NLiMjRozAjRs38N5770GtViMgIAB79+7VTVJNT0+HTPbnTToRERHYuHEj3nnnHbz11lvw8fHBjh070L1794Z+NBERERmhBq8zYky4fknL4Vi3HI51y+J4txyOdctp6bE26WfTmJubY/78+XXmp1Dz4Fi3HI51y+J4txyOdctp6bE26TMjREREJD2TPjNCRERE0mMZISIiIkmxjBAREZGkWEaIiIhIUiZdRhYtWgQvLy9YWFggNDQUcXFxUkcyeAsWLEBwcDBsbW3h7OyMoUOH4uLFi3X2KS8vx8yZM+Ho6AgbGxs8++yzdzwygBrmk08+gSAIdVY65jg3rczMTIwZMwaOjo6wtLSEn58f4uPjde+Looj33nsPbm5usLS0RGRkJC5fvixhYsNUU1ODd999F97e3rC0tESHDh3w4Ycf1nm2Cce6cQ4fPozBgwfD3d0dgiBgx44ddd6vz7gWFBRg9OjRsLOzg729PSZNmoSSkpIHDyeaqM2bN4tmZmbiqlWrxHPnzolTpkwR7e3txZycHKmjGbQBAwaIq1evFpOSksTExETxySefFD09PcWSkhLdPtOnTxc9PDzEqKgoMT4+XgwLCxMjIiIkTG3Y4uLiRC8vL7FHjx7i7Nmzdds5zk2noKBAbNeunfjiiy+KsbGxYmpqqrhv3z4xJSVFt88nn3wiqlQqcceOHeLp06fFp59+WvT29hZv3bolYXLD89FHH4mOjo7i7t27xatXr4pbt24VbWxsxK+++kq3D8e6cfbs2SO+/fbb4rZt20QA4vbt2+u8X59xfeKJJ0R/f3/x+PHj4pEjR8SOHTuKI0eOfOBsJltGQkJCxJkzZ+r+XFNTI7q7u4sLFiyQMJXxyc3NFQGIhw4dEkVRFAsLC0WlUilu3bpVt8+FCxdEAGJMTIxUMQ1WcXGx6OPjI+7fv1/s27evroxwnJvWP//5T7FPnz73fF+r1Yqurq7ip59+qttWWFgompubi5s2bWqJiEZj0KBB4sSJE+tse+aZZ8TRo0eLosixbir/W0bqM67nz58XAYgnTpzQ7fPLL7+IgiCImZmZD5THJC/TVFZWIiEhAZGRkbptMpkMkZGRiImJkTCZ8SkqKgIAODg4AAASEhJQVVVVZ+x9fX3h6enJsW+EmTNnYtCgQXXGE+A4N7Vdu3ahV69eGD58OJydnREYGIgVK1bo3r969SrUanWd8VapVAgNDeV4N1BERASioqJw6dIlAMDp06dx9OhRDBw4EADHurnUZ1xjYmJgb2+PXr166faJjIyETCZDbGzsA31+g59NYwzy8vJQU1Oje57ObS4uLkhOTpYolfHRarWYM2cOevfurXsWkVqthpmZGezt7evs6+LiArVaLUFKw7V582acPHkSJ06cuOM9jnPTSk1NxZIlSzB37ly89dZbOHHiBF555RWYmZlh/PjxujG9298pHO+GefPNN6HRaODr6wu5XI6amhp89NFHGD16NABwrJtJfcZVrVbD2dm5zvsKhQIODg4PPPYmWUaoZcycORNJSUk4evSo1FGMTkZGBmbPno39+/fDwsJC6jhGT6vVolevXvj4448BAIGBgUhKSsLSpUsxfvx4idMZl++//x4bNmzAxo0b0a1bNyQmJmLOnDlwd3fnWBsxk7xM4+TkBLlcfsedBTk5OXB1dZUolXGZNWsWdu/ejYMHD6Jt27a67a6urqisrERhYWGd/Tn2DZOQkIDc3Fz07NkTCoUCCoUChw4dwtdffw2FQgEXFxeOcxNyc3ND165d62zr0qUL0tPTAUA3pvw75cG9/vrrePPNN/HCCy/Az88PY8eOxauvvooFCxYA4Fg3l/qMq6urK3Jzc+u8X11djYKCggcee5MsI2ZmZggKCkJUVJRum1arRVRUFMLDwyVMZvhEUcSsWbOwfft2HDhwAN7e3nXeDwoKglKprDP2Fy9eRHp6Ose+Afr374+zZ88iMTFR9+rVqxdGjx6t+2+Oc9Pp3bv3HbeoX7p0Ce3atQMAeHt7w9XVtc54azQaxMbGcrwbqKysDDJZ3a8muVwOrVYLgGPdXOozruHh4SgsLERCQoJunwMHDkCr1SI0NPTBAjzQ9FcDtnnzZtHc3Fxcs2aNeP78eXHq1Kmivb29qFarpY5m0F566SVRpVKJv//+u5idna17lZWV6faZPn266OnpKR44cECMj48Xw8PDxfDwcAlTG4e/3k0jihznphQXFycqFArxo48+Ei9fvixu2LBBtLKyEtevX6/b55NPPhHt7e3FnTt3imfOnBGHDBnC200bYfz48WKbNm10t/Zu27ZNdHJyEt944w3dPhzrxikuLhZPnTolnjp1SgQgfv755+KpU6fEtLQ0URTrN65PPPGEGBgYKMbGxopHjx4VfXx8eGvvg/rmm29ET09P0czMTAwJCRGPHz8udSSDB+Cur9WrV+v2uXXrljhjxgyxVatWopWVlThs2DAxOztbutBG4n/LCMe5af30009i9+7dRXNzc9HX11dcvnx5nfe1Wq347rvvii4uLqK5ubnYv39/8eLFixKlNVwajUacPXu26OnpKVpYWIjt27cX3377bbGiokK3D8e6cQ4ePHjXv5/Hjx8vimL9xjU/P18cOXKkaGNjI9rZ2YkTJkwQi4uLHzibIIp/WdaOiIiIqIWZ5JwRIiIi0h8sI0RERCQplhEiIiKSFMsIERERSYplhIiIiCTFMkJERESSYhkhIiIiSbGMEBERkaRYRoiIiEhSLCNEREQkKZYRIiIikhTLCBEREUnq/wGJU24OMCRRYgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "losses = []\n",
        "for n in range(100):\n",
        "    print('.', end='')\n",
        "    logs = translator.train_step([example_input_batch, example_target_batch])\n",
        "    losses.append(logs['batch_loss'].numpy())\n",
        "\n",
        "print()\n",
        "plt.plot(losses)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 466
        },
        "id": "lhilpydRD7Tq",
        "outputId": "48746bdd-95a3-4a01-87e4-43c2fde6c0a5"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "outputs": [],
      "source": [
        "train_translator = TrainTranslator(\n",
        "    embedding_dim, units,\n",
        "    input_text_processor=input_text_processor,\n",
        "    output_text_processor=output_text_processor\n",
        ")\n",
        "\n",
        "# Configure the loss and optimizer\n",
        "train_translator.compile(\n",
        "    optimizer=tf.optimizers.Adam(),\n",
        "    loss=MaskedLoss(),\n",
        ")"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "_h_LtvXCD7Tq"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "outputs": [],
      "source": [
        "class BatchLogs(tf.keras.callbacks.Callback):\n",
        "    def __init__(self, key):\n",
        "        self.key = key\n",
        "        self.logs = []\n",
        "\n",
        "    def on_train_batch_end(self, n, logs):\n",
        "        self.logs.append(logs[self.key])\n",
        "\n",
        "\n",
        "batch_loss = BatchLogs('batch_loss')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "pvMIzD-dD7Tr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/3\n",
            "  48/1025 [>.............................] - ETA: 1:14:51 - batch_loss: 2.6575"
          ]
        }
      ],
      "source": [
        "train_translator.fit(\n",
        "    dataset,\n",
        "    epochs=3,\n",
        "    callbacks=[batch_loss]\n",
        ")"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5lc-3jXDD7Tr",
        "outputId": "83da9290-ae8e-45c3-97b5-1f7f0474ca8d"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "plt.plot(batch_loss.logs)\n",
        "plt.ylim([0, 3])\n",
        "plt.xlabel('Batch #')\n",
        "plt.ylabel('CE/token')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "_BXR9R1iD7Tr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "class Translator(tf.Module):\n",
        "\n",
        "    def __init__(self, encoder, decoder, input_text_processor, output_text_processor):\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.input_text_processor = input_text_processor\n",
        "        self.output_text_processor = output_text_processor\n",
        "\n",
        "        self.output_token_string_from_index = (\n",
        "            tf.keras.layers.StringLookup(\n",
        "                vocabulary=output_text_processor.get_vocabulary(),\n",
        "                mask_token='',\n",
        "                invert=True\n",
        "            )\n",
        "        )\n",
        "\n",
        "        # The output should never generate padding, unknown, or start.\n",
        "        index_from_string = tf.keras.layers.StringLookup(\n",
        "            vocabulary=output_text_processor.get_vocabulary(),\n",
        "            mask_token=''\n",
        "        )\n",
        "        token_mask_ids = index_from_string(['', '[UNK]', '[START]']).numpy()\n",
        "\n",
        "        token_mask = np.zeros([index_from_string.vocabulary_size()], dtype=np.bool)\n",
        "        token_mask[np.array(token_mask_ids)] = True\n",
        "        self.token_mask = token_mask\n",
        "\n",
        "        self.start_token = index_from_string(tf.constant('[START]'))\n",
        "        self.end_token = index_from_string(tf.constant('[END]'))\n"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "wK701oOoD7Tr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "translator = Translator(\n",
        "    encoder=train_translator.encoder,\n",
        "    decoder=train_translator.decoder,\n",
        "    input_text_processor=input_text_processor,\n",
        "    output_text_processor=output_text_processor,\n",
        ")"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "FBQUtfpED7Tr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "def tokens_to_text(self, result_tokens):\n",
        "    shape_checker = ShapeChecker()\n",
        "    shape_checker(result_tokens, ('batch', 't'))\n",
        "    result_text_tokens = self.output_token_string_from_index(result_tokens)\n",
        "    shape_checker(result_text_tokens, ('batch', 't'))\n",
        "\n",
        "    result_text = tf.strings.reduce_join(\n",
        "        result_text_tokens,\n",
        "        axis=1,\n",
        "        separator=' '\n",
        "    )\n",
        "    shape_checker(result_text, ('batch',))\n",
        "\n",
        "    result_text = tf.strings.strip(result_text)\n",
        "    shape_checker(result_text, ('batch',))\n",
        "    return result_text"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "RMDTIdOkD7Tr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "Translator.tokens_to_text = tokens_to_text"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "gdqGMq8zD7Ts"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "example_output_tokens = tf.random.uniform(\n",
        "    shape=[5, 2],\n",
        "    minval=0,\n",
        "    dtype=tf.int64,\n",
        "    maxval=output_text_processor.vocabulary_size()\n",
        ")\n",
        "translator.tokens_to_text(example_output_tokens).numpy()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "u57ILZr6D7Ts"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "def sample(self, logits, temperature):\n",
        "    shape_checker = ShapeChecker()\n",
        "    # 't' is usually 1 here.\n",
        "    shape_checker(logits, ('batch', 't', 'vocab'))\n",
        "    shape_checker(self.token_mask, ('vocab',))\n",
        "\n",
        "    token_mask = self.token_mask[tf.newaxis, tf.newaxis, :]\n",
        "    shape_checker(token_mask, ('batch', 't', 'vocab'), broadcast=True)\n",
        "\n",
        "    # Set the logits for all masked tokens to -inf, so they are never chosen.\n",
        "    logits = tf.where(self.token_mask, -np.inf, logits)\n",
        "\n",
        "    if temperature == 0.0:\n",
        "        new_tokens = tf.argmax(logits, axis=-1)\n",
        "    else:\n",
        "        logits = tf.squeeze(logits, axis=1)\n",
        "        new_tokens = tf.random.categorical(logits / temperature,\n",
        "                                           num_samples=1)\n",
        "\n",
        "    shape_checker(new_tokens, ('batch', 't'))\n",
        "\n",
        "    return new_tokens\n"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "OebIk9NhD7Ts"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "Translator.sample = sample"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "cxT0-H8gD7Ts"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "example_logits = tf.random.normal([5, 1, output_text_processor.vocabulary_size()])\n",
        "example_output_tokens = translator.sample(example_logits, temperature=1.0)\n",
        "example_output_tokens"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "AtnZaQdMD7Ts"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "def translate_unrolled(\n",
        "        self,\n",
        "        input_text,\n",
        "        *,\n",
        "        max_length=50,\n",
        "        return_attention=True,\n",
        "        temperature=1.0\n",
        "):\n",
        "    batch_size = tf.shape(input_text)[0]\n",
        "    input_tokens = self.input_text_processor(input_text)\n",
        "    enc_output, enc_state = self.encoder(input_tokens)\n",
        "\n",
        "    dec_state = enc_state\n",
        "    new_tokens = tf.fill([batch_size, 1], self.start_token)\n",
        "\n",
        "    result_tokens = []\n",
        "    attention = []\n",
        "    done = tf.zeros([batch_size, 1], dtype=tf.bool)\n",
        "\n",
        "    for _ in range(max_length):\n",
        "        dec_input = DecoderInput(\n",
        "            new_tokens=new_tokens,\n",
        "            enc_output=enc_output,\n",
        "            mask=(input_tokens != 0)\n",
        "        )\n",
        "\n",
        "        dec_result, dec_state = self.decoder(dec_input, state=dec_state)\n",
        "\n",
        "        attention.append(dec_result.attention_weights)\n",
        "\n",
        "        new_tokens = self.sample(dec_result.logits, temperature)\n",
        "\n",
        "        # If a sequence produces an `end_token`, set it `done`\n",
        "        done = done | (new_tokens == self.end_token)\n",
        "        # Once a sequence is done it only produces 0-padding.\n",
        "        new_tokens = tf.where(done, tf.constant(0, dtype=tf.int64), new_tokens)\n",
        "\n",
        "        # Collect the generated tokens\n",
        "        result_tokens.append(new_tokens)\n",
        "\n",
        "        if tf.executing_eagerly() and tf.reduce_all(done):\n",
        "            break\n",
        "\n",
        "    # Convert the list of generates token ids to a list of strings.\n",
        "    result_tokens = tf.concat(result_tokens, axis=-1)\n",
        "    result_text = self.tokens_to_text(result_tokens)\n",
        "\n",
        "    if return_attention:\n",
        "        attention_stack = tf.concat(attention, axis=1)\n",
        "        return {'text': result_text, 'attention': attention_stack}\n",
        "    else:\n",
        "        return {'text': result_text}"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "DzUXvmLED7Ts"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "Translator.translate = translate_unrolled"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "mOMJdvdSD7Ts"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%time\n",
        "input_text = tf.constant([\n",
        "    'm o t t o',  # mɒtəʊ\n",
        "    'u n t i d y',  # ʌntaɪdi\n",
        "    'r a b b i t', # rabbit\n",
        "    'a b e r c r o m b i e',\n",
        "    's u m i n s k i'\n",
        "])\n",
        "\n",
        "result = translator.translate(input_text=input_text)\n",
        "\n",
        "print(\"\".join(result['text'][0].numpy().decode().split(' ')))\n",
        "print(\"\".join(result['text'][1].numpy().decode().split(' ')))\n",
        "print(\"\".join(result['text'][2].numpy().decode().split(' ')))\n",
        "print(\"\".join(result['text'][3].numpy().decode().split(' ')))\n",
        "print(\"\".join(result['text'][4].numpy().decode().split(' ')))\n",
        "print()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n",
          "is_executing": true
        },
        "id": "BmwALC3iD7Ts"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "@tf.function(input_signature=[tf.TensorSpec(dtype=tf.string, shape=[None])])\n",
        "def tf_translate(self, input_text):\n",
        "    return self.translate(input_text)\n",
        "\n",
        "\n",
        "Translator.tf_translate = tf_translate"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "s85E4kCXD7Tt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%time\n",
        "result = translator.tf_translate(input_text=input_text)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "_gYfkfl5D7Tt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%time\n",
        "result = translator.tf_translate(input_text=input_text)\n",
        "print(input_text)\n",
        "\n",
        "print(\"\".join(result['text'][0].numpy().decode().split(' ')))\n",
        "print(\"\".join(result['text'][1].numpy().decode().split(' ')))\n",
        "print(\"\".join(result['text'][2].numpy().decode().split(' ')))\n",
        "print(\"\".join(result['text'][3].numpy().decode().split(' ')))\n",
        "print()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "QbHGdQkpD7Tt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "def translate_symbolic(self,\n",
        "                       input_text,\n",
        "                       *,\n",
        "                       max_length=50,\n",
        "                       return_attention=True,\n",
        "                       temperature=1.0):\n",
        "  shape_checker = ShapeChecker()\n",
        "  shape_checker(input_text, ('batch',))\n",
        "\n",
        "  batch_size = tf.shape(input_text)[0]\n",
        "\n",
        "  # Encode the input\n",
        "  input_tokens = self.input_text_processor(input_text)\n",
        "  shape_checker(input_tokens, ('batch', 's'))\n",
        "\n",
        "  enc_output, enc_state = self.encoder(input_tokens)\n",
        "  shape_checker(enc_output, ('batch', 's', 'enc_units'))\n",
        "  shape_checker(enc_state, ('batch', 'enc_units'))\n",
        "\n",
        "  # Initialize the decoder\n",
        "  dec_state = enc_state\n",
        "  new_tokens = tf.fill([batch_size, 1], self.start_token)\n",
        "  shape_checker(new_tokens, ('batch', 't1'))\n",
        "\n",
        "  # Initialize the accumulators\n",
        "  result_tokens = tf.TensorArray(tf.int64, size=1, dynamic_size=True)\n",
        "  attention = tf.TensorArray(tf.float32, size=1, dynamic_size=True)\n",
        "  done = tf.zeros([batch_size, 1], dtype=tf.bool)\n",
        "  shape_checker(done, ('batch', 't1'))\n",
        "\n",
        "  for t in tf.range(max_length):\n",
        "    dec_input = DecoderInput(\n",
        "        new_tokens=new_tokens, enc_output=enc_output, mask=(input_tokens != 0))\n",
        "\n",
        "    dec_result, dec_state = self.decoder(dec_input, state=dec_state)\n",
        "\n",
        "    shape_checker(dec_result.attention_weights, ('batch', 't1', 's'))\n",
        "    attention = attention.write(t, dec_result.attention_weights)\n",
        "\n",
        "    new_tokens = self.sample(dec_result.logits, temperature)\n",
        "    shape_checker(dec_result.logits, ('batch', 't1', 'vocab'))\n",
        "    shape_checker(new_tokens, ('batch', 't1'))\n",
        "\n",
        "    # If a sequence produces an `end_token`, set it `done`\n",
        "    done = done | (new_tokens == self.end_token)\n",
        "    # Once a sequence is done it only produces 0-padding.\n",
        "    new_tokens = tf.where(done, tf.constant(0, dtype=tf.int64), new_tokens)\n",
        "\n",
        "    # Collect the generated tokens\n",
        "    result_tokens = result_tokens.write(t, new_tokens)\n",
        "\n",
        "    if tf.reduce_all(done):\n",
        "      break\n",
        "\n",
        "  # Convert the list of generated token ids to a list of strings.\n",
        "  result_tokens = result_tokens.stack()\n",
        "  shape_checker(result_tokens, ('t', 'batch', 't0'))\n",
        "  result_tokens = tf.squeeze(result_tokens, -1)\n",
        "  result_tokens = tf.transpose(result_tokens, [1, 0])\n",
        "  shape_checker(result_tokens, ('batch', 't'))\n",
        "\n",
        "  result_text = self.tokens_to_text(result_tokens)\n",
        "  shape_checker(result_text, ('batch',))\n",
        "\n",
        "  if return_attention:\n",
        "    attention_stack = attention.stack()\n",
        "    shape_checker(attention_stack, ('t', 'batch', 't1', 's'))\n",
        "\n",
        "    attention_stack = tf.squeeze(attention_stack, 2)\n",
        "    shape_checker(attention_stack, ('t', 'batch', 's'))\n",
        "\n",
        "    attention_stack = tf.transpose(attention_stack, [1, 0, 2])\n",
        "    shape_checker(attention_stack, ('batch', 't', 's'))\n",
        "\n",
        "    return {'text': result_text, 'attention': attention_stack}\n",
        "  else:\n",
        "    return {'text': result_text}"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "gQ2sZaGAD7Tt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "Translator.translate = translate_symbolic\n"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "363-5qz3D7Tt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%time\n",
        "result = translator.translate(\n",
        "    input_text = input_text)\n",
        "\n",
        "print(result['text'][0].numpy().decode())\n",
        "print(result['text'][1].numpy().decode())\n",
        "print()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "-tS5iPADD7Tt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "@tf.function(input_signature=[tf.TensorSpec(dtype=tf.string, shape=[None])])\n",
        "def tf_translate(self, input_text):\n",
        "  return self.translate(input_text)\n",
        "\n",
        "Translator.tf_translate = tf_translate"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "JuL02PsxD7Tu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%time\n",
        "result = translator.tf_translate(\n",
        "    input_text = input_text)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "7sYTM3DUD7Tu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%time\n",
        "result = translator.tf_translate(\n",
        "    input_text = input_text)\n",
        "\n",
        "print(result['text'][0].numpy().decode())\n",
        "print(result['text'][1].numpy().decode())\n",
        "print()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "8Qz50qHlD7Tu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "a = result['attention'][0]\n",
        "\n",
        "print(np.sum(a, axis=-1))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "S7KLMUh7D7Tu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "_ = plt.bar(range(len(a[0, :])), a[0, :])\n"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "SrOPr38XD7Tu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "plt.imshow(np.array(a), vmin=0.0)\n"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "-LLI6e14D7Tu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "def plot_attention(attention, sentence, predicted_sentence):\n",
        "  sentence = tf_lower_and_split_punct(sentence).numpy().decode().split()\n",
        "  predicted_sentence = predicted_sentence.numpy().decode().split() + ['[END]']\n",
        "  fig = plt.figure(figsize=(10, 10))\n",
        "  ax = fig.add_subplot(1, 1, 1)\n",
        "\n",
        "  attention = attention[:len(predicted_sentence), :len(sentence)]\n",
        "\n",
        "  ax.matshow(attention, cmap='viridis', vmin=0.0)\n",
        "\n",
        "  fontdict = {'fontsize': 14}\n",
        "\n",
        "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
        "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
        "\n",
        "  ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "  ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "\n",
        "  ax.set_xlabel('Input text')\n",
        "  ax.set_ylabel('Output text')\n",
        "  plt.suptitle('Attention weights')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "zuIzzcIuD7Tu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "i=0\n",
        "plot_attention(result['attention'][i], input_text[i], result['text'][i])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "BiXvonBtD7Tv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%time\n",
        "three_input_text = tf.constant([\n",
        "    # This is my life.\n",
        "    'e x t r a v a g a n t',\n",
        "    # Are they still home?\n",
        "    'o l i v e',\n",
        "    # Try to find out.'\n",
        "    'm e n t i o n',\n",
        "])\n",
        "\n",
        "result = translator.tf_translate(three_input_text)\n",
        "\n",
        "for tr in result['text']:\n",
        "  print(tr.numpy().decode())\n",
        "\n",
        "print()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "b_lLDOGkD7Tv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "result['text']\n"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "no-N2FE-D7Tv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "i = 0\n",
        "plot_attention(result['attention'][i], three_input_text[i], result['text'][i])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "fCF6KODhD7Tv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "i = 1\n",
        "plot_attention(result['attention'][i], three_input_text[i], result['text'][i])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "1QhVSypND7Tv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "i = 2\n",
        "\n",
        "plot_attention(result['attention'][i], three_input_text[i], result['text'][i])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "qYnLIJ3HD7Tv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "long_str = 'p n e u m o n o u l t r a m i c r o s c o p i c s i l i c o v o l c a n o c o n i o s i s'\n",
        "long_input_text = tf.constant([long_str])\n",
        "\n",
        "import textwrap\n",
        "print('Expected output:\\n', ''.join(long_str.split(' ')))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "26H4udrOD7Tv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "result = translator.tf_translate(long_input_text)\n",
        "\n",
        "i = 0\n",
        "plot_attention(result['attention'][i], long_input_text[i], result['text'][i])\n",
        "_ = plt.suptitle('This never works')"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "A7jd0uWLD7Tw"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "# tf.saved_model.save(translator, 'ipa_translator',\n",
        "#                     signatures={'serving_default': translator.tf_translate})"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "zdaP0MbTD7Tw"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "# reloaded = tf.saved_model.load('ipa_translator')\n",
        "# result = reloaded.tf_translate(three_input_text)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "SJuNxICOD7Tw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tf.saved_model.save(translator, '/content/drive/MyDrive/eng-to-ipa/ipa_translator',\n",
        "                    signatures={'serving_default': translator.tf_translate})"
      ],
      "metadata": {
        "id": "leM332OPEi98"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "reloaded = tf.saved_model.load('/content/drive/MyDrive/eng-to-ipa/ipa_translator')\n",
        "result = reloaded.tf_translate(three_input_text)"
      ],
      "metadata": {
        "id": "blZaIt5HEvtE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%time\n",
        "result = reloaded.tf_translate(three_input_text)\n",
        "\n",
        "for tr in result['text']:\n",
        "  print(''.join(tr.numpy().decode().split(' ')))\n",
        "\n",
        "print()"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "UN9dacO1D7Tw"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "apTAc5Z2D7Tw"
      }
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 2
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython2",
      "version": "2.7.6"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}