ClassificaçãoDeSentimento_BERT(Embedding).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ClassificaçãoDeSentimento_BERT(Embedding).ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOVaBevEsGtV695QmxEN3pL"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "skIXkBCxjBrN"
      },
      "source": [
        "Nesse modelo utilizará a camada Embedding da Arquitetura BERT."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Muw02dm_it1F"
      },
      "source": [
        "###Importando as bibliotecas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y8B3FEiPidJo"
      },
      "source": [
        "import numpy as np\n",
        "import math\n",
        "import re\n",
        "import pandas as pd\n",
        "from bs4 import BeautifulSoup\n",
        "import random\n",
        "from google.colab import drive"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t_J8I1e8jXjK",
        "outputId": "80991275-2e0c-4ff0-b03e-58671ef6f4a6"
      },
      "source": [
        "!pip install bert-for-tf2\n",
        "!pip install sentencepiece"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: bert-for-tf2 in /usr/local/lib/python3.7/dist-packages (0.14.9)\n",
            "Requirement already satisfied: py-params>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from bert-for-tf2) (0.10.2)\n",
            "Requirement already satisfied: params-flow>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from bert-for-tf2) (0.8.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (1.19.5)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (4.41.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.96)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "akjJVW4ajk8i",
        "outputId": "6130ab0f-aed1-40e4-e68e-477c6229a5b5"
      },
      "source": [
        "!pip install tensorflow==2.2.0-rc3"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow==2.2.0-rc3 in /usr/local/lib/python3.7/dist-packages (2.2.0rc3)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (0.2.0)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (2.10.0)\n",
            "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (1.19.5)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (3.3.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (1.34.1)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (3.17.3)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (1.6.3)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (1.12.1)\n",
            "Requirement already satisfied: tensorboard<2.3.0,>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (2.2.2)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (0.36.2)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (1.15.0)\n",
            "Requirement already satisfied: scipy==1.4.1; python_version >= \"3\" in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (1.4.1)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (0.3.3)\n",
            "Requirement already satisfied: tensorflow-estimator<2.3.0,>=2.2.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (2.2.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (0.12.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (1.1.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0-rc3) (1.1.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (3.3.4)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (57.0.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (1.0.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (2.23.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (1.32.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (1.8.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (0.4.4)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (4.6.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (2021.5.30)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (4.2.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (1.3.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (3.4.1)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.7/dist-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0-rc3) (3.1.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "E8Jj20ZFm47x",
        "outputId": "2a64e110-c9c1-4962-acf9-32c4ed2f799d"
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "from tensorflow.keras import layers\n",
        "import bert\n",
        "tf.__version__\n",
        "# !pip install tensorflow==2.2.0-rc3"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'2.2.0-rc3'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2mW2ZHZkjfNh"
      },
      "source": [
        "###Carregamento dos Arquivos"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NSwYDa3DjqXJ",
        "outputId": "71a3b598-9edc-4812-94fe-6dd7bee7f868"
      },
      "source": [
        "drive.mount(\"/content/drive\")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BjnDZ8bKjswZ"
      },
      "source": [
        "cols = ['sentiment', 'id', 'date', 'query', 'user', 'text']"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KSOwSX8FkEgR"
      },
      "source": [
        "data = pd.read_csv('/content/drive/MyDrive/PLN/Classificação de Sentimento com BERT/training.1600000.processed.noemoticon.csv',\n",
        "                   header = None,\n",
        "                   names = cols,\n",
        "                   engine='python',\n",
        "                   encoding='latin1')"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S9fQsmNjkuy-",
        "outputId": "7dfb20f6-28cf-46b5-da5a-3dad69fadc57"
      },
      "source": [
        "data.shape"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1600000, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "NQ2NUmI2kbcg",
        "outputId": "ffddae9f-e577-4431-b8ce-ba22e8b99973"
      },
      "source": [
        "data.head()\n"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "      <th>id</th>\n",
              "      <th>date</th>\n",
              "      <th>query</th>\n",
              "      <th>user</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>1467810369</td>\n",
              "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>_TheSpecialOne_</td>\n",
              "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>1467810672</td>\n",
              "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>scotthamilton</td>\n",
              "      <td>is upset that he can't update his Facebook by ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>1467810917</td>\n",
              "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>mattycus</td>\n",
              "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>1467811184</td>\n",
              "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>ElleCTF</td>\n",
              "      <td>my whole body feels itchy and like its on fire</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1467811193</td>\n",
              "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>Karoli</td>\n",
              "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   sentiment  ...                                               text\n",
              "0          0  ...  @switchfoot http://twitpic.com/2y1zl - Awww, t...\n",
              "1          0  ...  is upset that he can't update his Facebook by ...\n",
              "2          0  ...  @Kenichan I dived many times for the ball. Man...\n",
              "3          0  ...    my whole body feels itchy and like its on fire \n",
              "4          0  ...  @nationwideclass no, it's not behaving at all....\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h-j2mEQFkk8l"
      },
      "source": [
        "###Pré-processamento"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i1Cke6R3kp5W"
      },
      "source": [
        "#Limpeza\n",
        "def clean_tweet(tweet):\n",
        "    tweet = BeautifulSoup(tweet, \"lxml\").get_text()\n",
        "    tweet = re.sub(r\"@[A-Za-z0-9]+\", ' ', tweet)\n",
        "    tweet = re.sub(r\"https?://[A-Za-z0-9./]+\", ' ', tweet)\n",
        "    tweet = re.sub(r\"[^a-zA-Z.!?']\", ' ', tweet)\n",
        "    tweet = re.sub(r\" +\", ' ', tweet)\n",
        "    return tweet"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l4fpo6zPkz6G"
      },
      "source": [
        "data_clean = [clean_tweet(tweet) for tweet in data.text]"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I1NXV8mNk1nO"
      },
      "source": [
        "data_labels = data.sentiment.values\n",
        "data_labels[data_labels == 4] = 1"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_mBZhaQTk4s2"
      },
      "source": [
        "### Tokenização"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kaglql9Jk5ht"
      },
      "source": [
        "FullTokenizer = bert.bert_tokenization.FullTokenizer\n",
        "bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n",
        "                            trainable=False)\n",
        "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
        "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
        "tokenizer = FullTokenizer(vocab_file, do_lower_case)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ouq4cPWk94F"
      },
      "source": [
        "def encode_sentence(sent):\n",
        "  return [\"[CLS]\"] + tokenizer.tokenize(sent) + [\"[SEP]\"]"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ncoMVCj1k-nd",
        "outputId": "d7e4ddcc-aecb-4909-fcd5-5c2b115296de"
      },
      "source": [
        "encode_sentence(\"My dog likes strawberries.\")"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS]', 'my', 'dog', 'likes', 'straw', '##berries', '.', '[SEP]']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nE7P9SIolBe1"
      },
      "source": [
        "data_inputs = [encode_sentence(sentence) for sentence in data_clean]"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h5iLD6SylC8t",
        "outputId": "dcf9b6bc-8225-45e3-d930-091f96d8fd2a"
      },
      "source": [
        "print(data_inputs[0:2])"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[['[CLS]', 'aw', '##w', '##w', 'that', \"'\", 's', 'a', 'bum', '##mer', '.', 'you', 'should', '##a', 'got', 'david', 'carr', 'of', 'third', 'day', 'to', 'do', 'it', '.', 'd', '[SEP]'], ['[CLS]', 'is', 'upset', 'that', 'he', 'can', \"'\", 't', 'update', 'his', 'facebook', 'by', 'text', '##ing', 'it', '.', '.', '.', 'and', 'might', 'cry', 'as', 'a', 'result', 'school', 'today', 'also', '.', 'blah', '!', '[SEP]']]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAMbtvo2lOfb"
      },
      "source": [
        "### Criação da base de dados"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nxg23UJMlOFL"
      },
      "source": [
        "def get_ids(tokens):\n",
        "  return tokenizer.convert_tokens_to_ids(tokens)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j1Lyw0WcrjdA",
        "outputId": "47a13afc-28d6-4870-be60-d1f835f4bbeb"
      },
      "source": [
        "get_ids(tokenizer.tokenize(\"my cat like grapes.\"))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2026, 4937, 2066, 16575, 1012]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "45HUrlMOrxce"
      },
      "source": [
        "def get_mask(tokens):\n",
        "  return np.char.not_equal(tokens, \"(PAD)\").astype(int)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hIysyAJ_rxXG",
        "outputId": "dc191c1a-d8c4-40b5-f288-7920426673c8"
      },
      "source": [
        "get_mask(tokenizer.tokenize(\"my cat like grapes.\"))"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 1, 1, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qqZrc9B_g823"
      },
      "source": [
        "def get_segments(tokens):\n",
        "  seg_ids = []\n",
        "  current_seg_id = 0\n",
        "  for tok in tokens:\n",
        "    seg_ids.append(current_seg_id)\n",
        "    if tok == \"[SEP]\":\n",
        "      current_seg_id = 1 - current_seg_id\n",
        "  return seg_ids"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "txFlES6dhQz5",
        "outputId": "c83ba358-a4dc-4f91-968f-e8080e17059d"
      },
      "source": [
        "print(data_inputs[0])"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['[CLS]', 'aw', '##w', '##w', 'that', \"'\", 's', 'a', 'bum', '##mer', '.', 'you', 'should', '##a', 'got', 'david', 'carr', 'of', 'third', 'day', 'to', 'do', 'it', '.', 'd', '[SEP]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fj51v7R3hiXU",
        "outputId": "9d8216ff-f2ba-4702-f1fb-c9be78df0c18"
      },
      "source": [
        "get_segments(data_inputs[0])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XVv_ecN0yPMY",
        "outputId": "715903ef-a3fb-4ae2-e7ee-1e211cee9030"
      },
      "source": [
        "my_sent = [\"[CLS]\"]+ tokenizer.tokenize(\"Roses are beautiful\") +[\"[SEP]\"]\n",
        "my_sent"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS]', 'roses', 'are', 'beautiful', '[SEP]']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I-B8PWuL4fZf",
        "outputId": "3b5d2b76-b4b6-4400-c902-a795c32abac9"
      },
      "source": [
        "bert_layer([\n",
        "            tf.expand_dims(tf.cast(get_ids(my_sent), tf.int32),0),\n",
        "            tf.expand_dims(tf.cast(get_mask(my_sent), tf.int32),0),\n",
        "            tf.expand_dims(tf.cast(get_segments(my_sent), tf.int32),0)\n",
        "          ])"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<tf.Tensor: shape=(1, 768), dtype=float32, numpy=\n",
              " array([[-9.01942849e-01, -3.59286964e-01, -4.81667012e-01,\n",
              "          7.72407770e-01,  3.03145021e-01, -1.53753683e-01,\n",
              "          9.09623086e-01,  2.33455285e-01, -3.11014801e-01,\n",
              "         -9.99968350e-01, -2.53063262e-01,  7.49138534e-01,\n",
              "          9.86961246e-01,  5.14592677e-02,  9.56657887e-01,\n",
              "         -5.21562576e-01,  6.44484833e-02, -6.17905974e-01,\n",
              "          3.85748565e-01, -6.78136349e-01,  7.31310010e-01,\n",
              "          9.98348594e-01,  3.04465950e-01,  3.01394522e-01,\n",
              "          4.86608267e-01,  9.08823371e-01, -6.43433690e-01,\n",
              "          9.46487129e-01,  9.64284658e-01,  7.10817516e-01,\n",
              "         -5.96172571e-01,  1.82842106e-01, -9.90260959e-01,\n",
              "         -1.58804402e-01, -4.53707665e-01, -9.93388534e-01,\n",
              "          3.19348425e-01, -7.84084797e-01,  4.81384061e-02,\n",
              "         -4.91123693e-03, -9.08263087e-01,  3.38724315e-01,\n",
              "          9.99753475e-01,  5.86365648e-02,  2.48249605e-01,\n",
              "         -2.79796869e-01, -9.99998629e-01,  3.34662199e-01,\n",
              "         -8.86526823e-01,  5.10368824e-01,  4.31769103e-01,\n",
              "          2.34674901e-01,  1.20707795e-01,  4.89223868e-01,\n",
              "          4.24683332e-01,  1.52473941e-01, -1.15433142e-01,\n",
              "          1.48909524e-01, -1.94844082e-01, -5.89608073e-01,\n",
              "         -6.03527129e-01,  3.36631417e-01, -6.01385713e-01,\n",
              "         -9.21593487e-01,  4.03072655e-01,  4.38024178e-02,\n",
              "         -1.73303187e-01, -2.65163600e-01, -5.48305847e-02,\n",
              "         -1.25641093e-01,  8.70846629e-01,  1.99257135e-01,\n",
              "          1.87556952e-01, -8.52465034e-01,  2.91651264e-02,\n",
              "          2.44563937e-01, -4.93135870e-01,  1.00000000e+00,\n",
              "         -3.23883563e-01, -9.82844293e-01,  2.71762609e-01,\n",
              "          2.22626448e-01,  4.27682579e-01,  2.10606247e-01,\n",
              "         -2.34253347e-01, -1.00000000e+00,  4.39189881e-01,\n",
              "         -1.05686955e-01, -9.91412699e-01,  2.72565395e-01,\n",
              "          4.93780762e-01, -1.98211595e-01, -2.54629135e-01,\n",
              "          4.63502854e-01, -2.32893795e-01, -3.64443779e-01,\n",
              "         -3.42620105e-01, -3.81382495e-01, -2.19803616e-01,\n",
              "         -2.82813907e-01,  5.73559180e-02, -2.17191339e-01,\n",
              "         -1.75041303e-01, -2.82257706e-01,  2.64806360e-01,\n",
              "         -4.54839110e-01, -5.22527039e-01,  2.63798416e-01,\n",
              "         -1.08273827e-01,  6.88393950e-01,  2.90187806e-01,\n",
              "         -3.39561582e-01,  4.35308725e-01, -9.61667776e-01,\n",
              "          6.61060870e-01, -3.01564455e-01, -9.82341528e-01,\n",
              "         -5.06386638e-01, -9.91477966e-01,  6.94956362e-01,\n",
              "         -4.30858061e-02, -2.65636623e-01,  9.72999036e-01,\n",
              "          2.42902651e-01,  2.43390754e-01, -1.07505053e-01,\n",
              "         -3.44764650e-01, -1.00000000e+00, -5.86594164e-01,\n",
              "         -3.83891463e-01,  8.18746015e-02, -2.31817961e-01,\n",
              "         -9.81481493e-01, -9.56458211e-01,  6.01826668e-01,\n",
              "          9.52626765e-01,  9.90601107e-02,  9.99554813e-01,\n",
              "         -3.58983457e-01,  9.47182178e-01, -6.00847341e-02,\n",
              "         -3.51423472e-01,  2.94598132e-01, -4.74489927e-01,\n",
              "          5.18657386e-01,  2.72842258e-01, -6.56743646e-01,\n",
              "          1.82775229e-01, -6.08229525e-02,  3.46414477e-01,\n",
              "         -3.38947207e-01, -2.97778875e-01, -1.78932801e-01,\n",
              "         -9.44985688e-01, -3.78242910e-01,  9.48611856e-01,\n",
              "         -1.19614817e-01, -3.99079770e-01,  4.97900575e-01,\n",
              "         -2.66617656e-01, -4.49947625e-01,  8.49364161e-01,\n",
              "          3.76487881e-01,  4.13278490e-01, -2.42755204e-01,\n",
              "          4.20503259e-01, -9.15566683e-02,  6.05690897e-01,\n",
              "         -8.06434512e-01,  5.90924695e-02,  4.05566514e-01,\n",
              "         -2.53965795e-01, -2.57560909e-01, -9.82939780e-01,\n",
              "         -3.68019551e-01,  5.17293930e-01,  9.90783453e-01,\n",
              "          7.37467110e-01,  2.63947278e-01,  5.35044432e-01,\n",
              "         -2.67976344e-01,  6.30989373e-01, -9.54010248e-01,\n",
              "          9.83304679e-01, -2.30649173e-01,  2.65186697e-01,\n",
              "         -1.77363548e-02,  1.43154571e-02, -8.35832119e-01,\n",
              "         -1.47215828e-01,  8.37411165e-01, -6.91978693e-01,\n",
              "         -8.32621813e-01,  1.50374174e-02, -4.41991270e-01,\n",
              "         -4.21659261e-01, -4.54188079e-01,  3.89146924e-01,\n",
              "         -3.02545756e-01, -4.63714689e-01, -8.73663351e-02,\n",
              "          9.27748561e-01,  9.58773792e-01,  8.47922385e-01,\n",
              "         -2.26385832e-01,  5.66705942e-01, -9.33712840e-01,\n",
              "         -4.24724132e-01,  1.45269901e-01,  1.91611096e-01,\n",
              "          2.30547354e-01,  9.95596051e-01, -3.78383517e-01,\n",
              "         -1.69058025e-01, -9.34112549e-01, -9.86466110e-01,\n",
              "          3.00476421e-02, -8.93988013e-01, -8.87397230e-02,\n",
              "         -6.38086140e-01,  5.35483837e-01,  9.17722955e-02,\n",
              "          2.00246796e-01,  4.45546478e-01, -9.89215255e-01,\n",
              "         -7.67423391e-01,  3.91269237e-01, -2.89757848e-01,\n",
              "          4.31150466e-01, -2.64066041e-01,  5.96501768e-01,\n",
              "          6.36996686e-01, -5.40935218e-01,  6.65923476e-01,\n",
              "          8.59350681e-01, -2.27525249e-01, -6.98844552e-01,\n",
              "          8.17817509e-01, -3.22353482e-01,  8.35967779e-01,\n",
              "         -6.85630143e-01,  9.85197484e-01,  6.12212300e-01,\n",
              "          6.18163347e-01, -9.49822068e-01, -4.72982712e-02,\n",
              "         -8.58131647e-01, -3.07183683e-01, -1.04696535e-01,\n",
              "         -3.87877762e-01,  4.93267268e-01,  4.84796047e-01,\n",
              "          3.73757064e-01,  6.96242452e-01, -6.40415549e-01,\n",
              "          9.96926606e-01, -6.65428281e-01, -9.60178971e-01,\n",
              "         -3.12579185e-01, -2.11926192e-01, -9.88677382e-01,\n",
              "          4.64666277e-01,  3.18201035e-01, -7.69948065e-02,\n",
              "         -4.66677368e-01, -5.35357356e-01, -9.63733435e-01,\n",
              "          8.87693286e-01,  1.18420593e-01,  9.86166477e-01,\n",
              "         -1.61282495e-01, -9.23395514e-01, -4.84322757e-01,\n",
              "         -9.25513268e-01, -1.73516288e-01, -7.84410164e-02,\n",
              "          2.70191103e-01, -1.59091964e-01, -9.72253263e-01,\n",
              "          5.00119209e-01,  5.46000719e-01,  4.90458071e-01,\n",
              "         -8.96118432e-02,  9.98123884e-01,  9.99994874e-01,\n",
              "          9.75206554e-01,  8.98603797e-01,  9.01734531e-01,\n",
              "         -9.93791163e-01, -4.98523623e-01,  9.99989212e-01,\n",
              "         -8.91929805e-01, -9.99999940e-01, -9.37087238e-01,\n",
              "         -6.48166180e-01,  2.28224874e-01, -1.00000000e+00,\n",
              "         -8.80058408e-02, -3.67160775e-02, -9.29881752e-01,\n",
              "          4.23431844e-02,  9.79039907e-01,  9.92348373e-01,\n",
              "         -1.00000000e+00,  8.75119627e-01,  9.42290425e-01,\n",
              "         -5.75835228e-01,  7.55000591e-01, -4.06861812e-01,\n",
              "          9.73026037e-01,  2.82401532e-01,  5.04219174e-01,\n",
              "         -2.00506702e-01,  3.81657839e-01, -5.76035678e-01,\n",
              "         -8.54604840e-01, -4.95400326e-03, -1.36505187e-01,\n",
              "          9.54689324e-01,  1.88861459e-01, -6.85867548e-01,\n",
              "         -9.50170875e-01,  2.54146457e-01, -1.10848382e-01,\n",
              "         -1.74096629e-01, -9.67124641e-01, -2.12644130e-01,\n",
              "          1.72594979e-01,  6.55898452e-01,  1.32753938e-01,\n",
              "          2.65600860e-01, -7.91767120e-01,  2.50454009e-01,\n",
              "         -6.05572701e-01,  4.33277994e-01,  6.07434392e-01,\n",
              "         -9.57376778e-01, -5.56698322e-01, -4.38185222e-02,\n",
              "         -1.30459458e-01, -1.56823143e-01, -9.50645268e-01,\n",
              "          9.73752320e-01, -4.21877235e-01,  6.44528151e-01,\n",
              "          1.00000000e+00,  3.37422520e-01, -9.04730201e-01,\n",
              "          3.83494675e-01,  1.71158582e-01, -1.95391491e-01,\n",
              "          1.00000000e+00,  6.35517478e-01, -9.82658207e-01,\n",
              "         -4.74902809e-01,  4.83974040e-01, -5.20029366e-01,\n",
              "         -5.21592617e-01,  9.99165475e-01, -2.24691361e-01,\n",
              "         -1.03662960e-01,  1.74850136e-01,  9.77034807e-01,\n",
              "         -9.91630912e-01,  8.65979195e-01, -8.94858062e-01,\n",
              "         -9.77853596e-01,  9.62676883e-01,  9.51985478e-01,\n",
              "         -2.94502854e-01, -6.56061530e-01,  1.20081715e-01,\n",
              "         -2.58382916e-01,  2.90035009e-01, -9.58885610e-01,\n",
              "          6.77586317e-01,  3.96036416e-01, -1.14011765e-01,\n",
              "          9.08458889e-01, -8.19891453e-01, -4.20862556e-01,\n",
              "          4.02934700e-01, -1.81048363e-01,  1.24809273e-01,\n",
              "          5.83733559e-01,  5.41741848e-01, -2.45000660e-01,\n",
              "          3.20034288e-02, -3.53538245e-01, -3.74435812e-01,\n",
              "         -9.73977506e-01,  1.87118202e-01,  1.00000000e+00,\n",
              "         -1.06215812e-01,  2.17152238e-01, -2.53672838e-01,\n",
              "         -6.38874918e-02, -1.67170942e-01,  3.93938512e-01,\n",
              "          4.79607135e-01, -3.22272688e-01, -8.60086977e-01,\n",
              "          5.24431467e-01, -9.43300426e-01, -9.87688243e-01,\n",
              "          7.82827914e-01,  1.87499657e-01, -2.26510957e-01,\n",
              "          9.99939919e-01,  3.98263603e-01,  2.32968584e-01,\n",
              "          9.99454781e-02,  8.26481819e-01, -4.37556319e-02,\n",
              "          5.93567133e-01,  2.09022447e-01,  9.77329135e-01,\n",
              "         -2.81685472e-01,  4.68313366e-01,  8.28769207e-01,\n",
              "         -4.33572620e-01, -3.05175006e-01, -6.61348939e-01,\n",
              "          2.02642642e-02, -9.27993298e-01,  7.20506683e-02,\n",
              "         -9.57940817e-01,  9.69746888e-01,  4.95764852e-01,\n",
              "          3.50384504e-01,  2.05659539e-01,  2.00349450e-01,\n",
              "          1.00000000e+00, -4.23790157e-01,  5.30502677e-01,\n",
              "         -2.46109813e-01,  8.40972126e-01, -9.89320457e-01,\n",
              "         -7.69298255e-01, -4.29166257e-01, -2.24339981e-02,\n",
              "         -2.51372844e-01, -3.30883086e-01,  2.66782314e-01,\n",
              "         -9.67793167e-01,  2.98828632e-01,  4.03377771e-01,\n",
              "         -9.86727595e-01, -9.93520916e-01,  6.87678307e-02,\n",
              "          6.71883702e-01,  1.32057533e-01, -8.77877116e-01,\n",
              "         -7.30091929e-01, -6.53584957e-01,  6.05216265e-01,\n",
              "         -2.60879308e-01, -9.55511332e-01,  3.08668107e-01,\n",
              "         -2.19063893e-01,  4.67363775e-01, -2.20100626e-01,\n",
              "          4.55168486e-01,  1.83375612e-01,  7.77715385e-01,\n",
              "          1.58306621e-02,  9.27143451e-03, -9.26853493e-02,\n",
              "         -7.58189797e-01,  7.63870239e-01, -7.82054663e-01,\n",
              "         -6.45930588e-01, -1.04551278e-01,  1.00000000e+00,\n",
              "         -3.39304745e-01,  3.82392645e-01,  7.09459662e-01,\n",
              "          6.86580539e-01, -1.13866739e-01,  1.90145507e-01,\n",
              "          5.56000054e-01,  2.85348415e-01, -1.72060251e-01,\n",
              "         -1.28483906e-01, -5.86428285e-01, -4.33644474e-01,\n",
              "          4.79115993e-01, -2.62847338e-02,  6.31739870e-02,\n",
              "          8.30064535e-01,  5.66340506e-01,  1.34964705e-01,\n",
              "          1.19130142e-01, -2.46074516e-02,  9.99417305e-01,\n",
              "         -3.22203040e-01, -2.31371373e-01, -5.05009711e-01,\n",
              "         -1.00562878e-01, -3.08146268e-01, -2.13023812e-01,\n",
              "          1.00000000e+00,  3.16602111e-01,  2.56174892e-01,\n",
              "         -9.92172360e-01, -3.92894596e-01, -9.06759202e-01,\n",
              "          9.99973595e-01,  8.03131700e-01, -8.18137527e-01,\n",
              "          5.65144658e-01,  5.40869713e-01, -3.81595120e-02,\n",
              "          7.52698183e-01, -2.26761714e-01, -2.49459401e-01,\n",
              "          2.78297216e-01,  2.21921310e-01,  9.68497217e-01,\n",
              "         -4.53986704e-01, -9.68264699e-01, -5.37610412e-01,\n",
              "          4.19648886e-01, -9.68508303e-01,  9.96323407e-01,\n",
              "         -4.86729234e-01, -2.10847586e-01, -3.55371982e-01,\n",
              "         -1.43109903e-01,  4.92008269e-01,  2.79853381e-02,\n",
              "         -9.82771873e-01, -2.00740576e-01,  8.67756307e-02,\n",
              "          9.68920588e-01,  2.19682738e-01, -4.74881917e-01,\n",
              "         -9.09893274e-01,  2.53739208e-01,  2.36539438e-01,\n",
              "         -4.28386360e-01, -9.62927520e-01,  9.77291763e-01,\n",
              "         -9.86663997e-01,  5.73570371e-01,  9.99999881e-01,\n",
              "          2.46395946e-01, -3.17345083e-01,  2.28852749e-01,\n",
              "         -3.28404039e-01,  3.61363441e-01, -2.63045341e-01,\n",
              "          5.40634394e-01, -9.63114142e-01, -3.20881456e-01,\n",
              "         -2.26352319e-01,  3.41209531e-01, -1.81002021e-01,\n",
              "         -2.24406391e-01,  7.19184399e-01,  2.37603575e-01,\n",
              "         -4.37926143e-01, -6.03422523e-01, -1.03420086e-01,\n",
              "          4.19157118e-01,  8.19905400e-01, -3.60471338e-01,\n",
              "         -1.25407964e-01,  5.32093085e-02, -6.94316551e-02,\n",
              "         -9.52618957e-01, -2.91375875e-01, -3.43361378e-01,\n",
              "         -9.99464273e-01,  5.11516571e-01, -1.00000000e+00,\n",
              "          8.92060101e-02, -2.89422661e-01, -1.94389850e-01,\n",
              "          8.55795979e-01,  3.36869717e-01,  2.48827234e-01,\n",
              "         -8.03951025e-01, -2.33361796e-01,  7.67609537e-01,\n",
              "          7.99095869e-01, -2.82423913e-01, -2.18262747e-01,\n",
              "         -7.71437049e-01,  2.82911509e-01, -6.62792847e-02,\n",
              "          3.67000282e-01, -2.28723764e-01,  6.83219016e-01,\n",
              "         -1.99098617e-01,  1.00000000e+00,  3.80167440e-02,\n",
              "         -4.01903749e-01, -9.78657365e-01,  2.67579556e-01,\n",
              "         -2.27030456e-01,  9.99998331e-01, -8.44745398e-01,\n",
              "         -9.55228925e-01,  4.00676996e-01, -4.58193064e-01,\n",
              "         -8.17702770e-01,  3.23068559e-01, -7.24987984e-02,\n",
              "         -7.60673285e-01, -6.16359472e-01,  9.68263030e-01,\n",
              "          7.94629455e-01, -4.90923434e-01,  5.56297779e-01,\n",
              "         -3.27064961e-01, -4.77250993e-01,  2.74127815e-02,\n",
              "          3.99176955e-01,  9.88836706e-01,  5.11960149e-01,\n",
              "          8.86786103e-01,  1.76029876e-01, -1.58961445e-01,\n",
              "          9.70406711e-01,  1.89883307e-01,  5.13420105e-01,\n",
              "          1.83898106e-01,  1.00000000e+00,  3.28188509e-01,\n",
              "         -9.22208548e-01,  1.11967921e-01, -9.87209797e-01,\n",
              "         -1.48996308e-01, -9.55541849e-01,  3.42820793e-01,\n",
              "          1.51888266e-01,  8.93353760e-01, -2.49662295e-01,\n",
              "          9.68732059e-01, -2.65313029e-01,  8.09206814e-02,\n",
              "         -3.04256260e-01,  1.87247455e-01,  4.34572428e-01,\n",
              "         -9.28390622e-01, -9.86927092e-01, -9.89480615e-01,\n",
              "          4.84827131e-01, -5.19685149e-01, -1.70916319e-04,\n",
              "          2.74852812e-01,  1.51712209e-01,  4.19454545e-01,\n",
              "          4.62032706e-01, -9.99999940e-01,  9.37361479e-01,\n",
              "          4.83479291e-01,  3.41941088e-01,  9.75571156e-01,\n",
              "          4.86426562e-01,  4.55111116e-01,  2.71694034e-01,\n",
              "         -9.87719119e-01, -9.76743281e-01, -4.20871437e-01,\n",
              "         -2.80589998e-01,  8.02912116e-01,  6.62912965e-01,\n",
              "          8.60613346e-01,  4.54538196e-01, -4.71164793e-01,\n",
              "         -2.91899890e-01, -6.23664185e-02, -6.37614369e-01,\n",
              "         -9.94037151e-01,  3.96607012e-01, -8.61150399e-02,\n",
              "         -9.52212572e-01,  9.60968912e-01, -4.97506261e-01,\n",
              "         -1.28533229e-01,  3.63528192e-01, -4.19996947e-01,\n",
              "          9.40451086e-01,  7.74116993e-01,  3.36847574e-01,\n",
              "          1.43873751e-01,  4.64011401e-01,  9.14025307e-01,\n",
              "          9.34051692e-01,  9.89038885e-01, -3.86325359e-01,\n",
              "          7.38938689e-01, -2.56426394e-01,  4.57736075e-01,\n",
              "          5.95485508e-01, -9.39401329e-01,  1.30768001e-01,\n",
              "          2.25463822e-01, -2.07483992e-01,  2.93672830e-01,\n",
              "         -2.17222914e-01, -9.66611683e-01,  5.81852794e-01,\n",
              "         -2.25286096e-01,  4.62061822e-01, -4.80302721e-01,\n",
              "          1.11256748e-01, -4.10552084e-01, -1.99773788e-01,\n",
              "         -6.77387595e-01, -4.35732037e-01,  5.32664180e-01,\n",
              "          1.70831949e-01,  9.23621356e-01,  7.05481291e-01,\n",
              "         -7.92733654e-02, -6.78504407e-01, -2.42628753e-01,\n",
              "         -8.87851715e-02, -9.27459359e-01,  9.33241844e-01,\n",
              "         -7.17555061e-02,  2.28424788e-01, -5.68528548e-02,\n",
              "          2.30149161e-02,  8.54988813e-01, -1.00780815e-01,\n",
              "         -4.10149246e-01, -2.63856143e-01, -6.69977784e-01,\n",
              "          8.99009526e-01, -2.85295039e-01, -5.00977755e-01,\n",
              "         -5.10450482e-01,  7.84535289e-01,  3.23077679e-01,\n",
              "          9.99119222e-01, -3.15375894e-01, -5.55617929e-01,\n",
              "         -2.86061317e-01, -3.15957010e-01,  3.28047395e-01,\n",
              "         -3.41845661e-01, -9.99999940e-01,  4.14885461e-01,\n",
              "         -2.01224312e-01,  3.35492373e-01, -3.09935749e-01,\n",
              "          1.41130075e-01, -3.72104555e-01, -9.84939396e-01,\n",
              "         -2.69945502e-01,  2.18257591e-01,  7.20306858e-02,\n",
              "         -5.04191637e-01, -6.32864356e-01,  4.96034861e-01,\n",
              "         -3.25772911e-01,  7.96132147e-01,  8.93272698e-01,\n",
              "          1.46873161e-01,  5.70307851e-01,  5.32689571e-01,\n",
              "         -5.45884520e-02, -7.42458284e-01,  9.18511271e-01]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(1, 5, 768), dtype=float32, numpy=\n",
              " array([[[-0.01591275,  0.18899521, -0.17319141, ..., -0.41866958,\n",
              "           0.06151769,  0.18743435],\n",
              "         [ 0.30460766,  0.2426804 ,  0.38542086, ..., -0.50512993,\n",
              "           0.1441811 , -0.02852558],\n",
              "         [ 0.18967548, -0.0896458 ,  0.67940813, ..., -0.44082603,\n",
              "          -0.2292399 ,  0.11063544],\n",
              "         [ 0.18867767, -0.12536146, -0.02388474, ...,  0.04004088,\n",
              "          -0.4757753 , -0.62682366],\n",
              "         [ 0.7511532 , -0.15957607, -0.19766289, ..., -0.00684081,\n",
              "          -0.6228115 , -0.27298236]]], dtype=float32)>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "29hHa9Ys4VZg"
      },
      "source": [
        "all_dataset = tf.data.Dataset.from_generator(lambda: sorted_all,\n",
        "                                             output_types=(tf.int32, tf.int32))"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q4ibaia0-W0g"
      },
      "source": [
        "BATCH_SIZE = 32\n",
        "all_batched = all_dataset.padded_batch(BATCH_SIZE,\n",
        "                                       padded_shapes = ((3, None), ()),\n",
        "                                       padding_values=(0,0))"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2o9977zx_d3Z"
      },
      "source": [
        "data_with_len = [[sent, data_labels[i], len(sent)]\n",
        "                 for i, sent in enumerate(data_inputs)]\n",
        "\n",
        "random.shuffle(data_with_len)\n",
        "data_with_len.sort(key = lambda x: x[2])\n",
        "sorted_all = [([get_ids(sent_lab[0]),\n",
        "               get_mask(sent_lab[0]),\n",
        "               get_segments(sent_lab[0])],\n",
        "\n",
        "               sent_lab[1])\n",
        "              for sent_lab in data_with_len if sent_lab[2]>7]"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EnwDFrgDFNNa"
      },
      "source": [
        "all_dataset = tf.data.Dataset.from_generator(lambda: sorted_all,\n",
        "                                             output_types=(tf.int32, tf.int32))"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X2cEYr_hFfXQ"
      },
      "source": [
        "all_batched = all_dataset.padded_batch(BATCH_SIZE,\n",
        "                                       padded_shapes = ((3, None), ()),\n",
        "                                       padding_values = (0,0))"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ZKkcplNFycI"
      },
      "source": [
        "NB_BATCHES = len(sorted_all) // BATCH_SIZE\n",
        "NB_BATCHES_TEST = NB_BATCHES // 10\n",
        "all_batched.shuffle(NB_BATCHES)\n",
        "test_dataset = all_batched.take(NB_BATCHES_TEST)\n",
        "train_dataset = all_batched.skip(NB_BATCHES_TEST)"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BYqeI8aIG0ut"
      },
      "source": [
        "###Construção do Modelo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nqVUPopoG605"
      },
      "source": [
        "class DCNNBERTEmbedding(tf.keras.Model):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 nb_filters=50,\n",
        "                 FFN_units=512,\n",
        "                 nb_classes=2,\n",
        "                 dropout_rate=0.1,\n",
        "                 name=\"dcnn\"):\n",
        "        super(DCNNBERTEmbedding, self).__init__(name=name)\n",
        "        \n",
        "        self.bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n",
        "                                         trainable = False)\n",
        "\n",
        "        self.bigram = layers.Conv1D(filters=nb_filters,\n",
        "                                    kernel_size=2,\n",
        "                                    padding=\"valid\",\n",
        "                                    activation=\"relu\")\n",
        "        self.trigram = layers.Conv1D(filters=nb_filters,\n",
        "                                     kernel_size=3,\n",
        "                                     padding=\"valid\",\n",
        "                                     activation=\"relu\")\n",
        "        self.fourgram = layers.Conv1D(filters=nb_filters,\n",
        "                                      kernel_size=4,\n",
        "                                      padding=\"valid\",\n",
        "                                      activation=\"relu\")\n",
        "        self.pool = layers.GlobalMaxPool1D()\n",
        "        self.dense_1 = layers.Dense(units=FFN_units, activation=\"relu\")\n",
        "        self.dropout = layers.Dropout(rate=dropout_rate)\n",
        "        if nb_classes == 2:\n",
        "            self.last_dense = layers.Dense(units=1,\n",
        "                                           activation=\"sigmoid\")\n",
        "        else:\n",
        "            self.last_dense = layers.Dense(units=nb_classes,\n",
        "                                           activation=\"softmax\")\n",
        "    \n",
        "    def embed_with_bert(self, all_tokens):\n",
        "      _, embs = self.bert_layer([all_tokens[:, 0, :],\n",
        "                                 all_tokens[:, 1, :],\n",
        "                                 all_tokens[:, 2, :]])\n",
        "      return embs\n",
        "\n",
        "    def call(self, inputs, training):\n",
        "        x = self.embed_with_bert(inputs)\n",
        "        \n",
        "        x_1 = self.bigram(x)\n",
        "        x_1 = self.pool(x_1)\n",
        "        x_2 = self.trigram(x)\n",
        "        x_2 = self.pool(x_2)\n",
        "        x_3 = self.fourgram(x)\n",
        "        x_3 = self.pool(x_3)\n",
        "        \n",
        "        merged = tf.concat([x_1, x_2, x_3], axis=-1) # (batch_size, 3 * nb_filters)\n",
        "        merged = self.dense_1(merged)\n",
        "        merged = self.dropout(merged, training)\n",
        "        output = self.last_dense(merged)\n",
        "        \n",
        "        return output"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OKNcRxnbJtYq"
      },
      "source": [
        "###Treinamento"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EEbfhy2GJsnM"
      },
      "source": [
        "NB_FILTERS = 100\n",
        "FFN_UNITS = 256\n",
        "NB_CLASSES = 2\n",
        "DROPOUT_RATE = 0.2\n",
        "BATCH_SIZE = 32\n",
        "NB_EPOCHS = 5"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AEVRKx3zKrHc"
      },
      "source": [
        "Dcnn = DCNNBERTEmbedding(nb_filters=NB_FILTERS,\n",
        "                         FFN_units=FFN_UNITS,\n",
        "                         nb_classes=NB_CLASSES,\n",
        "                         dropout_rate=DROPOUT_RATE)"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mXCA64mcK-6A"
      },
      "source": [
        "if NB_CLASSES == 2:\n",
        "    Dcnn.compile(loss='binary_crossentropy',\n",
        "                 optimizer='adam', \n",
        "                 metrics=['accuracy']) #Usada quando temos 2 classes - no caso, positivo e negativo\n",
        "else: \n",
        "    Dcnn.compile(loss='spare_categorical_crossentropy',\n",
        "                 optmizer = 'adam',\n",
        "                 metrixs =['sparse_categorical_accuracy'])"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oyISVxRoOOwq"
      },
      "source": [
        "checkpoint_path = '/content/drive/MyDrive/PLN'\n",
        "\n",
        "ckpt = tf.train.Checkpoint(Dcnn=Dcnn)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=1)\n",
        "\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "  ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "  print('Latest checkpoint restored!')"
      ],
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gAf1tpXLObTH"
      },
      "source": [
        "class MyCustomCallback(tf.keras.callbacks.Callback):\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        ckpt_manager.save()\n",
        "        print(\"Checkpoint saved at {}.\".format(checkpoint_path))"
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HiNITWBmYFf5"
      },
      "source": [
        "**Acelerando o treinamento: Step_per_epoch = 100 é uma forma de definir um número para os registros. Porém, o resultado vai gerar uma Acurácia menor do que comparada quando não é definido o número de registros.**\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aJRRJ8rROwDc",
        "outputId": "b2759e4b-7da4-45ab-d803-ac8af900dee1"
      },
      "source": [
        "history = Dcnn.fit(train_dataset,\n",
        "                   epochs=NB_EPOCHS,\n",
        "                   steps_per_epoch = 100,\n",
        "                   callbacks=[MyCustomCallback()])"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "100/100 [==============================] - ETA: 0s - loss: 0.3281 - accuracy: 0.8591Checkpoint saved at /content/drive/MyDrive/PLN.\n",
            "100/100 [==============================] - 9s 94ms/step - loss: 0.3281 - accuracy: 0.8591\n",
            "Epoch 2/5\n",
            "100/100 [==============================] - ETA: 0s - loss: 0.3480 - accuracy: 0.8503Checkpoint saved at /content/drive/MyDrive/PLN.\n",
            "100/100 [==============================] - 11s 110ms/step - loss: 0.3480 - accuracy: 0.8503\n",
            "Epoch 3/5\n",
            "100/100 [==============================] - ETA: 0s - loss: 0.3499 - accuracy: 0.8466Checkpoint saved at /content/drive/MyDrive/PLN.\n",
            "100/100 [==============================] - 10s 102ms/step - loss: 0.3499 - accuracy: 0.8466\n",
            "Epoch 4/5\n",
            "100/100 [==============================] - ETA: 0s - loss: 0.3527 - accuracy: 0.8466Checkpoint saved at /content/drive/MyDrive/PLN.\n",
            "100/100 [==============================] - 9s 92ms/step - loss: 0.3527 - accuracy: 0.8466\n",
            "Epoch 5/5\n",
            "100/100 [==============================] - ETA: 0s - loss: 0.3555 - accuracy: 0.8519Checkpoint saved at /content/drive/MyDrive/PLN.\n",
            "100/100 [==============================] - 9s 91ms/step - loss: 0.3555 - accuracy: 0.8519\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4hyBUeJlYHvp"
      },
      "source": [
        "###Avaliação"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dpl7X7W4YOKZ",
        "outputId": "ab2f857c-10b6-447f-a3e7-ff69c8ea5fba"
      },
      "source": [
        "results = Dcnn.evaluate(test_dataset)\n",
        "print(results)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4513/4513 [==============================] - 266s 59ms/step - loss: 0.3503 - accuracy: 0.8494\n",
            "[0.35025113821029663, 0.8493865132331848]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_2jBz57YPhh"
      },
      "source": [
        "def get_prediction(sentence):\n",
        "  tokens = encode_sentence(sentence)\n",
        "\n",
        "  input_ids = get_ids(tokens)\n",
        "  input_mask = get_mask(tokens)\n",
        "  segment_ids = get_segments(tokens)\n",
        "\n",
        "  inputs = tf.stack(\n",
        "      [\n",
        "       tf.cast(input_ids, dtype=tf.int32),\n",
        "       tf.cast(input_mask, dtype=tf.int32),\n",
        "       tf.cast(segment_ids, dtype=tf.int32),\n",
        "      ], axis = 0)\n",
        "  inputs = tf.expand_dims(inputs, 0)\n",
        "\n",
        "  output = Dcnn(inputs, training=False)\n",
        "\n",
        "  sentiment = math.floor(output*2)\n",
        "\n",
        "  if sentiment == 0:\n",
        "    print(\"Output of the model: {}\\nPredicted sentiment: negative\".format(output))\n",
        "  elif sentiment == 1:\n",
        "    print(\"Output of the model: {}\\nPredicted sentiment: positive\".format(output))\n"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n9YJxRiGYSbx",
        "outputId": "e5a15f55-c528-4e03-bf6e-0d5855b30915"
      },
      "source": [
        "get_prediction(\"This actor is very bad.\")"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Output of the model: [[0.01695283]]\n",
            "Predicted sentiment: negative\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dt4M_yaFYTwt",
        "outputId": "4eafd1d8-62e5-4474-9fa4-63f09d058ca3"
      },
      "source": [
        "get_prediction(\"I like dogs.\")"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Output of the model: [[0.90014774]]\n",
            "Predicted sentiment: positive\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}