BertV2_Final_Version.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Bert_finetuned_V1.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f770a98812754ffabb329e42074af378": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d1703e0b79b44e028c014d0f71e4b358",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d85feebd77134ff3b4758b7eca86e0d5",
              "IPY_MODEL_ae8e92aa2e5a4ab6bbd9e1e2e4bdc1a1"
            ]
          }
        },
        "d1703e0b79b44e028c014d0f71e4b358": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d85feebd77134ff3b4758b7eca86e0d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_571c82eef6e443538069d3b91cf19d64",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 200,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 200,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f2e5924d0be466f891acbbe5ac7b9ea"
          }
        },
        "ae8e92aa2e5a4ab6bbd9e1e2e4bdc1a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1b107ad364154afdb8128a0ee7f37719",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 200/200 [13:35&lt;00:00,  3.98s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_efd05db63a4849e781c30f2ed33bded2"
          }
        },
        "571c82eef6e443538069d3b91cf19d64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f2e5924d0be466f891acbbe5ac7b9ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1b107ad364154afdb8128a0ee7f37719": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "efd05db63a4849e781c30f2ed33bded2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Nilanshrajput/AdvitiyaHackathon/blob/master/BertV2_Final_Version.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "447jWka-yLBL",
        "outputId": "4bbdd1be-60ac-45d8-f60c-444aae2b02f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 666
        }
      },
      "source": [
        "#Installing hugging face library for Fine tuning Bert Model\n",
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/fc/bd726a15ab2c66dc09306689d04da07a3770dad724f0883f0a4bfb745087/transformers-2.4.1-py3-none-any.whl (475kB)\n",
            "\r\u001b[K     |▊                               | 10kB 29.5MB/s eta 0:00:01\r\u001b[K     |█▍                              | 20kB 2.2MB/s eta 0:00:01\r\u001b[K     |██                              | 30kB 3.2MB/s eta 0:00:01\r\u001b[K     |██▊                             | 40kB 2.1MB/s eta 0:00:01\r\u001b[K     |███▍                            | 51kB 2.6MB/s eta 0:00:01\r\u001b[K     |████▏                           | 61kB 3.1MB/s eta 0:00:01\r\u001b[K     |████▉                           | 71kB 3.6MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 81kB 2.8MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 92kB 3.1MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 102kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 112kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 122kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████                       | 133kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 143kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 153kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████                     | 163kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 174kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 184kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 194kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 204kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 215kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 225kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 235kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 245kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 256kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 266kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 276kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 286kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 296kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 307kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 317kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 327kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 337kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 348kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 358kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 368kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 378kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 389kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 399kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 409kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 419kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 430kB 3.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 440kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 450kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 460kB 3.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 471kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 481kB 3.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.11.10)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\r\u001b[K     |▎                               | 10kB 30.2MB/s eta 0:00:01\r\u001b[K     |▋                               | 20kB 37.4MB/s eta 0:00:01\r\u001b[K     |█                               | 30kB 44.2MB/s eta 0:00:01\r\u001b[K     |█▎                              | 40kB 50.3MB/s eta 0:00:01\r\u001b[K     |█▋                              | 51kB 54.1MB/s eta 0:00:01\r\u001b[K     |██                              | 61kB 58.6MB/s eta 0:00:01\r\u001b[K     |██▏                             | 71kB 61.6MB/s eta 0:00:01\r\u001b[K     |██▌                             | 81kB 61.4MB/s eta 0:00:01\r\u001b[K     |██▉                             | 92kB 63.2MB/s eta 0:00:01\r\u001b[K     |███▏                            | 102kB 65.4MB/s eta 0:00:01\r\u001b[K     |███▌                            | 112kB 65.4MB/s eta 0:00:01\r\u001b[K     |███▉                            | 122kB 65.4MB/s eta 0:00:01\r\u001b[K     |████                            | 133kB 65.4MB/s eta 0:00:01\r\u001b[K     |████▍                           | 143kB 65.4MB/s eta 0:00:01\r\u001b[K     |████▊                           | 153kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████                           | 163kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 174kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 184kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████                          | 194kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 204kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 215kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████                         | 225kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 235kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 245kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 256kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 266kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 276kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 286kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 296kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 307kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 317kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████                      | 327kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 337kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 348kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████                     | 358kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 368kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 378kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████                    | 389kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 399kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 409kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 419kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 430kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 440kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 450kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 460kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 471kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 481kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 491kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 501kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 512kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████                | 522kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 532kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 542kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 552kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 563kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 573kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 583kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 593kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 604kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 614kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 624kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 634kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 645kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 655kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 665kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 675kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 686kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 696kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 706kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 716kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 727kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 737kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 747kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 757kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 768kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 778kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 788kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 798kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 808kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 819kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 829kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 839kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 849kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 860kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 870kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 880kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 890kB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 901kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 911kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 921kB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 931kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 942kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 952kB 65.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 962kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 972kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 983kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 993kB 65.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.0MB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.0MB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 1.0MB 65.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.0MB 65.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.0MB 65.4MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 60.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers==0.0.11\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5e/36/7af38d572c935f8e0462ec7b4f7a46d73a2b3b1a938f50a5e8132d5b2dc5/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl (3.1MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1MB 48.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.5)\n",
            "Requirement already satisfied: botocore<1.15.0,>=1.14.10 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.14.10)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.2)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.10->boto3->transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.10->boto3->transformers) (2.6.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=1eaf2390782ced39936722d441525b542e7eadc5a5cdca9cb43e2dcbf80b6fc5\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sentencepiece, sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.0.11 transformers-2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "5hYcPCU-4Jkm",
        "outputId": "03392697-cc9c-458a-97a7-15b29ce37553",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 63
        }
      },
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from transformers import BertModel, BertTokenizer, BertForSequenceClassification\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torch import nn\n",
        "\n",
        "from time import time\n",
        "\n",
        "\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y4Ork0sDd5Dl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "%%capture\n",
        "from tqdm import tnrange\n",
        "from tqdm import tqdm_notebook as tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-ZJfAJDPN9dC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "e767bd06-d1a8-4185-fd6c-41b7dbf7413e"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "f7TlkavJOGYt",
        "colab": {}
      },
      "source": [
        "drive_pt=\"/content/drive/My Drive/dl_projects/AdvitiyaHackathon/\"\n",
        "\n",
        "git_path = \"AdvitiyaHackathon/\"\n",
        "path = drive_pt+\"ScrappeData\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-1YQJ4nw4N6Y",
        "colab": {}
      },
      "source": [
        "bert_model = BertModel.from_pretrained('bert-base-uncased')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ReefGhAI4Pp4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ffc077b6-afc9-462a-a44f-a3ebfa645668"
      },
      "source": [
        "\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "#DataSet class\n",
        "class LoadDataset(Dataset):\n",
        "\n",
        "    def __init__(self, filename, maxlen):\n",
        "\n",
        "        # Store the contents of the file in a pandas dataframe\n",
        "        self.df = pd.read_csv(filename, delimiter=',',index_col=0)\n",
        "\n",
        "        # Initialize the BERT tokenizer\n",
        "        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "        # Define the Maxlength for padding/truncating\n",
        "        self.maxlen = maxlen\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.df)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "\n",
        "        # Selecting the sentence and label at the specified index in the data frame\n",
        "        sentence = self.df.loc[index, 'text']\n",
        "        sentence = sentence[10:]\n",
        "        #sentence = re.sub(' +', ' ',sentence)\n",
        "        label = torch.tensor((self.df.loc[index, 'cat']))\n",
        "\n",
        "        # Tokenize the sentence\n",
        "        tokens_ids = self.tokenizer.encode(str(sentence),add_special_tokens = True,max_length = self.maxlen, pad_to_max_length=True)\n",
        "        \"\"\"\n",
        "        tokens_ids = pad_sequences([tokens_ids], maxlen=self.maxlen, dtype=\"long\", \n",
        "                          value=0, truncating=\"post\", padding=\"post\")\n",
        "        \"\"\"   \n",
        "        # Inserting the CLS and SEP token at the beginning and end of the sentence\n",
        "        #tokens = ['[CLS]'] + tokens \n",
        "        \"\"\"\n",
        "        if len(tokens_ids) < self.maxlen:\n",
        "            tokens_ids = [tokens_ids + [0] for _ in range(self.maxlen - len(tokens_ids))]\n",
        "        else:\n",
        "            tokens_ids = tokens_ids[:self.maxlen-1]         \n",
        "        \"\"\"  \n",
        "        # Padding/truncating the sentences to the maximum length\n",
        "\n",
        "\n",
        "        # Convert the sequence to ids with BERT Vocabulary\n",
        "        #tokens_ids = self.tokenizer.convert_tokens_to_ids(tokens)\n",
        "        \n",
        "        # Converting the list to a pytorch tensor\n",
        "        tokens_ids_tensor = torch.tensor(tokens_ids)\n",
        "\n",
        "        # Obtaining the attention mask\n",
        "        attn_mask = (tokens_ids_tensor != 0).long()\n",
        "\n",
        "        return tokens_ids_tensor, attn_mask, label"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6rwgMlFO4YtY",
        "colab": {}
      },
      "source": [
        "# Creating instances of training and validation set\n",
        "train_set = LoadDataset(filename = drive_pt+'train/all_text_train_all.csv', maxlen = 200)\n",
        "val_set = LoadDataset(filename = drive_pt+'train/val_all.csv', maxlen = 200)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-cfCx-474oHQ",
        "colab": {}
      },
      "source": [
        "\n",
        "# Creating intsances of training and validation dataloaders\n",
        "train_loader = DataLoader(train_set, batch_size = 16, shuffle = True, num_workers = 5)\n",
        "val_loader = DataLoader(val_set, batch_size = 32, shuffle = True, num_workers = 5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tn5tAZeq4uEp",
        "outputId": "989b1858-dc8e-4e39-fe9e-e60abd7aecc8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single \n",
        "# linear classification layer on top. \n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 4, # The number of output labels--2 for binary classification.\n",
        "                    # You can increase this for multi-class tasks.   \n",
        "    output_attentions = True, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "#model = PatentClassifier(freeze_bert= False)\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=4, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "--fVYpH-4wV6",
        "outputId": "8b3cfdf0-d334-4729-c360-cf181b4276ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 612
        }
      },
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (4, 768)\n",
            "classifier.bias                                                 (4,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "LW42kAlLUHpv",
        "colab": {}
      },
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
        "# I believe the 'W' stands for 'Weight Decay fix\"\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 4e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qNq70tAaUKXz",
        "colab": {}
      },
      "source": [
        "\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 30\n",
        "\n",
        "# Total number of training steps is number of batches * number of epochs.\n",
        "total_steps = len(train_loader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "o0lOSnbZUKR-",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "BWawbH25UKPa",
        "colab": {}
      },
      "source": [
        "\n",
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "l-UEv8yafgOe",
        "outputId": "8edcc8e0-206d-4079-efe1-7842c7bb5a32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "\n",
        "\n",
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2yxi55x1UVMv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "5fa1d00f-c85d-4ab5-d8af-11980094e1fd"
      },
      "source": [
        "#Loading our model\n",
        "import os\n",
        "save_path = drive_pt+ \"Models/\"\n",
        "\n",
        "\n",
        "\"\"\"# Loading the checkpoints for resuming training\n",
        "checkpoint = torch.load(save_path+'finalbertv4_claims_512_8_20.pth')\n",
        "model.load_state_dict(checkpoint['model_state_dict'])\"\"\""
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"# Loading the checkpoints for resuming training\\ncheckpoint = torch.load(save_path+'finalbertv4_claims_512_8_20.pth')\\nmodel.load_state_dict(checkpoint['model_state_dict'])\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9QxCln39UKM2",
        "outputId": "a228f396-4c0b-429e-9f75-25549338f400",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "import random\n",
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# Store the average loss after each epoch so we can plot them.\n",
        "loss_values = []\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_loss = 0\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to \n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_loader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_loader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "       \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because \n",
        "        # accumulating the gradients is \"convenient while training RNNs\". \n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()        \n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # This will return the loss (rather than the model output) because we\n",
        "        # have provided the `labels`.\n",
        "        # The documentation for this `model` function is here: \n",
        "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "        outputs = model(b_input_ids, \n",
        "                    token_type_ids=None, \n",
        "                    attention_mask=b_input_mask, \n",
        "                    labels=b_labels)\n",
        "        \n",
        "        # The call to `model` always returns a tuple, so we need to pull the \n",
        "        # loss value out of the tuple.\n",
        "        loss = outputs[0]\n",
        "\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value \n",
        "        # from the tensor.\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over the training data.\n",
        "    avg_train_loss = total_loss / len(train_loader)            \n",
        "    \n",
        "    # Store the loss value for plotting the learning curve.\n",
        "    loss_values.append(avg_train_loss)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    eval_loss, eval_accuracy = 0, 0\n",
        "    nb_eval_steps, nb_eval_examples = 0, 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in val_loader:\n",
        "        \n",
        "        # Add batch to GPU\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        \n",
        "        # Unpack the inputs from our dataloader\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        \n",
        "        # Telling the model not to compute or store gradients, saving memory and\n",
        "        # speeding up validation\n",
        "        with torch.no_grad():        \n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # This will return the logits rather than the loss because we have\n",
        "            # not provided labels.\n",
        "            # token_type_ids is the same as the \"segment ids\", which \n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            # The documentation for this `model` function is here: \n",
        "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "            outputs = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask)\n",
        "        \n",
        "        # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "        # values prior to applying an activation function like the softmax.\n",
        "        logits = outputs[0]\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "        \n",
        "        # Calculate the accuracy for this batch of test sentences.\n",
        "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
        "        \n",
        "        # Accumulate the total accuracy.\n",
        "        eval_accuracy += tmp_eval_accuracy\n",
        "\n",
        "        # Track the number of batches\n",
        "        nb_eval_steps += 1\n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
        "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.32\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 2 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:34.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:04.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:03.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.07\n",
            "  Training epcoh took: 0:02:51\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 3 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.03\n",
            "  Training epcoh took: 0:02:49\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 4 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.02\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 5 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.03\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 6 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:01.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.02\n",
            "  Training epcoh took: 0:02:49\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 7 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 8 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 9 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:02.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:01.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.02\n",
            "  Training epcoh took: 0:02:49\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 10 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:02.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:31.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:01.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:49\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 11 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 12 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:01.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:49\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 13 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:34.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 14 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:02.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:01.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:09\n",
            "\n",
            "======== Epoch 15 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 16 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 17 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 18 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 19 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:32.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:02.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:31.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:01.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:02:49\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 20 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:34.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 21 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 22 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 23 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:34.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:04.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:03.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:51\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 24 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:31.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 25 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:32.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:51\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:09\n",
            "\n",
            "======== Epoch 26 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:34.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:04.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:03.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:33.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:51\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:09\n",
            "\n",
            "======== Epoch 27 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:03.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:33.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:51\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 28 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:03.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:51\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 29 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:02.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:32.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:50\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 30 / 30 ========\n",
            "Training...\n",
            "  Batch    40  of    225.    Elapsed: 0:00:33.\n",
            "  Batch    80  of    225.    Elapsed: 0:01:03.\n",
            "  Batch   120  of    225.    Elapsed: 0:01:33.\n",
            "  Batch   160  of    225.    Elapsed: 0:02:03.\n",
            "  Batch   200  of    225.    Elapsed: 0:02:33.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:52\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 1.00\n",
            "  Validation took: 0:00:09\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "T6uchpW-8SAe",
        "colab": {}
      },
      "source": [
        "# Saving our model\n",
        "import os\n",
        "save_path = drive_pt+ \"Models/\"\n",
        "if not os.path.isdir(save_path):\n",
        "    os.mkdir(save_path)\n",
        "\n",
        "torch.save({\n",
        "    'model_state_dict': model.state_dict(),\n",
        "    'optimizer_state_dict': optimizer.state_dict()\n",
        "}, save_path+'all_final_200_32_30.pth')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ADjoeV2luh5P",
        "colab_type": "text"
      },
      "source": [
        "Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kb1pF3fuH-UJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Saving our model\n",
        "import os\n",
        "save_path = drive_pt+ \"Models/\"\n",
        "model1,model2,model3,model4,model5, model6 = model,model,model,model,model,model\n",
        "\n",
        "# Loading the checkpoints for resuming training\n",
        "checkpoint = torch.load(save_path+'finalbertv5_abstract_512_8_20.pth')\n",
        "model1.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "# Loading the checkpoints for resuming training\n",
        "checkpoint = torch.load(save_path+'bertv6_classification_512_8_20.pth')\n",
        "model2.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "# Loading the checkpoints for resuming training\n",
        "checkpoint = torch.load(save_path+'finalbertv4_claims_512_8_20.pth')\n",
        "#model3.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "# Loading the checkpoints for resuming training\n",
        "checkpoint = torch.load(save_path+'all_final_calims_128_32_30.pth')\n",
        "#model4.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "# Loading the checkpoints for resuming training\n",
        "checkpoint = torch.load(save_path+'all_final_calims_300_16_20.pth')\n",
        "#model5.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "# Loading the checkpoints for resuming training\n",
        "checkpoint = torch.load(save_path+'all_final_200_32_30.pth')\n",
        "#model6.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s-yUJXGdsmNy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#auxilary  Functions for Evaluating on Test set\n",
        "def most_frequent(List): \n",
        "    counter = 0\n",
        "    num = List[0] \n",
        "      \n",
        "    for i in List: \n",
        "        curr_frequency = List.count(i) \n",
        "        if(curr_frequency> counter): \n",
        "            counter = curr_frequency \n",
        "            num = i \n",
        "  \n",
        "    return num \n",
        "\n",
        "\n",
        "\n",
        "def return_model_output(model, sentences, dev ):\n",
        "  results=[]\n",
        "  model.eval()\n",
        "  for sentence in sentences:\n",
        "    b_input_ids, b_input_mask = pre_process(sentence, maxlen = 512,device=dev)\n",
        "    output = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask)\n",
        "    logits = output[0]\n",
        "    # Move logits and labels to CPU\n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    cat = np.argmax(logits, axis=1).flatten()\n",
        "    results.append(int(cat))\n",
        "\n",
        "  return results\n",
        "\n",
        "def pre_process(sentence, maxlen,device):\n",
        "  tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "  #sentence = sentence[50:]\n",
        "  tokens_ids = tokenizer.encode(str(sentence),add_special_tokens = True,max_length = maxlen, pad_to_max_length=True)\n",
        "  # Converting the list to a pytorch tensor\n",
        "  tokens_ids_tensor = torch.tensor(tokens_ids)\n",
        "\n",
        "  # Obtaining the attention mask\n",
        "  attn_mask = (tokens_ids_tensor != 0).long()\n",
        "\n",
        "  return tokens_ids_tensor.unsqueeze(dim = 0).to(device), attn_mask.unsqueeze(dim = 0).to(device)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jDJk7nOZveNT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Loading Test set\n",
        "df1 = pd.read_csv(drive_pt+\"claims_final_test_clean.csv\", index_col= 0 )\n",
        "df2 = pd.read_csv(drive_pt+\"classification_final_test_clean.csv\", index_col= 0 )\n",
        "df3 = pd.read_csv(drive_pt+\"abstract_final_test_clean.csv\", index_col= 0 )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6tB9aK8TFdsv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "f770a98812754ffabb329e42074af378",
            "d1703e0b79b44e028c014d0f71e4b358",
            "d85feebd77134ff3b4758b7eca86e0d5",
            "ae8e92aa2e5a4ab6bbd9e1e2e4bdc1a1",
            "571c82eef6e443538069d3b91cf19d64",
            "6f2e5924d0be466f891acbbe5ac7b9ea",
            "1b107ad364154afdb8128a0ee7f37719",
            "efd05db63a4849e781c30f2ed33bded2"
          ]
        },
        "outputId": "70e62171-a028-4210-8715-e4f94030d17c"
      },
      "source": [
        "#Predicting on Test Set\n",
        "\n",
        "for index in tnrange(0,len(df1)):\n",
        "  \n",
        "  sentence1 = df1.loc[index, 'text']\n",
        "  sentence2 = df2.loc[index, 'text']\n",
        "  sentence3 = df3.loc[index, 'text']\n",
        "  sentences = [sentence1, sentence2, sentence3]\n",
        "  results = return_model_output(model1, sentences= sentences,dev = device)\n",
        "  #results += return_model_output(model2, sentences= sentences,dev = device)\n",
        "  results += return_model_output(model3, sentences= sentences,dev = device)\n",
        "  results += return_model_output(model4, sentences= sentences,dev = device)\n",
        "  results += return_model_output(model5, sentences= sentences,dev = device)\n",
        "  results += return_model_output(model6, sentences= sentences,dev = device)\n",
        "\n",
        "  final_cat = most_frequent(results)\n",
        "  df1.loc[index, 'cat'] = final_cat\n"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f770a98812754ffabb329e42074af378",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=200), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8jbaUULrME6U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#get the test data \n",
        "dftest = pd.read_csv(drive_pt+\"test.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YIf-kvOKLPI1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dftest[\"Category\"] = df1.iloc[:,1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TJPoqWoxe1Dm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Cnpnverting encoded cateegoris to textual form\n",
        "\n",
        "names = [\"Non-Alcohol\",\"Alcohol\", \"Non-Autonomous Vehicles\", \"Autonomous Vehicles\"]\n",
        "for i in range(0,4):\n",
        "  mask = dftest.Category == i\n",
        "  column_name = 'Category'\n",
        "  dftest.loc[mask, column_name] = names[i]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VFyuA4SMD-du",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4857d88b-0828-416f-82f8-024f5d09cfa2"
      },
      "source": [
        "#checking from predicted\n",
        "dftest.loc[34, \"Category\"]"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Alcohol'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Ww6NmOClCqz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#saving csv\n",
        "dftest.to_csv(\"submission_5lastmodel.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}