go_emotion_of_transformers_multilabel_text_classification_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "go emotion of transformers_multilabel-text-classification-v2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c360e7314a0648cab8ad45bfc5c7f3e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8bbc07631d7141a7ba6c61b7a3bcb48b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0f3c4417840849e1a113c78f7cd6e7f5",
              "IPY_MODEL_50b6de569fad4bed9104fce993d93d21",
              "IPY_MODEL_df55147b24914a1484e205e86d6d8480"
            ]
          }
        },
        "8bbc07631d7141a7ba6c61b7a3bcb48b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f3c4417840849e1a113c78f7cd6e7f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b5ea51091e2c485b9a59427f2359f04d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d461b2ecab8047cd813b0a7212efbae0"
          }
        },
        "50b6de569fad4bed9104fce993d93d21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_899fd24b7aa04e248ab3cc42fa372479",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_be4af11dd381470cb751df15156de3cf"
          }
        },
        "df55147b24914a1484e205e86d6d8480": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b0a704e5b4744c59b74fbfc33c30da6a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00, 20.67it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a3d09c3436374b64844f4f2bb161e5cd"
          }
        },
        "b5ea51091e2c485b9a59427f2359f04d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d461b2ecab8047cd813b0a7212efbae0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "899fd24b7aa04e248ab3cc42fa372479": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "be4af11dd381470cb751df15156de3cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b0a704e5b4744c59b74fbfc33c30da6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a3d09c3436374b64844f4f2bb161e5cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1ea376d31b0942c58301973b778d4b74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_06798b45bc3d486a9bc7b9b5e8c65fae",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c7d651016ca64488b4897456a353ef0c",
              "IPY_MODEL_13b4483bbc364fcea90820ec93e2cde9",
              "IPY_MODEL_94ef6bdd3f0842a79de673caa896d0c7"
            ]
          }
        },
        "06798b45bc3d486a9bc7b9b5e8c65fae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7d651016ca64488b4897456a353ef0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2637c6d539724f27bb4e01ed8e26b120",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Upload file pytorch_model.bin: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e2dda27caab74038bf7815606c168e5f"
          }
        },
        "13b4483bbc364fcea90820ec93e2cde9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_84bf702e3fef43aaba8e83df0e1c8e28",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 267940017,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 267940017,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_33d5f793a357444a8852f4443d1f4587"
          }
        },
        "94ef6bdd3f0842a79de673caa896d0c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e25dc60ee9414028a67c04c7ff709ad9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 256M/256M [04:21&lt;00:00, 1.14MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c8f21770d07a47e28eebfba384a7121a"
          }
        },
        "2637c6d539724f27bb4e01ed8e26b120": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e2dda27caab74038bf7815606c168e5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "84bf702e3fef43aaba8e83df0e1c8e28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "33d5f793a357444a8852f4443d1f4587": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e25dc60ee9414028a67c04c7ff709ad9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c8f21770d07a47e28eebfba384a7121a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bhadreshpsavani/UnderstandingNLP/blob/master/go_emotion_of_transformers_multilabel_text_classification_v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AjLimE05RH1P",
        "outputId": "2a2370d1-9629-47c2-9780-0b78718145e1"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mon Nov 29 09:17:00 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 495.44       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   42C    P8    29W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jvcU6VpuJWeS"
      },
      "source": [
        "%%capture\n",
        "!pip install transformers==4.12.5 pandas torch"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ytdiy3hJJ88P",
        "outputId": "3b36a025-1880-4b40-9aca-5f0964e67413"
      },
      "source": [
        "import transformers\n",
        "\n",
        "print(f\"Running on transformers v{transformers.__version__}\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on transformers v4.12.5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "snQ0FmPFJ4A3"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z92qtM8tJ2-F"
      },
      "source": [
        "import torch\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from pathlib import Path\n",
        "from transformers import (AutoTokenizer, AutoModelForSequenceClassification, \n",
        "                          PreTrainedModel, DistilBertModel, DistilBertForSequenceClassification,\n",
        "                          TrainingArguments, Trainer)\n",
        "from transformers.modeling_outputs import SequenceClassifierOutput"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MrruHb_oNnjD"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Xn_eAN_Kdq9"
      },
      "source": [
        "!pip install -q datasets"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "c360e7314a0648cab8ad45bfc5c7f3e4",
            "8bbc07631d7141a7ba6c61b7a3bcb48b",
            "0f3c4417840849e1a113c78f7cd6e7f5",
            "50b6de569fad4bed9104fce993d93d21",
            "df55147b24914a1484e205e86d6d8480",
            "b5ea51091e2c485b9a59427f2359f04d",
            "d461b2ecab8047cd813b0a7212efbae0",
            "899fd24b7aa04e248ab3cc42fa372479",
            "be4af11dd381470cb751df15156de3cf",
            "b0a704e5b4744c59b74fbfc33c30da6a",
            "a3d09c3436374b64844f4f2bb161e5cd"
          ]
        },
        "id": "3znhfpEJFYbY",
        "outputId": "be4c089a-1007-4062-fb23-18eb8cc9a76f"
      },
      "source": [
        "from datasets import load_dataset\n",
        "emotions = load_dataset(\"go_emotions\", \"raw\")"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reusing dataset go_emotions (/root/.cache/huggingface/datasets/go_emotions/raw/0.0.0/2637cfdd4e64d30249c3ed2150fa2b9d279766bfcd6a809b9f085c61a90d776d)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c360e7314a0648cab8ad45bfc5c7f3e4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WT0tJLZSIwk6",
        "outputId": "766e7e6a-dda1-4516-b349-ec59f2d5afac"
      },
      "source": [
        "emotions"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['text', 'id', 'author', 'subreddit', 'link_id', 'parent_id', 'created_utc', 'rater_id', 'example_very_unclear', 'admiration', 'amusement', 'anger', 'annoyance', 'approval', 'caring', 'confusion', 'curiosity', 'desire', 'disappointment', 'disapproval', 'disgust', 'embarrassment', 'excitement', 'fear', 'gratitude', 'grief', 'joy', 'love', 'nervousness', 'optimism', 'pride', 'realization', 'relief', 'remorse', 'sadness', 'surprise', 'neutral'],\n",
              "        num_rows: 211225\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eEhF20D8Gwfa"
      },
      "source": [
        "df = emotions['train'].to_pandas()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NjsgScIZHbUY",
        "outputId": "403c8e82-6c2c-475d-e027-ffc18a1af595"
      },
      "source": [
        "label_cols = ['admiration', 'amusement', 'anger', 'annoyance', 'approval', 'caring', 'confusion', 'curiosity', 'desire', 'disappointment', 'disapproval', 'disgust', 'embarrassment', 'excitement', 'fear', 'gratitude', 'grief', 'joy', 'love', 'nervousness', 'optimism', 'pride', 'realization', 'relief', 'remorse', 'sadness', 'surprise', 'neutral']\n",
        "len(label_cols)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "28"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uz5A6BiBkpVO"
      },
      "source": [
        "id2label = {str(i):label for i, label in enumerate(label_cols)}\n",
        "label2id = {label:str(i) for i, label in enumerate(label_cols)}"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QDYmET3dlGz3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b5075936-e527-4818-acf2-def021b3d01f"
      },
      "source": [
        "id2label"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': 'admiration',\n",
              " '1': 'amusement',\n",
              " '10': 'disapproval',\n",
              " '11': 'disgust',\n",
              " '12': 'embarrassment',\n",
              " '13': 'excitement',\n",
              " '14': 'fear',\n",
              " '15': 'gratitude',\n",
              " '16': 'grief',\n",
              " '17': 'joy',\n",
              " '18': 'love',\n",
              " '19': 'nervousness',\n",
              " '2': 'anger',\n",
              " '20': 'optimism',\n",
              " '21': 'pride',\n",
              " '22': 'realization',\n",
              " '23': 'relief',\n",
              " '24': 'remorse',\n",
              " '25': 'sadness',\n",
              " '26': 'surprise',\n",
              " '27': 'neutral',\n",
              " '3': 'annoyance',\n",
              " '4': 'approval',\n",
              " '5': 'caring',\n",
              " '6': 'confusion',\n",
              " '7': 'curiosity',\n",
              " '8': 'desire',\n",
              " '9': 'disappointment'}"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NWPqOkhKlI5-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b67a60ec-46af-4cd2-c7cb-0105d3e1ebf6"
      },
      "source": [
        "label2id"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'admiration': '0',\n",
              " 'amusement': '1',\n",
              " 'anger': '2',\n",
              " 'annoyance': '3',\n",
              " 'approval': '4',\n",
              " 'caring': '5',\n",
              " 'confusion': '6',\n",
              " 'curiosity': '7',\n",
              " 'desire': '8',\n",
              " 'disappointment': '9',\n",
              " 'disapproval': '10',\n",
              " 'disgust': '11',\n",
              " 'embarrassment': '12',\n",
              " 'excitement': '13',\n",
              " 'fear': '14',\n",
              " 'gratitude': '15',\n",
              " 'grief': '16',\n",
              " 'joy': '17',\n",
              " 'love': '18',\n",
              " 'nervousness': '19',\n",
              " 'neutral': '27',\n",
              " 'optimism': '20',\n",
              " 'pride': '21',\n",
              " 'realization': '22',\n",
              " 'relief': '23',\n",
              " 'remorse': '24',\n",
              " 'sadness': '25',\n",
              " 'surprise': '26'}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NdzcH9y0md_8"
      },
      "source": [
        "## Preprocess data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 678
        },
        "id": "g7q5pPiXScfU",
        "outputId": "7de70453-8cab-41e1-d548-dc65a2da8d6b"
      },
      "source": [
        "df[\"labels\"] = df[label_cols].values.tolist()\n",
        "df.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>id</th>\n",
              "      <th>author</th>\n",
              "      <th>subreddit</th>\n",
              "      <th>link_id</th>\n",
              "      <th>parent_id</th>\n",
              "      <th>created_utc</th>\n",
              "      <th>rater_id</th>\n",
              "      <th>example_very_unclear</th>\n",
              "      <th>admiration</th>\n",
              "      <th>amusement</th>\n",
              "      <th>anger</th>\n",
              "      <th>annoyance</th>\n",
              "      <th>approval</th>\n",
              "      <th>caring</th>\n",
              "      <th>confusion</th>\n",
              "      <th>curiosity</th>\n",
              "      <th>desire</th>\n",
              "      <th>disappointment</th>\n",
              "      <th>disapproval</th>\n",
              "      <th>disgust</th>\n",
              "      <th>embarrassment</th>\n",
              "      <th>excitement</th>\n",
              "      <th>fear</th>\n",
              "      <th>gratitude</th>\n",
              "      <th>grief</th>\n",
              "      <th>joy</th>\n",
              "      <th>love</th>\n",
              "      <th>nervousness</th>\n",
              "      <th>optimism</th>\n",
              "      <th>pride</th>\n",
              "      <th>realization</th>\n",
              "      <th>relief</th>\n",
              "      <th>remorse</th>\n",
              "      <th>sadness</th>\n",
              "      <th>surprise</th>\n",
              "      <th>neutral</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>That game hurt.</td>\n",
              "      <td>eew5j0j</td>\n",
              "      <td>Brdd9</td>\n",
              "      <td>nrl</td>\n",
              "      <td>t3_ajis4z</td>\n",
              "      <td>t1_eew18eq</td>\n",
              "      <td>1.548381e+09</td>\n",
              "      <td>1</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>&gt;sexuality shouldn’t be a grouping category I...</td>\n",
              "      <td>eemcysk</td>\n",
              "      <td>TheGreen888</td>\n",
              "      <td>unpopularopinion</td>\n",
              "      <td>t3_ai4q37</td>\n",
              "      <td>t3_ai4q37</td>\n",
              "      <td>1.548084e+09</td>\n",
              "      <td>37</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>You do right, if you don't care then fuck 'em!</td>\n",
              "      <td>ed2mah1</td>\n",
              "      <td>Labalool</td>\n",
              "      <td>confessions</td>\n",
              "      <td>t3_abru74</td>\n",
              "      <td>t1_ed2m7g7</td>\n",
              "      <td>1.546428e+09</td>\n",
              "      <td>37</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Man I love reddit.</td>\n",
              "      <td>eeibobj</td>\n",
              "      <td>MrsRobertshaw</td>\n",
              "      <td>facepalm</td>\n",
              "      <td>t3_ahulml</td>\n",
              "      <td>t3_ahulml</td>\n",
              "      <td>1.547965e+09</td>\n",
              "      <td>18</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[NAME] was nowhere near them, he was by the Fa...</td>\n",
              "      <td>eda6yn6</td>\n",
              "      <td>American_Fascist713</td>\n",
              "      <td>starwarsspeculation</td>\n",
              "      <td>t3_ackt2f</td>\n",
              "      <td>t1_eda65q2</td>\n",
              "      <td>1.546669e+09</td>\n",
              "      <td>2</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  ...                                             labels\n",
              "0                                    That game hurt.  ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
              "1   >sexuality shouldn’t be a grouping category I...  ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
              "2     You do right, if you don't care then fuck 'em!  ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
              "3                                 Man I love reddit.  ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
              "4  [NAME] was nowhere near them, he was by the Fa...  ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
              "\n",
              "[5 rows x 38 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rqGlQOhlmPvw",
        "outputId": "73575db3-0c5f-4bc4-c3ee-eea3910a2239"
      },
      "source": [
        "# take sample for quick prototyping\n",
        "df_sample = df.sample(n=1000)\n",
        "df_sample.shape"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 38)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "96ElHa1KmirO",
        "outputId": "e8122b92-3b09-438c-f04f-38e9d9c8d9c5"
      },
      "source": [
        "# create train / test splits\n",
        "mask = np.random.rand(len(df)) < 0.8\n",
        "df_train = df[mask]\n",
        "df_test = df[~mask]\n",
        "\n",
        "(df_train.shape, df_test.shape)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((168844, 38), (42381, 38))"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Owa5ygouT-Dv"
      },
      "source": [
        "## Tokenize and encode "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6d869uZsT9MH"
      },
      "source": [
        "model_ckpt = \"distilbert-base-uncased\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OfGoTKyfm4jt"
      },
      "source": [
        "train_encodings = tokenizer(df_train[\"text\"].values.tolist(), truncation=True)\n",
        "test_encodings = tokenizer(df_test[\"text\"].values.tolist(), truncation=True)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uONYv-CLnZDn"
      },
      "source": [
        "train_labels = df_train[\"labels\"].values.tolist()\n",
        "test_labels = df_test[\"labels\"].values.tolist()"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z0mjklbsnS94"
      },
      "source": [
        "class GoEmotionDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u1QyX0yZnjJi"
      },
      "source": [
        "train_dataset = GoEmotionDataset(train_encodings, train_labels)\n",
        "test_dataset = GoEmotionDataset(test_encodings, test_labels)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6UgWjPEinl11",
        "outputId": "e8fafc00-fe88-448c-a443-7c45a248f2b1"
      },
      "source": [
        "train_dataset[0]"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'attention_mask': tensor([1, 1, 1, 1, 1, 1]),\n",
              " 'input_ids': tensor([ 101, 2008, 2208, 3480, 1012,  102]),\n",
              " 'labels': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 1, 0, 0])}"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "vBlN3W0xnpgx",
        "outputId": "b5ab4a8b-275c-4f08-afb9-0c4cfce33e39"
      },
      "source": [
        "# sanity check\n",
        "tokenizer.decode(train_dataset[0][\"input_ids\"])"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'[CLS] that game hurt. [SEP]'"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dvLh--IvX8sF"
      },
      "source": [
        "## Fine-tuning\n",
        "\n",
        "There are two ways we can implement multi-label classification:\n",
        "\n",
        "* Creating a custom BERT model that overrides the `forward` method\n",
        "* Creating a custom `Trainer` that overrides the `compute_loss` method\n",
        "\n",
        "The second method does not work with v4.2.1 of `transformers` due to some bugs, so we'll work with the first approach instead :)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4mO5wge2mEs2"
      },
      "source": [
        "### Creating a Custom Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XZEN8MhaL54M"
      },
      "source": [
        "class DistilBertForMultilabelSequenceClassification(DistilBertForSequenceClassification):\n",
        "    def __init__(self, config):\n",
        "      super().__init__(config)\n",
        "\n",
        "    def forward(self,\n",
        "        input_ids=None,\n",
        "        attention_mask=None,\n",
        "        head_mask=None,\n",
        "        inputs_embeds=None,\n",
        "        labels=None,\n",
        "        output_attentions=None,\n",
        "        output_hidden_states=None,\n",
        "        return_dict=None):\n",
        "        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n",
        "\n",
        "        outputs = self.distilbert(input_ids,\n",
        "            attention_mask=attention_mask,\n",
        "            head_mask=head_mask,\n",
        "            inputs_embeds=inputs_embeds,\n",
        "            output_attentions=output_attentions,\n",
        "            output_hidden_states=output_hidden_states,\n",
        "            return_dict=return_dict)\n",
        "\n",
        "        hidden_state = outputs[0]\n",
        "        pooled_output = hidden_state[:, 0]  \n",
        "        pooled_output = self.dropout(pooled_output)\n",
        "        logits = self.classifier(pooled_output)\n",
        "\n",
        "        loss = None\n",
        "        if labels is not None:\n",
        "            loss_fct = torch.nn.BCEWithLogitsLoss()\n",
        "            loss = loss_fct(logits.view(-1, self.num_labels), \n",
        "                            labels.float().view(-1, self.num_labels))\n",
        "\n",
        "        if not return_dict:\n",
        "            output = (logits,) + outputs[2:]\n",
        "            return ((loss,) + output) if loss is not None else output\n",
        "\n",
        "        return SequenceClassifierOutput(loss=loss,\n",
        "            logits=logits,\n",
        "            hidden_states=outputs.hidden_states,\n",
        "            attentions=outputs.attentions)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4c81NkyZYCab",
        "outputId": "29a743b7-54f6-4b16-9d76-1cbaac24d84d"
      },
      "source": [
        "num_labels=28\n",
        "model = DistilBertForMultilabelSequenceClassification.from_pretrained(model_ckpt, num_labels=num_labels).to('cuda')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForMultilabelSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_layer_norm.bias', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertForMultilabelSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForMultilabelSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of DistilBertForMultilabelSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89fCFR3kmfAM",
        "outputId": "6de43bd6-80e8-40f0-84a1-bdc28978e0e0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "model.config.id2label = {\n",
        "    \"0\": \"admiration\",\n",
        "    \"1\": \"amusement\",\n",
        "    \"2\": \"anger\",\n",
        "    \"3\": \"annoyance\",\n",
        "    \"4\": \"approval\",\n",
        "    \"5\": \"caring\",\n",
        "    \"6\": \"confusion\",\n",
        "    \"7\": \"curiosity\",\n",
        "    \"8\": \"desire\",\n",
        "    \"9\": \"disappointment\",\n",
        "    \"10\": \"disapproval\",\n",
        "    \"11\": \"disgust\",\n",
        "    \"12\": \"embarrassment\",\n",
        "    \"13\": \"excitement\",\n",
        "    \"14\": \"fear\",\n",
        "    \"15\": \"gratitude\",\n",
        "    \"16\": \"grief\",\n",
        "    \"17\": \"joy\",\n",
        "    \"18\": \"love\",\n",
        "    \"19\": \"nervousness\",\n",
        "    \"20\": \"optimism\",\n",
        "    \"21\": \"pride\",\n",
        "    \"22\": \"realization\",\n",
        "    \"23\": \"relief\",\n",
        "    \"24\": \"remorse\",\n",
        "    \"25\": \"sadness\",\n",
        "    \"26\": \"surprise\",\n",
        "    \"27\": \"neutral\"\n",
        "  },\n",
        "model.config.label2id ={\n",
        "    \"admiration\": 0,\n",
        "    \"amusement\": 1,\n",
        "    \"anger\": 2,\n",
        "    \"annoyance\": 3,\n",
        "    \"approval\": 4,\n",
        "    \"caring\": 5,\n",
        "    \"confusion\": 6,\n",
        "    \"curiosity\": 7,\n",
        "    \"desire\": 8,\n",
        "    \"disappointment\": 9,\n",
        "    \"disapproval\": 10,\n",
        "    \"disgust\": 11,\n",
        "    \"embarrassment\": 12,\n",
        "    \"excitement\": 13,\n",
        "    \"fear\": 14,\n",
        "    \"gratitude\": 15,\n",
        "    \"grief\": 16,\n",
        "    \"joy\": 17,\n",
        "    \"love\": 18,\n",
        "    \"nervousness\": 19,\n",
        "    \"neutral\": 27,\n",
        "    \"optimism\": 20,\n",
        "    \"pride\": 21,\n",
        "    \"realization\": 22,\n",
        "    \"relief\": 23,\n",
        "    \"remorse\": 24,\n",
        "    \"sadness\": 25,\n",
        "    \"surprise\": 26\n",
        "  }\n",
        "model.config"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DistilBertConfig {\n",
              "  \"_name_or_path\": \"distilbert-base-uncased\",\n",
              "  \"activation\": \"gelu\",\n",
              "  \"architectures\": [\n",
              "    \"DistilBertForMaskedLM\"\n",
              "  ],\n",
              "  \"attention_dropout\": 0.1,\n",
              "  \"dim\": 768,\n",
              "  \"dropout\": 0.1,\n",
              "  \"hidden_dim\": 3072,\n",
              "  \"id2label\": [\n",
              "    {\n",
              "      \"0\": \"admiration\",\n",
              "      \"1\": \"amusement\",\n",
              "      \"10\": \"disapproval\",\n",
              "      \"11\": \"disgust\",\n",
              "      \"12\": \"embarrassment\",\n",
              "      \"13\": \"excitement\",\n",
              "      \"14\": \"fear\",\n",
              "      \"15\": \"gratitude\",\n",
              "      \"16\": \"grief\",\n",
              "      \"17\": \"joy\",\n",
              "      \"18\": \"love\",\n",
              "      \"19\": \"nervousness\",\n",
              "      \"2\": \"anger\",\n",
              "      \"20\": \"optimism\",\n",
              "      \"21\": \"pride\",\n",
              "      \"22\": \"realization\",\n",
              "      \"23\": \"relief\",\n",
              "      \"24\": \"remorse\",\n",
              "      \"25\": \"sadness\",\n",
              "      \"26\": \"surprise\",\n",
              "      \"27\": \"neutral\",\n",
              "      \"3\": \"annoyance\",\n",
              "      \"4\": \"approval\",\n",
              "      \"5\": \"caring\",\n",
              "      \"6\": \"confusion\",\n",
              "      \"7\": \"curiosity\",\n",
              "      \"8\": \"desire\",\n",
              "      \"9\": \"disappointment\"\n",
              "    }\n",
              "  ],\n",
              "  \"initializer_range\": 0.02,\n",
              "  \"label2id\": {\n",
              "    \"admiration\": 0,\n",
              "    \"amusement\": 1,\n",
              "    \"anger\": 2,\n",
              "    \"annoyance\": 3,\n",
              "    \"approval\": 4,\n",
              "    \"caring\": 5,\n",
              "    \"confusion\": 6,\n",
              "    \"curiosity\": 7,\n",
              "    \"desire\": 8,\n",
              "    \"disappointment\": 9,\n",
              "    \"disapproval\": 10,\n",
              "    \"disgust\": 11,\n",
              "    \"embarrassment\": 12,\n",
              "    \"excitement\": 13,\n",
              "    \"fear\": 14,\n",
              "    \"gratitude\": 15,\n",
              "    \"grief\": 16,\n",
              "    \"joy\": 17,\n",
              "    \"love\": 18,\n",
              "    \"nervousness\": 19,\n",
              "    \"neutral\": 27,\n",
              "    \"optimism\": 20,\n",
              "    \"pride\": 21,\n",
              "    \"realization\": 22,\n",
              "    \"relief\": 23,\n",
              "    \"remorse\": 24,\n",
              "    \"sadness\": 25,\n",
              "    \"surprise\": 26\n",
              "  },\n",
              "  \"max_position_embeddings\": 512,\n",
              "  \"model_type\": \"distilbert\",\n",
              "  \"n_heads\": 12,\n",
              "  \"n_layers\": 6,\n",
              "  \"pad_token_id\": 0,\n",
              "  \"qa_dropout\": 0.1,\n",
              "  \"seq_classif_dropout\": 0.2,\n",
              "  \"sinusoidal_pos_embds\": false,\n",
              "  \"tie_weights_\": true,\n",
              "  \"transformers_version\": \"4.12.5\",\n",
              "  \"vocab_size\": 30522\n",
              "}"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fHt7W1ljoZu6"
      },
      "source": [
        "def accuracy_thresh(y_pred, y_true, thresh=0.5, sigmoid=True): \n",
        "    y_pred = torch.from_numpy(y_pred)\n",
        "    y_true = torch.from_numpy(y_true)\n",
        "    if sigmoid: \n",
        "      y_pred = y_pred.sigmoid()\n",
        "    return ((y_pred>thresh)==y_true.bool()).float().mean().item()"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q0K6faAuogef"
      },
      "source": [
        "def compute_metrics(eval_pred):\n",
        "    predictions, labels = eval_pred\n",
        "    return {'accuracy_thresh': accuracy_thresh(predictions, labels)}"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MF_-WbOtoJa-"
      },
      "source": [
        "batch_size = 32\n",
        "# configure logging so we see training loss\n",
        "logging_steps = len(train_dataset) // batch_size\n",
        "\n",
        "args = TrainingArguments(\n",
        "    output_dir=\"emotion\",\n",
        "    evaluation_strategy = \"epoch\",\n",
        "    save_strategy='epoch',\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=batch_size,\n",
        "    per_device_eval_batch_size=batch_size,\n",
        "    num_train_epochs=3,\n",
        "    weight_decay=0.01,\n",
        "    logging_steps=logging_steps\n",
        ")"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zI10-X_YowwH"
      },
      "source": [
        "trainer = Trainer(\n",
        "    model,\n",
        "    args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=test_dataset,\n",
        "    compute_metrics=compute_metrics,\n",
        "    tokenizer=tokenizer)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "id": "CpfAvgcDNaRu",
        "outputId": "f3fa6c23-19b3-4c72-b91a-ca50145e2999"
      },
      "source": [
        "# sanity check that we can run evaluation\n",
        "trainer.evaluate()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 42381\n",
            "  Batch size = 32\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1325' max='1325' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1325/1325 01:35]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'eval_accuracy_thresh': 0.4506003260612488,\n",
              " 'eval_loss': 0.7234644889831543,\n",
              " 'eval_runtime': 95.663,\n",
              " 'eval_samples_per_second': 443.024,\n",
              " 'eval_steps_per_second': 13.851}"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 852
        },
        "id": "m2tvwbqaz8dv",
        "outputId": "b0124897-cca4-4e16-d35b-802ab1f6a1bd"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running training *****\n",
            "  Num examples = 168844\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 32\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 15831\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='15831' max='15831' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [15831/15831 1:07:07, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy Thresh</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.126200</td>\n",
              "      <td>0.113293</td>\n",
              "      <td>0.961871</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.110000</td>\n",
              "      <td>0.110863</td>\n",
              "      <td>0.962302</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.105500</td>\n",
              "      <td>0.110903</td>\n",
              "      <td>0.962024</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 42381\n",
            "  Batch size = 32\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2650' max='1325' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1325/1325 24:08]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to emotion/checkpoint-5277\n",
            "Configuration saved in emotion/checkpoint-5277/config.json\n",
            "Model weights saved in emotion/checkpoint-5277/pytorch_model.bin\n",
            "tokenizer config file saved in emotion/checkpoint-5277/tokenizer_config.json\n",
            "Special tokens file saved in emotion/checkpoint-5277/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 42381\n",
            "  Batch size = 32\n",
            "Saving model checkpoint to emotion/checkpoint-10554\n",
            "Configuration saved in emotion/checkpoint-10554/config.json\n",
            "Model weights saved in emotion/checkpoint-10554/pytorch_model.bin\n",
            "tokenizer config file saved in emotion/checkpoint-10554/tokenizer_config.json\n",
            "Special tokens file saved in emotion/checkpoint-10554/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 42381\n",
            "  Batch size = 32\n",
            "Saving model checkpoint to emotion/checkpoint-15831\n",
            "Configuration saved in emotion/checkpoint-15831/config.json\n",
            "Model weights saved in emotion/checkpoint-15831/pytorch_model.bin\n",
            "tokenizer config file saved in emotion/checkpoint-15831/tokenizer_config.json\n",
            "Special tokens file saved in emotion/checkpoint-15831/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=15831, training_loss=0.11387918509534, metrics={'train_runtime': 4028.2919, 'train_samples_per_second': 125.744, 'train_steps_per_second': 3.93, 'total_flos': 4792093044377088.0, 'train_loss': 0.11387918509534, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8v6zaHbGlSS9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "outputId": "612390e0-cd19-430c-fd0a-0747d58c32ad"
      },
      "source": [
        "# sanity check that we can run evaluation\n",
        "trainer.evaluate()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 42381\n",
            "  Batch size = 32\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1325' max='1325' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1325/1325 01:36]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'epoch': 3.0,\n",
              " 'eval_accuracy_thresh': 0.962023913860321,\n",
              " 'eval_loss': 0.11090277135372162,\n",
              " 'eval_runtime': 97.0732,\n",
              " 'eval_samples_per_second': 436.588,\n",
              " 'eval_steps_per_second': 13.649}"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7BD1KSWcJsEU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1db9fbfe-49e7-43bb-e872-8d39d4215603"
      },
      "source": [
        "!transformers-cli login"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m\u001b[31mWARNING! `transformers-cli login` is deprecated and will be removed in v5. Please use `huggingface-cli login` instead.\u001b[0m\n",
            "\n",
            "        _|    _|  _|    _|    _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|_|_|_|    _|_|      _|_|_|  _|_|_|_|\n",
            "        _|    _|  _|    _|  _|        _|          _|    _|_|    _|  _|            _|        _|    _|  _|        _|\n",
            "        _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|\n",
            "        _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|\n",
            "        _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|\n",
            "\n",
            "        \n",
            "Username: bhadresh-savani\n",
            "Password: \n",
            "Login successful\n",
            "Your token: USSrKekMAIdNYfcxwutSzdOSvEXmxQJcQWvJIQYlcxLUEOdLuBlwYFYRPUksnUhbBXDghVggoqQhdHoXexdlwHWVyytklJLRARIxPrkYfecZgatctBQwalZlMrMVWVyG \n",
            "\n",
            "Your token has been saved to /root/.huggingface/token\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A_KDDGcBOAvu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4eb89e45-7793-436e-985c-6f83e4a95aea"
      },
      "source": [
        "!sudo apt-get install git-lfs"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  git-lfs\n",
            "0 upgraded, 1 newly installed, 0 to remove and 37 not upgraded.\n",
            "Need to get 2,129 kB of archives.\n",
            "After this operation, 7,662 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 git-lfs amd64 2.3.4-1 [2,129 kB]\n",
            "Fetched 2,129 kB in 1s (1,808 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 1.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package git-lfs.\n",
            "(Reading database ... 155222 files and directories currently installed.)\n",
            "Preparing to unpack .../git-lfs_2.3.4-1_amd64.deb ...\n",
            "Unpacking git-lfs (2.3.4-1) ...\n",
            "Setting up git-lfs (2.3.4-1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-VJZJfEOnNH"
      },
      "source": [
        "!git config --global user.email \"bhadreshpsavani@gmail.com\"\n",
        "!git config --global user.name \"bhadresh-savani\"\n",
        "!git config --global user.password \"#####\""
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NP2pyaW6PBk0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 244,
          "referenced_widgets": [
            "1ea376d31b0942c58301973b778d4b74",
            "06798b45bc3d486a9bc7b9b5e8c65fae",
            "c7d651016ca64488b4897456a353ef0c",
            "13b4483bbc364fcea90820ec93e2cde9",
            "94ef6bdd3f0842a79de673caa896d0c7",
            "2637c6d539724f27bb4e01ed8e26b120",
            "e2dda27caab74038bf7815606c168e5f",
            "84bf702e3fef43aaba8e83df0e1c8e28",
            "33d5f793a357444a8852f4443d1f4587",
            "e25dc60ee9414028a67c04c7ff709ad9",
            "c8f21770d07a47e28eebfba384a7121a"
          ]
        },
        "outputId": "402ef341-0bbe-4d0d-b8e0-ed4a9f9763d0"
      },
      "source": [
        "trainer.model.push_to_hub('distilbert-base-uncased-go-emotion')"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/huggingface_hub/hf_api.py:718: FutureWarning: `create_repo` now takes `token` as an optional positional argument. Be sure to adapt your code!\n",
            "  FutureWarning,\n",
            "Cloning https://huggingface.co/bhadresh-savani/distilbert-base-uncased-go-emotion into local empty directory.\n",
            "Configuration saved in distilbert-base-uncased-go-emotion/config.json\n",
            "Model weights saved in distilbert-base-uncased-go-emotion/pytorch_model.bin\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1ea376d31b0942c58301973b778d4b74",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Upload file pytorch_model.bin:   0%|          | 3.36k/256M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "To https://huggingface.co/bhadresh-savani/distilbert-base-uncased-go-emotion\n",
            "   f6496b4..8a54d32  main -> main\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'https://huggingface.co/bhadresh-savani/distilbert-base-uncased-go-emotion/commit/8a54d32eda0d6a25c3aabb08c4ffa907f54b3d5f'"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MpKTSsRXPTqT",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "480d31e0-1679-4dea-dc73-00eeb8edb39f"
      },
      "source": [
        "trainer.tokenizer.push_to_hub('distilbert-base-uncased-go-emotion')"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "tokenizer config file saved in distilbert-base-uncased-go-emotion/tokenizer_config.json\n",
            "Special tokens file saved in distilbert-base-uncased-go-emotion/special_tokens_map.json\n",
            "To https://huggingface.co/bhadresh-savani/distilbert-base-uncased-go-emotion\n",
            "   8a54d32..8e86a59  main -> main\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'https://huggingface.co/bhadresh-savani/distilbert-base-uncased-go-emotion/commit/8e86a597d46ce7e42b16ba8dd9ce516d5279d6f0'"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUDdPH2_j1Rv"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}