hugging_electra_fine_tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hugging-electra-fine-tuning",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6897eefdf2a449e2955dadffd2030e3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d61d5d31272943869bfdf2e40919f827",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_53fcc3b4d66e426f889585a62fb04f39",
              "IPY_MODEL_e6ac1cf637c442258223ff20be546dae"
            ]
          }
        },
        "d61d5d31272943869bfdf2e40919f827": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "53fcc3b4d66e426f889585a62fb04f39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ac356b3e959e437f92d742146d716f65",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e34743995aa45048f72258a5ec5324d"
          }
        },
        "e6ac1cf637c442258223ff20be546dae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_865043bcbc464be08326c467ee8c00cd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [07:17&lt;00:00, 145.89s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_efc077ca358644c68b81c59104ef3d07"
          }
        },
        "ac356b3e959e437f92d742146d716f65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e34743995aa45048f72258a5ec5324d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "865043bcbc464be08326c467ee8c00cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "efc077ca358644c68b81c59104ef3d07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "tAJRr-QJQ3xd"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZxBiKEOBb9ya"
      },
      "source": [
        "import torch\n",
        "import pandas as pd\n",
        "from transformers import ElectraForSequenceClassification, ElectraTokenizerFast"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LFxcN_TeSVjh"
      },
      "source": [
        "df = pd.read_csv('/content/drive/MyDrive/propaganda-on-social-media-v1.2.csv', index_col=None)\n",
        "\n",
        "# df[\"label\"] = df.category.apply(lambda cat: 1 if cat == 'Discrediting Pages' else 2 if cat == 'Ideologized Propaganda ' else 3 if cat == 'False Media Pages' else 4 if cat == 'False Supporting Pages' else 5 if cat == 'Government Propaganda' else 0)\n",
        "df[\"label\"] = df.category.apply(lambda cat: 1 if cat == 'Ideologized Propaganda ' else 0)\n",
        "\n",
        "df_ = pd.concat([df[df.label == 1], df[df.label == 0].sample(15000)])\n",
        "\n",
        "sentences = df_.post.values\n",
        "labels = df_.label.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lri-V5jF3ft9"
      },
      "source": [
        "tokenizer = ElectraTokenizerFast.from_pretrained(\"jnz/electra-ka-fake-news-tagging\", do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dKVqb4hqjcmV"
      },
      "source": [
        "config = {\n",
        "    'learning_rate': 5e-5,\n",
        "    'batch_size': 16,\n",
        "    'epochs':3\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "szLbS4wCSxhY"
      },
      "source": [
        "# Tonekize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sB5DjEmqf7KY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0d9d267-466b-4ad3-d78f-866526a337ab"
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 128,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "    \n",
        "    # Add the encoded sentence to the list.    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    \n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor(labels)\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Original:  დიდი ხნის შემდეგ ვბრუნდებით მაგრამ თქვენთვის ცუდი ამბები გვაქვს რადგან გვინდა გაჩვენოთ ადამიანები რომლებსაც ჰგონიათ რომ საქართველოს ისტორია სულ რაღაც ცხრა წელს ითვლის\n",
            "Token IDs: tensor([62001,  3804, 29338,  8389, 62000,   804,  3293,   982, 14902,  1390,\n",
            "          540, 38480,   307, 24418,  6849,  6656,    68,     3, 62000, 23985,\n",
            "        60435,  1770, 29778,   794,  5787, 33140,  4473, 10657, 32229, 36035,\n",
            "          228,   428, 10941, 62003, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999,\n",
            "        61999, 61999, 61999, 61999, 61999, 61999, 61999, 61999])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lIS1rTmLS1la"
      },
      "source": [
        "# Train-Test Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FlR_Qmhlf_bk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d45d4d24-2549-4e04-92b4-fa15dd04ff32"
      },
      "source": [
        "from torch.utils.data import TensorDataset, random_split\n",
        "\n",
        "# Combine the training inputs into a TensorDataset.\n",
        "dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "\n",
        "# Create a 90-10 train-validation split.\n",
        "\n",
        "# Calculate the number of samples to include in each set.\n",
        "train_size = int(0.9 * len(dataset))\n",
        "val_size = len(dataset) - train_size\n",
        "\n",
        "# Divide the dataset by randomly selecting samples.\n",
        "train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
        "\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print('{:>5,} validation samples'.format(val_size))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "26,187 training samples\n",
            "2,910 validation samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z0gDjZm5gI9E"
      },
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "# import wandb\n",
        "# WANDB PARAMETER\n",
        "def ret_dataloader():\n",
        "    batch_size = config[\"batch_size\"]\n",
        "    print('batch_size = ', batch_size)\n",
        "    train_dataloader = DataLoader(\n",
        "                train_dataset,  # The training samples.\n",
        "                sampler = RandomSampler(train_dataset), # Select batches randomly\n",
        "                batch_size = batch_size # Trains with this batch size.\n",
        "            )\n",
        "\n",
        "    validation_dataloader = DataLoader(\n",
        "                val_dataset, # The validation samples.\n",
        "                sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
        "                batch_size = batch_size # Evaluate with this batch size.\n",
        "            )\n",
        "    return train_dataloader,validation_dataloader"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zUH_e0tOS6Vq"
      },
      "source": [
        "# Load Pre-trained BERT model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UjchAuWsgQ3e"
      },
      "source": [
        "from transformers import ElectraForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "def ret_model():\n",
        "    model = ElectraForSequenceClassification.from_pretrained(\"jnz/electra-ka-fake-news-tagging\", \n",
        "                                                             num_labels = 2)\n",
        "                                                            #  output_attentions = False, \n",
        "                                                            #  output_hidden_states = False )\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_aJq1nnsg83t"
      },
      "source": [
        "def ret_optim(model):\n",
        "    print('Learning_rate = ', config[\"learning_rate\"] )\n",
        "    optimizer = AdamW(model.parameters(),\n",
        "                      lr = config[\"learning_rate\"], \n",
        "                      eps = 1e-8 \n",
        "                    )\n",
        "    return optimizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q1lahCKxhChw"
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "def ret_scheduler(train_dataloader,optimizer):\n",
        "    epochs = config[\"epochs\"]\n",
        "    print('epochs =>', epochs)\n",
        "    # Total number of training steps is [number of batches] x [number of epochs]. \n",
        "    # (Note that this is not the same as the number of training samples).\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    # Create the learning rate scheduler.\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                                num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                                num_training_steps = total_steps)\n",
        "    return scheduler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0eSTULg4hNrB"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ogiISYYqnGaB"
      },
      "source": [
        "torch.multiprocessing.set_start_method('spawn', force=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dksu3fWPkWiG"
      },
      "source": [
        "#Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bTrJ84Y-kVO5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "6897eefdf2a449e2955dadffd2030e3e",
            "d61d5d31272943869bfdf2e40919f827",
            "53fcc3b4d66e426f889585a62fb04f39",
            "e6ac1cf637c442258223ff20be546dae",
            "ac356b3e959e437f92d742146d716f65",
            "0e34743995aa45048f72258a5ec5324d",
            "865043bcbc464be08326c467ee8c00cd",
            "efc077ca358644c68b81c59104ef3d07"
          ]
        },
        "outputId": "e0f677c3-1085-450b-befe-b3fec411563d"
      },
      "source": [
        "from tqdm import tqdm_notebook as tqdm\n",
        "import random\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)\n",
        "model = ret_model()\n",
        "model.to(device)\n",
        "#wandb.init(config=sweep_defaults)\n",
        "train_dataloader,validation_dataloader = ret_dataloader()\n",
        "optimizer = ret_optim(model)\n",
        "scheduler = ret_scheduler(train_dataloader,optimizer)\n",
        "\n",
        "#print(\"config \",wandb.config.learning_rate, \"\\n\",wandb.config)\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "#torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# We'll store a number of quantities such as training and validation loss, \n",
        "# validation accuracy, and timings.\n",
        "training_stats = []\n",
        "\n",
        "# Measure the total training time for the whole run.\n",
        "total_t0 = time.time()\n",
        "epochs = config[\"epochs\"]\n",
        "# For each epoch...\n",
        "for epoch_i in tqdm(range(0, epochs)):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_train_loss = 0\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to \n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because \n",
        "        # accumulating the gradients is \"convenient while training RNNs\". \n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()        \n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # The documentation for this `model` function is here: \n",
        "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "        # It returns different numbers of parameters depending on what arguments\n",
        "        # arge given and what flags are set. For our useage here, it returns\n",
        "        # the loss (because we provided labels) and the \"logits\"--the model\n",
        "        # outputs prior to activation.\n",
        "        loss, logits = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask, \n",
        "                            labels=b_labels)\n",
        "        # wandb.log({'train_batch_loss':loss.item()})\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value \n",
        "        # from the tensor.\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "    \n",
        "    # Measure how long this epoch took.\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    # wandb.log({'avg_train_loss':avg_train_loss})\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "        \n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using \n",
        "        # the `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].cuda()\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        \n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():        \n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # token_type_ids is the same as the \"segment ids\", which \n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            # The documentation for this `model` function is here: \n",
        "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "            # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "            # values prior to applying an activation function like the softmax.\n",
        "            (loss, logits) = model(b_input_ids, \n",
        "                                    token_type_ids=None, \n",
        "                                    attention_mask=b_input_mask,\n",
        "                                    labels=b_labels)\n",
        "            \n",
        "        # Accumulate the validation loss.\n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences, and\n",
        "        # accumulate it over all batches.\n",
        "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
        "        \n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "    \n",
        "    # Measure how long the validation run took.\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    # wandb.log({'val_accuracy':avg_val_accuracy,'avg_val_loss':avg_val_loss})\n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at /content/electra/ were not used when initializing ElectraForSequenceClassification: ['generator.embeddings.word_embeddings.weight', 'generator.embeddings.position_embeddings.weight', 'generator.embeddings.token_type_embeddings.weight', 'generator.embeddings.LayerNorm.weight', 'generator.embeddings.LayerNorm.bias', 'generator.encoder.layer.0.attention.self.query.weight', 'generator.encoder.layer.0.attention.self.query.bias', 'generator.encoder.layer.0.attention.self.key.weight', 'generator.encoder.layer.0.attention.self.key.bias', 'generator.encoder.layer.0.attention.self.value.weight', 'generator.encoder.layer.0.attention.self.value.bias', 'generator.encoder.layer.0.attention.output.dense.weight', 'generator.encoder.layer.0.attention.output.dense.bias', 'generator.encoder.layer.0.attention.output.LayerNorm.weight', 'generator.encoder.layer.0.attention.output.LayerNorm.bias', 'generator.encoder.layer.0.intermediate.dense.weight', 'generator.encoder.layer.0.intermediate.dense.bias', 'generator.encoder.layer.0.output.dense.weight', 'generator.encoder.layer.0.output.dense.bias', 'generator.encoder.layer.0.output.LayerNorm.weight', 'generator.encoder.layer.0.output.LayerNorm.bias', 'generator.encoder.layer.1.attention.self.query.weight', 'generator.encoder.layer.1.attention.self.query.bias', 'generator.encoder.layer.1.attention.self.key.weight', 'generator.encoder.layer.1.attention.self.key.bias', 'generator.encoder.layer.1.attention.self.value.weight', 'generator.encoder.layer.1.attention.self.value.bias', 'generator.encoder.layer.1.attention.output.dense.weight', 'generator.encoder.layer.1.attention.output.dense.bias', 'generator.encoder.layer.1.attention.output.LayerNorm.weight', 'generator.encoder.layer.1.attention.output.LayerNorm.bias', 'generator.encoder.layer.1.intermediate.dense.weight', 'generator.encoder.layer.1.intermediate.dense.bias', 'generator.encoder.layer.1.output.dense.weight', 'generator.encoder.layer.1.output.dense.bias', 'generator.encoder.layer.1.output.LayerNorm.weight', 'generator.encoder.layer.1.output.LayerNorm.bias', 'generator.encoder.layer.2.attention.self.query.weight', 'generator.encoder.layer.2.attention.self.query.bias', 'generator.encoder.layer.2.attention.self.key.weight', 'generator.encoder.layer.2.attention.self.key.bias', 'generator.encoder.layer.2.attention.self.value.weight', 'generator.encoder.layer.2.attention.self.value.bias', 'generator.encoder.layer.2.attention.output.dense.weight', 'generator.encoder.layer.2.attention.output.dense.bias', 'generator.encoder.layer.2.attention.output.LayerNorm.weight', 'generator.encoder.layer.2.attention.output.LayerNorm.bias', 'generator.encoder.layer.2.intermediate.dense.weight', 'generator.encoder.layer.2.intermediate.dense.bias', 'generator.encoder.layer.2.output.dense.weight', 'generator.encoder.layer.2.output.dense.bias', 'generator.encoder.layer.2.output.LayerNorm.weight', 'generator.encoder.layer.2.output.LayerNorm.bias', 'generator.encoder.layer.3.attention.self.query.weight', 'generator.encoder.layer.3.attention.self.query.bias', 'generator.encoder.layer.3.attention.self.key.weight', 'generator.encoder.layer.3.attention.self.key.bias', 'generator.encoder.layer.3.attention.self.value.weight', 'generator.encoder.layer.3.attention.self.value.bias', 'generator.encoder.layer.3.attention.output.dense.weight', 'generator.encoder.layer.3.attention.output.dense.bias', 'generator.encoder.layer.3.attention.output.LayerNorm.weight', 'generator.encoder.layer.3.attention.output.LayerNorm.bias', 'generator.encoder.layer.3.intermediate.dense.weight', 'generator.encoder.layer.3.intermediate.dense.bias', 'generator.encoder.layer.3.output.dense.weight', 'generator.encoder.layer.3.output.dense.bias', 'generator.encoder.layer.3.output.LayerNorm.weight', 'generator.encoder.layer.3.output.LayerNorm.bias', 'generator.encoder.layer.4.attention.self.query.weight', 'generator.encoder.layer.4.attention.self.query.bias', 'generator.encoder.layer.4.attention.self.key.weight', 'generator.encoder.layer.4.attention.self.key.bias', 'generator.encoder.layer.4.attention.self.value.weight', 'generator.encoder.layer.4.attention.self.value.bias', 'generator.encoder.layer.4.attention.output.dense.weight', 'generator.encoder.layer.4.attention.output.dense.bias', 'generator.encoder.layer.4.attention.output.LayerNorm.weight', 'generator.encoder.layer.4.attention.output.LayerNorm.bias', 'generator.encoder.layer.4.intermediate.dense.weight', 'generator.encoder.layer.4.intermediate.dense.bias', 'generator.encoder.layer.4.output.dense.weight', 'generator.encoder.layer.4.output.dense.bias', 'generator.encoder.layer.4.output.LayerNorm.weight', 'generator.encoder.layer.4.output.LayerNorm.bias', 'generator.encoder.layer.5.attention.self.query.weight', 'generator.encoder.layer.5.attention.self.query.bias', 'generator.encoder.layer.5.attention.self.key.weight', 'generator.encoder.layer.5.attention.self.key.bias', 'generator.encoder.layer.5.attention.self.value.weight', 'generator.encoder.layer.5.attention.self.value.bias', 'generator.encoder.layer.5.attention.output.dense.weight', 'generator.encoder.layer.5.attention.output.dense.bias', 'generator.encoder.layer.5.attention.output.LayerNorm.weight', 'generator.encoder.layer.5.attention.output.LayerNorm.bias', 'generator.encoder.layer.5.intermediate.dense.weight', 'generator.encoder.layer.5.intermediate.dense.bias', 'generator.encoder.layer.5.output.dense.weight', 'generator.encoder.layer.5.output.dense.bias', 'generator.encoder.layer.5.output.LayerNorm.weight', 'generator.encoder.layer.5.output.LayerNorm.bias', 'generator.encoder.layer.6.attention.self.query.weight', 'generator.encoder.layer.6.attention.self.query.bias', 'generator.encoder.layer.6.attention.self.key.weight', 'generator.encoder.layer.6.attention.self.key.bias', 'generator.encoder.layer.6.attention.self.value.weight', 'generator.encoder.layer.6.attention.self.value.bias', 'generator.encoder.layer.6.attention.output.dense.weight', 'generator.encoder.layer.6.attention.output.dense.bias', 'generator.encoder.layer.6.attention.output.LayerNorm.weight', 'generator.encoder.layer.6.attention.output.LayerNorm.bias', 'generator.encoder.layer.6.intermediate.dense.weight', 'generator.encoder.layer.6.intermediate.dense.bias', 'generator.encoder.layer.6.output.dense.weight', 'generator.encoder.layer.6.output.dense.bias', 'generator.encoder.layer.6.output.LayerNorm.weight', 'generator.encoder.layer.6.output.LayerNorm.bias', 'generator.encoder.layer.7.attention.self.query.weight', 'generator.encoder.layer.7.attention.self.query.bias', 'generator.encoder.layer.7.attention.self.key.weight', 'generator.encoder.layer.7.attention.self.key.bias', 'generator.encoder.layer.7.attention.self.value.weight', 'generator.encoder.layer.7.attention.self.value.bias', 'generator.encoder.layer.7.attention.output.dense.weight', 'generator.encoder.layer.7.attention.output.dense.bias', 'generator.encoder.layer.7.attention.output.LayerNorm.weight', 'generator.encoder.layer.7.attention.output.LayerNorm.bias', 'generator.encoder.layer.7.intermediate.dense.weight', 'generator.encoder.layer.7.intermediate.dense.bias', 'generator.encoder.layer.7.output.dense.weight', 'generator.encoder.layer.7.output.dense.bias', 'generator.encoder.layer.7.output.LayerNorm.weight', 'generator.encoder.layer.7.output.LayerNorm.bias', 'generator.encoder.layer.8.attention.self.query.weight', 'generator.encoder.layer.8.attention.self.query.bias', 'generator.encoder.layer.8.attention.self.key.weight', 'generator.encoder.layer.8.attention.self.key.bias', 'generator.encoder.layer.8.attention.self.value.weight', 'generator.encoder.layer.8.attention.self.value.bias', 'generator.encoder.layer.8.attention.output.dense.weight', 'generator.encoder.layer.8.attention.output.dense.bias', 'generator.encoder.layer.8.attention.output.LayerNorm.weight', 'generator.encoder.layer.8.attention.output.LayerNorm.bias', 'generator.encoder.layer.8.intermediate.dense.weight', 'generator.encoder.layer.8.intermediate.dense.bias', 'generator.encoder.layer.8.output.dense.weight', 'generator.encoder.layer.8.output.dense.bias', 'generator.encoder.layer.8.output.LayerNorm.weight', 'generator.encoder.layer.8.output.LayerNorm.bias', 'generator.encoder.layer.9.attention.self.query.weight', 'generator.encoder.layer.9.attention.self.query.bias', 'generator.encoder.layer.9.attention.self.key.weight', 'generator.encoder.layer.9.attention.self.key.bias', 'generator.encoder.layer.9.attention.self.value.weight', 'generator.encoder.layer.9.attention.self.value.bias', 'generator.encoder.layer.9.attention.output.dense.weight', 'generator.encoder.layer.9.attention.output.dense.bias', 'generator.encoder.layer.9.attention.output.LayerNorm.weight', 'generator.encoder.layer.9.attention.output.LayerNorm.bias', 'generator.encoder.layer.9.intermediate.dense.weight', 'generator.encoder.layer.9.intermediate.dense.bias', 'generator.encoder.layer.9.output.dense.weight', 'generator.encoder.layer.9.output.dense.bias', 'generator.encoder.layer.9.output.LayerNorm.weight', 'generator.encoder.layer.9.output.LayerNorm.bias', 'generator.encoder.layer.10.attention.self.query.weight', 'generator.encoder.layer.10.attention.self.query.bias', 'generator.encoder.layer.10.attention.self.key.weight', 'generator.encoder.layer.10.attention.self.key.bias', 'generator.encoder.layer.10.attention.self.value.weight', 'generator.encoder.layer.10.attention.self.value.bias', 'generator.encoder.layer.10.attention.output.dense.weight', 'generator.encoder.layer.10.attention.output.dense.bias', 'generator.encoder.layer.10.attention.output.LayerNorm.weight', 'generator.encoder.layer.10.attention.output.LayerNorm.bias', 'generator.encoder.layer.10.intermediate.dense.weight', 'generator.encoder.layer.10.intermediate.dense.bias', 'generator.encoder.layer.10.output.dense.weight', 'generator.encoder.layer.10.output.dense.bias', 'generator.encoder.layer.10.output.LayerNorm.weight', 'generator.encoder.layer.10.output.LayerNorm.bias', 'generator.encoder.layer.11.attention.self.query.weight', 'generator.encoder.layer.11.attention.self.query.bias', 'generator.encoder.layer.11.attention.self.key.weight', 'generator.encoder.layer.11.attention.self.key.bias', 'generator.encoder.layer.11.attention.self.value.weight', 'generator.encoder.layer.11.attention.self.value.bias', 'generator.encoder.layer.11.attention.output.dense.weight', 'generator.encoder.layer.11.attention.output.dense.bias', 'generator.encoder.layer.11.attention.output.LayerNorm.weight', 'generator.encoder.layer.11.attention.output.LayerNorm.bias', 'generator.encoder.layer.11.intermediate.dense.weight', 'generator.encoder.layer.11.intermediate.dense.bias', 'generator.encoder.layer.11.output.dense.weight', 'generator.encoder.layer.11.output.dense.bias', 'generator.encoder.layer.11.output.LayerNorm.weight', 'generator.encoder.layer.11.output.LayerNorm.bias', 'generator.embeddings_project.weight', 'generator.embeddings_project.bias', 'generator_predictions.bias', 'generator_predictions.dense.weight', 'generator_predictions.dense.bias', 'generator_predictions.LayerNorm.weight', 'generator_predictions.LayerNorm.bias', 'generator_predictions.decoder.weight', 'generator_predictions.decoder.bias', 'discriminator_predictions.dense.weight', 'discriminator_predictions.dense.bias', 'discriminator_predictions.LayerNorm.weight', 'discriminator_predictions.LayerNorm.bias', 'discriminator_predictions.classifier.weight', 'discriminator_predictions.classifier.bias']\n",
            "- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at /content/electra/ and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "batch_size =  16\n",
            "Learning_rate =  5e-05\n",
            "epochs => 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:29: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6897eefdf2a449e2955dadffd2030e3e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=3.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 3 ========\n",
            "Training...\n",
            "  Batch    40  of  1,637.    Elapsed: 0:00:04.\n",
            "  Batch    80  of  1,637.    Elapsed: 0:00:07.\n",
            "  Batch   120  of  1,637.    Elapsed: 0:00:10.\n",
            "  Batch   160  of  1,637.    Elapsed: 0:00:14.\n",
            "  Batch   200  of  1,637.    Elapsed: 0:00:17.\n",
            "  Batch   240  of  1,637.    Elapsed: 0:00:20.\n",
            "  Batch   280  of  1,637.    Elapsed: 0:00:24.\n",
            "  Batch   320  of  1,637.    Elapsed: 0:00:27.\n",
            "  Batch   360  of  1,637.    Elapsed: 0:00:31.\n",
            "  Batch   400  of  1,637.    Elapsed: 0:00:34.\n",
            "  Batch   440  of  1,637.    Elapsed: 0:00:38.\n",
            "  Batch   480  of  1,637.    Elapsed: 0:00:41.\n",
            "  Batch   520  of  1,637.    Elapsed: 0:00:45.\n",
            "  Batch   560  of  1,637.    Elapsed: 0:00:48.\n",
            "  Batch   600  of  1,637.    Elapsed: 0:00:52.\n",
            "  Batch   640  of  1,637.    Elapsed: 0:00:55.\n",
            "  Batch   680  of  1,637.    Elapsed: 0:00:59.\n",
            "  Batch   720  of  1,637.    Elapsed: 0:01:02.\n",
            "  Batch   760  of  1,637.    Elapsed: 0:01:06.\n",
            "  Batch   800  of  1,637.    Elapsed: 0:01:09.\n",
            "  Batch   840  of  1,637.    Elapsed: 0:01:13.\n",
            "  Batch   880  of  1,637.    Elapsed: 0:01:16.\n",
            "  Batch   920  of  1,637.    Elapsed: 0:01:20.\n",
            "  Batch   960  of  1,637.    Elapsed: 0:01:23.\n",
            "  Batch 1,000  of  1,637.    Elapsed: 0:01:27.\n",
            "  Batch 1,040  of  1,637.    Elapsed: 0:01:31.\n",
            "  Batch 1,080  of  1,637.    Elapsed: 0:01:34.\n",
            "  Batch 1,120  of  1,637.    Elapsed: 0:01:38.\n",
            "  Batch 1,160  of  1,637.    Elapsed: 0:01:41.\n",
            "  Batch 1,200  of  1,637.    Elapsed: 0:01:45.\n",
            "  Batch 1,240  of  1,637.    Elapsed: 0:01:48.\n",
            "  Batch 1,280  of  1,637.    Elapsed: 0:01:52.\n",
            "  Batch 1,320  of  1,637.    Elapsed: 0:01:55.\n",
            "  Batch 1,360  of  1,637.    Elapsed: 0:01:59.\n",
            "  Batch 1,400  of  1,637.    Elapsed: 0:02:02.\n",
            "  Batch 1,440  of  1,637.    Elapsed: 0:02:06.\n",
            "  Batch 1,480  of  1,637.    Elapsed: 0:02:09.\n",
            "  Batch 1,520  of  1,637.    Elapsed: 0:02:12.\n",
            "  Batch 1,560  of  1,637.    Elapsed: 0:02:16.\n",
            "  Batch 1,600  of  1,637.    Elapsed: 0:02:19.\n",
            "\n",
            "  Average training loss: 0.50\n",
            "  Training epcoh took: 0:02:23\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.80\n",
            "  Validation Loss: 0.42\n",
            "  Validation took: 0:00:04\n",
            "\n",
            "======== Epoch 2 / 3 ========\n",
            "Training...\n",
            "  Batch    40  of  1,637.    Elapsed: 0:00:04.\n",
            "  Batch    80  of  1,637.    Elapsed: 0:00:07.\n",
            "  Batch   120  of  1,637.    Elapsed: 0:00:10.\n",
            "  Batch   160  of  1,637.    Elapsed: 0:00:14.\n",
            "  Batch   200  of  1,637.    Elapsed: 0:00:17.\n",
            "  Batch   240  of  1,637.    Elapsed: 0:00:21.\n",
            "  Batch   280  of  1,637.    Elapsed: 0:00:24.\n",
            "  Batch   320  of  1,637.    Elapsed: 0:00:28.\n",
            "  Batch   360  of  1,637.    Elapsed: 0:00:31.\n",
            "  Batch   400  of  1,637.    Elapsed: 0:00:35.\n",
            "  Batch   440  of  1,637.    Elapsed: 0:00:38.\n",
            "  Batch   480  of  1,637.    Elapsed: 0:00:42.\n",
            "  Batch   520  of  1,637.    Elapsed: 0:00:45.\n",
            "  Batch   560  of  1,637.    Elapsed: 0:00:49.\n",
            "  Batch   600  of  1,637.    Elapsed: 0:00:52.\n",
            "  Batch   640  of  1,637.    Elapsed: 0:00:56.\n",
            "  Batch   680  of  1,637.    Elapsed: 0:00:59.\n",
            "  Batch   720  of  1,637.    Elapsed: 0:01:03.\n",
            "  Batch   760  of  1,637.    Elapsed: 0:01:06.\n",
            "  Batch   800  of  1,637.    Elapsed: 0:01:10.\n",
            "  Batch   840  of  1,637.    Elapsed: 0:01:13.\n",
            "  Batch   880  of  1,637.    Elapsed: 0:01:16.\n",
            "  Batch   920  of  1,637.    Elapsed: 0:01:20.\n",
            "  Batch   960  of  1,637.    Elapsed: 0:01:23.\n",
            "  Batch 1,000  of  1,637.    Elapsed: 0:01:27.\n",
            "  Batch 1,040  of  1,637.    Elapsed: 0:01:30.\n",
            "  Batch 1,080  of  1,637.    Elapsed: 0:01:34.\n",
            "  Batch 1,120  of  1,637.    Elapsed: 0:01:37.\n",
            "  Batch 1,160  of  1,637.    Elapsed: 0:01:41.\n",
            "  Batch 1,200  of  1,637.    Elapsed: 0:01:44.\n",
            "  Batch 1,240  of  1,637.    Elapsed: 0:01:48.\n",
            "  Batch 1,280  of  1,637.    Elapsed: 0:01:51.\n",
            "  Batch 1,320  of  1,637.    Elapsed: 0:01:55.\n",
            "  Batch 1,360  of  1,637.    Elapsed: 0:01:58.\n",
            "  Batch 1,400  of  1,637.    Elapsed: 0:02:02.\n",
            "  Batch 1,440  of  1,637.    Elapsed: 0:02:05.\n",
            "  Batch 1,480  of  1,637.    Elapsed: 0:02:09.\n",
            "  Batch 1,520  of  1,637.    Elapsed: 0:02:12.\n",
            "  Batch 1,560  of  1,637.    Elapsed: 0:02:15.\n",
            "  Batch 1,600  of  1,637.    Elapsed: 0:02:19.\n",
            "\n",
            "  Average training loss: 0.38\n",
            "  Training epcoh took: 0:02:22\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.82\n",
            "  Validation Loss: 0.40\n",
            "  Validation took: 0:00:04\n",
            "\n",
            "======== Epoch 3 / 3 ========\n",
            "Training...\n",
            "  Batch    40  of  1,637.    Elapsed: 0:00:03.\n",
            "  Batch    80  of  1,637.    Elapsed: 0:00:07.\n",
            "  Batch   120  of  1,637.    Elapsed: 0:00:10.\n",
            "  Batch   160  of  1,637.    Elapsed: 0:00:14.\n",
            "  Batch   200  of  1,637.    Elapsed: 0:00:17.\n",
            "  Batch   240  of  1,637.    Elapsed: 0:00:21.\n",
            "  Batch   280  of  1,637.    Elapsed: 0:00:24.\n",
            "  Batch   320  of  1,637.    Elapsed: 0:00:28.\n",
            "  Batch   360  of  1,637.    Elapsed: 0:00:31.\n",
            "  Batch   400  of  1,637.    Elapsed: 0:00:35.\n",
            "  Batch   440  of  1,637.    Elapsed: 0:00:38.\n",
            "  Batch   480  of  1,637.    Elapsed: 0:00:42.\n",
            "  Batch   520  of  1,637.    Elapsed: 0:00:45.\n",
            "  Batch   560  of  1,637.    Elapsed: 0:00:49.\n",
            "  Batch   600  of  1,637.    Elapsed: 0:00:52.\n",
            "  Batch   640  of  1,637.    Elapsed: 0:00:56.\n",
            "  Batch   680  of  1,637.    Elapsed: 0:00:59.\n",
            "  Batch   720  of  1,637.    Elapsed: 0:01:03.\n",
            "  Batch   760  of  1,637.    Elapsed: 0:01:06.\n",
            "  Batch   800  of  1,637.    Elapsed: 0:01:09.\n",
            "  Batch   840  of  1,637.    Elapsed: 0:01:13.\n",
            "  Batch   880  of  1,637.    Elapsed: 0:01:16.\n",
            "  Batch   920  of  1,637.    Elapsed: 0:01:20.\n",
            "  Batch   960  of  1,637.    Elapsed: 0:01:23.\n",
            "  Batch 1,000  of  1,637.    Elapsed: 0:01:27.\n",
            "  Batch 1,040  of  1,637.    Elapsed: 0:01:30.\n",
            "  Batch 1,080  of  1,637.    Elapsed: 0:01:34.\n",
            "  Batch 1,120  of  1,637.    Elapsed: 0:01:37.\n",
            "  Batch 1,160  of  1,637.    Elapsed: 0:01:41.\n",
            "  Batch 1,200  of  1,637.    Elapsed: 0:01:44.\n",
            "  Batch 1,240  of  1,637.    Elapsed: 0:01:48.\n",
            "  Batch 1,280  of  1,637.    Elapsed: 0:01:51.\n",
            "  Batch 1,320  of  1,637.    Elapsed: 0:01:54.\n",
            "  Batch 1,360  of  1,637.    Elapsed: 0:01:58.\n",
            "  Batch 1,400  of  1,637.    Elapsed: 0:02:01.\n",
            "  Batch 1,440  of  1,637.    Elapsed: 0:02:05.\n",
            "  Batch 1,480  of  1,637.    Elapsed: 0:02:08.\n",
            "  Batch 1,520  of  1,637.    Elapsed: 0:02:12.\n",
            "  Batch 1,560  of  1,637.    Elapsed: 0:02:15.\n",
            "  Batch 1,600  of  1,637.    Elapsed: 0:02:19.\n",
            "\n",
            "  Average training loss: 0.33\n",
            "  Training epcoh took: 0:02:22\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.83\n",
            "  Validation Loss: 0.40\n",
            "  Validation took: 0:00:04\n",
            "\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:07:18 (h:mm:ss)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YHUI5fxuLS0d"
      },
      "source": [
        "model.save_pretrained(\"/content/electra\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0wphy7BJdpVn"
      },
      "source": [
        "# Predict"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PfUTFHg4BwTJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0a83b12c-7f40-48b5-eff8-77e9c5eb9754"
      },
      "source": [
        "# df_ = pd.concat([df[df.label == 1], df[df.label == 0].sample(15000)])\n",
        "for i in range(10):\n",
        "    post = df[df.label == 1].sample(1).iloc[0].post\n",
        "    inputs = tokenizer(post, return_tensors=\"pt\")\n",
        "    inputs.to('cuda')\n",
        "    res = model(**inputs)\n",
        "    print(1, res[0].tolist()[0], post)\n",
        "for i in range(10):\n",
        "    post = df[df.label == 0].sample(1).iloc[0].post\n",
        "    inputs = tokenizer(post, return_tensors=\"pt\")\n",
        "    inputs.to('cuda')\n",
        "    res = model(**inputs)\n",
        "    print(0, res[0].tolist()[0], post)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1 [-0.3441304862499237, 0.3961566388607025] სერგის გატეხვით და დასჯით ჩვენი შვილების დაშინებაც სურთ\n",
            "1 [-1.6734694242477417, 1.8392516374588013] ტროპარიწმიდისაგან ხატისა შენისა ჰოი დედუფალო ღვთისმშობელო სიმრთელენი და კურნებანი უხვად მოეცემიან ყოველთა სარწმუნოებითა და სიყვარულით მისდა მოლტოლვილთა ამისთვის ჩემსაცა უძლურებასა მოხედე და შეიწყალე სული ჩემი სახიერო ხოლო ხორცით განმკურნე მადლითა შენითა ყოვლადწმიდაო\n",
            "1 [0.9612675309181213, -1.0451594591140747] ნაცების მდგომარეობა უფრო დამძიმდა\n",
            "1 [-1.6281352043151855, 1.7903770208358765] საქართველოში ყოველდღე დღესაცწიწამურთან კვლავ ტყვიას ისვრიანმოღრუბრულია ისევე ზეცაცდა აქ ყოველდღე კლავენ ილიას\n",
            "1 [-1.8438581228256226, 2.0151760578155518] დადუმებული ქართველობა updated their status\n",
            "1 [-1.8136552572250366, 1.9816479682922363] საქართველო უპირველეს ყოვლისა updated their status\n",
            "1 [-0.014555711299180984, 0.04023725911974907] იტალიის შს მინისტრი რიჩარდ გირს თუ ასე ღელავს მიგრანტებზე შეუძლია საკუთარი კერძო თვითმფრინავით ჰოლივუდში წაიყვანოს და თავის ვილებზე დაასახლოს\n",
            "1 [-1.8315101861953735, 2.012272596359253] http erovnuli fronti net 1 p 1368\n",
            "1 [-0.9664889574050903, 1.0770200490951538] ქართული ნაგაზი პატარა დათვის ბელია\n",
            "1 [-0.18189865350723267, 0.22039861977100372] ოკლედ ატეხილი ქალების ჯგუფი ხო ჯოჯოხეთია და ატეხილი კაცების ანალოგიური ჯგუფიც ყოფილა შექმნილი\n",
            "0 [-0.6166810989379883, 0.7056763768196106] რას ასწავლიდნენ საუკუნეების წინ გელათის და იყალთოს აკადემიებში iv საუკუნეში ქართულ სკოლებში სასწავლებლად ბერძნები ჩამოდიოდნენ\n",
            "0 [2.029486656188965, -2.2528233528137207] ჩვენ გვიყვარს გიჟური ფერადი სამყარო\n",
            "0 [2.026047945022583, -2.2561588287353516] საზაფხულო აქცია ელექტრო პროდუქციის სექციაში გემოვნებიანი და თანამედროვე შეიძინეთ ჭერის სანათი 20 ლარიდან გვესტუმრეთ დომინოს ჰიპერმარკეტში მისამართზე სავაჭრო ცენტრი east point ისთ ფოინთი კახეთის გზატკეცილი თვალჭრელიძის 2\n",
            "0 [-0.5167803168296814, 0.5979259610176086] გმირი რომლის ცხოვრებაც თქვენ შეცვალეთ\n",
            "0 [1.7313172817230225, -1.913447618484497] კარტოფილის კანი ჭაღარას მოგაშორებთ ვიდეო\n",
            "0 [1.7924782037734985, -1.9795708656311035] თქვენი აზრით რომელი ქალი გეგმავს ჩხუბს\n",
            "0 [1.7666908502578735, -1.9544235467910767] ხანდახან ყველაზე უმნიშვნელო რაღაცეებს შენს გულში ყველაზე დიდი ადგილი უკავია\n",
            "0 [1.251344919204712, -1.3718962669372559] გიორგი მარგველაშვილი ქართველ ჯარისკაცებს პროფესიონალიზმთან და პარტნიორთა მხარდაჭერასთან ერთად ზურგს სიმართლის მხარეს დგომა უმაგრებს\n",
            "0 [1.113386869430542, -1.2187983989715576] არქის გენერალური დირექტორი შიო ხეცურიანი საქმიან დილას ესტუმრა და 2019 წელი შეაჯამა\n",
            "0 [1.9972336292266846, -2.220655679702759] ოჯახში ბედნიერება არა მხოლოდ მისი წევრების პოზიტიურ განწყობაზეა დამოკიდებული არამედ ნივთებზეც და პროდუქტებზეც კი რომლებიც თითოეული ჩვენგანის სახლში არსებობს\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}