SLM.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gAAQLewKdtMH",
        "outputId": "ae5c0bc2-8f1c-462b-f990-f0ee8205576e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.1.0+cu121)\n",
            "Requirement already satisfied: torchtext in /usr/local/lib/python3.10/dist-packages (0.16.0)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.1.99)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (1.5.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.66.1)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.16.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.1.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchtext) (2.31.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchtext) (1.23.5)\n",
            "Requirement already satisfied: torchdata==0.7.0 in /usr/local/lib/python3.10/dist-packages (from torchtext) (0.7.0)\n",
            "Requirement already satisfied: urllib3>=1.25 in /usr/local/lib/python3.10/dist-packages (from torchdata==0.7.0->torchtext) (2.0.7)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2023.3.post1)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (10.0.1)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.7)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.15)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torchtext) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchtext) (3.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchtext) (2023.11.17)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install torch torchtext transformers sentencepiece pandas tqdm datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "a4HA4JSHeECb"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset, DatasetDict , Dataset\n",
        "import pandas as pd\n",
        "import ast\n",
        "import datasets\n",
        "from tqdm import tqdm\n",
        "import time\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7od0BpLkfceJ",
        "outputId": "a9a8e158-084e-46e9-8ad3-4fbaaeb329f9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/repocard.py:105: UserWarning: Repo card metadata block was not found. Setting CardData to empty.\n",
            "  warnings.warn(\"Repo card metadata block was not found. Setting CardData to empty.\")\n"
          ]
        }
      ],
      "source": [
        "#load data\n",
        "data_sample = load_dataset(\"QuyenAnhDE/Diseases_Symptoms\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4OpQTWG3fvS4",
        "outputId": "b0038201-35d3-4ce4-9fb0-870e2eceb8c0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['Code', 'Name', 'Symptoms', 'Treatments'],\n",
              "        num_rows: 400\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "data_sample"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "pZQF0EM6hh0H"
      },
      "outputs": [],
      "source": [
        "updated_data = [{'Name': item['Name'], 'Symptoms': item['Symptoms']} for item in data_sample['train']]\n",
        "df = pd.DataFrame(updated_data)\n",
        "df.to_csv('data.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "z0OxVSMohrXD",
        "outputId": "62a5615a-f69d-4d15-94e6-21686a36ba83"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                          Name  \\\n",
              "0               Panic disorder   \n",
              "1             Vocal cord polyp   \n",
              "2              Turner syndrome   \n",
              "3               Cryptorchidism   \n",
              "4  Ethylene glycol poisoning-1   \n",
              "\n",
              "                                            Symptoms  \n",
              "0  Palpitations, Sweating, Trembling, Shortness o...  \n",
              "1           Hoarseness, Vocal Changes, Vocal Fatigue  \n",
              "2  Short stature, Gonadal dysgenesis, Webbed neck...  \n",
              "3  Absence or undescended testicle(s), empty scro...  \n",
              "4  Nausea, vomiting, abdominal pain, General mala...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ee6e9cb3-936c-4251-8336-3ee50854081f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Symptoms</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Panic disorder</td>\n",
              "      <td>Palpitations, Sweating, Trembling, Shortness o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Vocal cord polyp</td>\n",
              "      <td>Hoarseness, Vocal Changes, Vocal Fatigue</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Turner syndrome</td>\n",
              "      <td>Short stature, Gonadal dysgenesis, Webbed neck...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Cryptorchidism</td>\n",
              "      <td>Absence or undescended testicle(s), empty scro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Ethylene glycol poisoning-1</td>\n",
              "      <td>Nausea, vomiting, abdominal pain, General mala...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ee6e9cb3-936c-4251-8336-3ee50854081f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ee6e9cb3-936c-4251-8336-3ee50854081f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ee6e9cb3-936c-4251-8336-3ee50854081f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-84d96737-67f1-4ed2-8c66-2c533c3aae94\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-84d96737-67f1-4ed2-8c66-2c533c3aae94')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-84d96737-67f1-4ed2-8c66-2c533c3aae94 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "JNKrXKChhvc6"
      },
      "outputs": [],
      "source": [
        "#just extract the sympthoms\n",
        "\n",
        "df['Symptoms'] = df['Symptoms'].apply(lambda x: ', '.join(x.split(', ')))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "wPprcECHDWYh"
      },
      "outputs": [],
      "source": [
        "from transformers import GPT2Tokenizer,GPT2LMHeadModel\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import DataLoader,Dataset,random_split"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if torch.cuda.is_available():\n",
        "    device = torch.device('cuda')\n",
        "else:\n",
        "    # If Apple Silicon, set to 'mps' - otherwise 'cpu' (not advised)\n",
        "    try:\n",
        "        device = torch.device('mps')\n",
        "    except Exception:\n",
        "        device = torch.device('cpu')"
      ],
      "metadata": {
        "id": "zM98rlSyFjf9"
      },
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s-yTjehFGBP7",
        "outputId": "aba3ba40-8a52-44bb-f49a-7298aacc8a99"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = GPT2Tokenizer.from_pretrained('distilgpt2')\n",
        "model = GPT2LMHeadModel.from_pretrained('distilgpt2').to(device)"
      ],
      "metadata": {
        "id": "Q0bTA0rGGN4O"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ejNFoWP_GmtQ",
        "outputId": "8ec81acf-5dea-429c-d467-695dbff39e5c"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2LMHeadModel(\n",
              "  (transformer): GPT2Model(\n",
              "    (wte): Embedding(50257, 768)\n",
              "    (wpe): Embedding(1024, 768)\n",
              "    (drop): Dropout(p=0.1, inplace=False)\n",
              "    (h): ModuleList(\n",
              "      (0-5): 6 x GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "BATCH_SIZE = 8"
      ],
      "metadata": {
        "id": "-l2-iCQNHBoa"
      },
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 174
        },
        "id": "wghi7KMNHTqr",
        "outputId": "4cf92df3-a5d3-4eea-a0c0-3d74efd10881"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Name                              Symptoms\n",
              "count        400                                   400\n",
              "unique       392                                   395\n",
              "top     Sciatica  Swelling, pain, dry mouth, bad taste\n",
              "freq           3                                     3"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3d701328-74da-4d42-902f-83b7a2f9b885\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Symptoms</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>400</td>\n",
              "      <td>400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>392</td>\n",
              "      <td>395</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>Sciatica</td>\n",
              "      <td>Swelling, pain, dry mouth, bad taste</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3d701328-74da-4d42-902f-83b7a2f9b885')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3d701328-74da-4d42-902f-83b7a2f9b885 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3d701328-74da-4d42-902f-83b7a2f9b885');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0102afca-4ead-4fe5-afff-e4a9bb2decdd\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0102afca-4ead-4fe5-afff-e4a9bb2decdd')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0102afca-4ead-4fe5-afff-e4a9bb2decdd button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#dataset prep\n",
        "class LanguageDataset(Dataset):\n",
        "    \"\"\"\n",
        "    An extension of the Dataset object to:\n",
        "      - Make training loop cleaner\n",
        "      - Make ingestion easier from pandas df's\n",
        "    \"\"\"\n",
        "    def __init__(self, df, tokenizer):\n",
        "        self.labels = df.columns\n",
        "        self.data = df.to_dict(orient='records')\n",
        "        self.tokenizer = tokenizer\n",
        "        x = self.fittest_max_length(df)\n",
        "        self.max_length = x\n",
        "\n",
        "        print(\"Max length set to: \", self.max_length)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        x = self.data[idx][self.labels[0]]\n",
        "        y = self.data[idx][self.labels[1]]\n",
        "        text = f\"{x} | {y}\"\n",
        "        tokens = self.tokenizer.encode_plus(text, return_tensors='pt', max_length=128, padding='max_length', truncation=True)\n",
        "        return tokens\n",
        "\n",
        "    def fittest_max_length(self, df):  # Fix here\n",
        "        \"\"\"\n",
        "        Smallest power of two larger than the longest term in the data set.\n",
        "        Important to set up max length to speed training time.\n",
        "        \"\"\"\n",
        "        max_length = max(len(max(df[self.labels[0]], key=len)), len(max(df[self.labels[1]], key=len)))\n",
        "        x = 2\n",
        "        while x < max_length: x = x * 2\n",
        "        return x"
      ],
      "metadata": {
        "id": "hpUysfLmHXms"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Cast the Huggingface data set as a LanguageDataset we defined above\n",
        "data_sample = LanguageDataset(df, tokenizer)\n",
        "\n",
        "train_size = int(0.8 * len(data_sample))\n",
        "valid_size = len(data_sample) - train_size\n",
        "\n",
        "train_data, valid_data = random_split( data_sample,[train_size,valid_size])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "njr4jM6jI-u-",
        "outputId": "93b6b64d-ba70-4836-84b8-841f685a3001"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max length set to:  512\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)\n",
        "valid_loader = DataLoader(valid_data, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "id": "VKtzc2oiJlD0"
      },
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "num_epochs = 10"
      ],
      "metadata": {
        "id": "EXr1TmzPJuth"
      },
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = BATCH_SIZE\n",
        "model_name = 'distilgpt2'\n",
        "gpu = 0\n",
        "\n"
      ],
      "metadata": {
        "id": "ycTauK_8J9EP"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = nn.CrossEntropyLoss(ignore_index = tokenizer.pad_token_id)\n",
        "optimizer = optim.Adam(model.parameters(), lr=5e-4)\n",
        "tokenizer.pad_token = tokenizer.eos_token\n"
      ],
      "metadata": {
        "id": "HF9JsbhXKI5h"
      },
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "results = pd.DataFrame(columns = ['epoch', 'transformer', 'batch_size', 'gpu','training_loss', 'validation_loss', 'epoch_duration_sec'])"
      ],
      "metadata": {
        "id": "lu9VI-LPK2L4"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# The training loop\n",
        "for epoch in range(num_epochs):\n",
        "    start_time = time.time()  # Start the timer for the epoch\n",
        "\n",
        "    # Training\n",
        "    ## This line tells the model we're in 'learning mode'\n",
        "    model.train()\n",
        "    epoch_training_loss = 0\n",
        "    train_iterator = tqdm(train_loader, desc=f\"Training Epoch {epoch+1}/{num_epochs} Batch Size: {batch_size}, Transformer: {model_name}\")\n",
        "    for batch in train_iterator:\n",
        "        optimizer.zero_grad()\n",
        "        inputs = batch['input_ids'].squeeze(1).to(device)\n",
        "        targets = inputs.clone()\n",
        "        outputs = model(input_ids=inputs, labels=targets)\n",
        "        loss = outputs.loss\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        train_iterator.set_postfix({'Training Loss': loss.item()})\n",
        "        epoch_training_loss += loss.item()\n",
        "    avg_epoch_training_loss = epoch_training_loss / len(train_iterator)\n",
        "\n",
        "    # Validation\n",
        "    ## This line below tells the model to 'stop learning'\n",
        "    model.eval()\n",
        "    epoch_validation_loss = 0\n",
        "    total_loss = 0\n",
        "    valid_iterator = tqdm(valid_loader, desc=f\"Validation Epoch {epoch+1}/{num_epochs}\")\n",
        "    with torch.no_grad():\n",
        "        for batch in valid_iterator:\n",
        "            inputs = batch['input_ids'].squeeze(1).to(device)\n",
        "            targets = inputs.clone()\n",
        "            outputs = model(input_ids=inputs, labels=targets)\n",
        "            loss = outputs.loss\n",
        "            total_loss += loss\n",
        "            valid_iterator.set_postfix({'Validation Loss': loss.item()})\n",
        "            epoch_validation_loss += loss.item()\n",
        "\n",
        "    avg_epoch_validation_loss = epoch_validation_loss / len(valid_loader)\n",
        "\n",
        "    end_time = time.time()  # End the timer for the epoch\n",
        "    epoch_duration_sec = end_time - start_time  # Calculate the duration in seconds\n",
        "\n",
        "    new_row = {'transformer': model_name,\n",
        "               'batch_size': batch_size,\n",
        "               'gpu': gpu,\n",
        "               'epoch': epoch+1,\n",
        "               'training_loss': avg_epoch_training_loss,\n",
        "               'validation_loss': avg_epoch_validation_loss,\n",
        "               'epoch_duration_sec': epoch_duration_sec}  # Add epoch_duration to the dataframe\n",
        "\n",
        "    results.loc[len(results)] = new_row\n",
        "    print(f\"Epoch: {epoch+1}, Validation Loss: {total_loss/len(valid_loader)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gn10bUG8Lruh",
        "outputId": "fff31555-0bfa-49b6-fee7-77dc555b2d48"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 1/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.28it/s, Training Loss=0.71]\n",
            "Validation Epoch 1/10: 100%|██████████| 10/10 [00:00<00:00, 17.17it/s, Validation Loss=0.698]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1, Validation Loss: 0.7166230082511902\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 2/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.31it/s, Training Loss=0.402]\n",
            "Validation Epoch 2/10: 100%|██████████| 10/10 [00:00<00:00, 17.87it/s, Validation Loss=0.696]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 2, Validation Loss: 0.7102593779563904\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 3/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.17it/s, Training Loss=0.361]\n",
            "Validation Epoch 3/10: 100%|██████████| 10/10 [00:00<00:00, 17.69it/s, Validation Loss=0.706]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 3, Validation Loss: 0.7148756384849548\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 4/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.18it/s, Training Loss=0.341]\n",
            "Validation Epoch 4/10: 100%|██████████| 10/10 [00:00<00:00, 17.58it/s, Validation Loss=0.735]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 4, Validation Loss: 0.7477447390556335\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 5/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.05it/s, Training Loss=0.269]\n",
            "Validation Epoch 5/10: 100%|██████████| 10/10 [00:00<00:00, 17.27it/s, Validation Loss=0.813]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 5, Validation Loss: 0.8134035468101501\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 6/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.02it/s, Training Loss=0.147]\n",
            "Validation Epoch 6/10: 100%|██████████| 10/10 [00:00<00:00, 16.70it/s, Validation Loss=0.835]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 6, Validation Loss: 0.8836427927017212\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 7/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:08<00:00,  4.99it/s, Training Loss=0.153]\n",
            "Validation Epoch 7/10: 100%|██████████| 10/10 [00:00<00:00, 17.24it/s, Validation Loss=0.864]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 7, Validation Loss: 0.8844618797302246\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 8/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.04it/s, Training Loss=0.143]\n",
            "Validation Epoch 8/10: 100%|██████████| 10/10 [00:00<00:00, 17.17it/s, Validation Loss=0.957]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 8, Validation Loss: 0.9617654085159302\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 9/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.08it/s, Training Loss=0.113]\n",
            "Validation Epoch 9/10: 100%|██████████| 10/10 [00:00<00:00, 17.36it/s, Validation Loss=0.957]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 9, Validation Loss: 0.9632261395454407\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Epoch 10/10 Batch Size: 8, Transformer: distilgpt2: 100%|██████████| 40/40 [00:07<00:00,  5.03it/s, Training Loss=0.0949]\n",
            "Validation Epoch 10/10: 100%|██████████| 10/10 [00:00<00:00, 17.24it/s, Validation Loss=1.05]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 10, Validation Loss: 1.0658142566680908\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input_str = \"Ethylene glycol poisoning-1\"\n",
        "input_ids = tokenizer.encode(input_str, return_tensors='pt').to(device)\n",
        "\n",
        "output = model.generate(\n",
        "                        input_ids,\n",
        "                        max_length=200,\n",
        "                        num_return_sequences=1,\n",
        "                        do_sample=True,\n",
        "                        top_k=8,\n",
        "                        top_p=0.95,\n",
        "                        temperature=0.5,\n",
        "                        repetition_penalty=1.2\n",
        "                        )\n",
        "\n",
        "decoded_output = tokenizer.decode(output[0], skip_special_tokens=True)\n",
        "print(decoded_output)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LHKK-uZ0OTlK",
        "outputId": "fb8e03e9-c1c0-4030-8f5b-d058b3ce4edf"
      },
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Ethylene glycol poisoning-1 | Nausea, vomiting, abdominal pain, General malaise, weakness and swelling of the legs\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model, 'SmallDisMedLM.pt')"
      ],
      "metadata": {
        "id": "4cq3wOz6PDx-"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model,'/content/drive/MyDrive/SmallLMs/SmallDisMedLM2.pt')"
      ],
      "metadata": {
        "id": "0oFrd9g7QdLz"
      },
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Z0mw10vxQ4V6"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}