fine_tuneing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "93d1020c-b5b7-479c-8d1a-320650ae9760",
   "metadata": {},
   "source": [
    "## Choose model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "36ec74cc-fe4a-436b-bee4-fdf361c8a3ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'distilbert-base-cased'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4335c93-a4b1-448b-b45f-15309a80f4d8",
   "metadata": {},
   "source": [
    "## Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f15396a9-464a-4249-a588-dc4ef40c1928",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset, DatasetDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8596a9c9-9fcc-4cd6-b5b2-6a7852941b51",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset(\"jonathanli/law-stack-exchange\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "490d2c78-297b-4e88-ad95-c1deef7c5f4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['Id', 'PostTypeId', 'CreationDate', 'Score', 'ViewCount', 'LastActivityDate', 'AnswerCount', 'CommentCount', 'ContentLicense', 'body', 'text_label', 'title'],\n",
       "        num_rows: 638\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['Id', 'PostTypeId', 'CreationDate', 'Score', 'ViewCount', 'LastActivityDate', 'AnswerCount', 'CommentCount', 'ContentLicense', 'body', 'text_label', 'title'],\n",
       "        num_rows: 319\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['Id', 'PostTypeId', 'CreationDate', 'Score', 'ViewCount', 'LastActivityDate', 'AnswerCount', 'CommentCount', 'ContentLicense', 'body', 'text_label', 'title'],\n",
       "        num_rows: 1596\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8cc13977-ff34-4ebb-a329-85cfed28b8c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Id': 71340,\n",
       " 'PostTypeId': 1,\n",
       " 'CreationDate': '2021-09-01T02:39:58.830',\n",
       " 'Score': 0,\n",
       " 'ViewCount': 128.0,\n",
       " 'LastActivityDate': '2021-09-01T06:43:55.503',\n",
       " 'AnswerCount': 1.0,\n",
       " 'CommentCount': 9,\n",
       " 'ContentLicense': 'CC BY-SA 4.0',\n",
       " 'body': \"I'm trying to comfortably use my laptop at night and now could not find an answer by web-search to the following: Why TVs / laptops etc. do not warn about too bright screen whereas many smartphones warn about too loud sound amplification? I'm asking on law site cause I suspect there were a number of successful lawsuits for impaired hearing from headphones but none for impaired vision from TVs. Am I right here? Digging down - has it happened because there are studies proving impairment for one and none for the other?\\n\\n\\nAdded: I actually doubt my second guess cause there are a lot of enhancements in screen brightness - auto-brightness, night mode.\\n\\n\\n\",\n",
       " 'text_label': 'civil-law',\n",
       " 'title': 'Why TVs / laptops etc. do not warn about too brirght screen whereas many smartphones warn about too loud sound amplification?'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8043dfc9-6ca8-4021-848a-2e638f0213ef",
   "metadata": {},
   "source": [
    "## Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "63496883-d74f-4e92-8b07-6cd1587d720e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.remove_columns([\"Id\", \"PostTypeId\", \"CreationDate\", \"Score\", \"ViewCount\", \"LastActivityDate\",\n",
    "                                 \"AnswerCount\", \"CommentCount\", \"ContentLicense\", \"title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5dcbc6cc-d43d-442a-8b98-1798a0d2e92b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.rename_column(\"text_label\", \"label\")\n",
    "dataset = dataset.rename_column(\"body\", \"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "be194e77-6bec-4d4b-890e-7038fde26e78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 638\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 319\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 1596\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2550e54b-6207-47b3-94a5-4b5c78ef5f92",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a5f88ee9-771e-4ec8-b4e8-1746831ffebf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='0'>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr4AAAGdCAYAAAD0V3BwAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbRJJREFUeJzt3QV4U+f7PvCnRQrF3WW4FJchw4cW3YaNHz5kuMs2hg39AkPGsDFgOAz34e5uw4szHIZL87/u57+TKwlNm0JLm5z7c11nbZKTIylr7zx53vd4WSwWixAREREReTjviD4AIiIiIqKPgcGXiIiIiEyBwZeIiIiITIHBl4iIiIhMgcGXiIiIiEyBwZeIiIiITIHBl4iIiIhMgcGXiIiIiEwhakQfAFFkERgYKDdu3JA4ceKIl5dXRB8OERERuQDXYvv3338lZcqU4u0dfE2XwZfoPwi9adKkiejDICIiovdw9epVSZ06dbDrMPgS/QeVXuN/nLhx40b04RAREZELHj9+rIUr4+94cBh8if5jtDcg9DL4EhERuRdX2hQ5uI2IiIiITIHBl4iIiIhMga0ORA78+q4Tbx/fCD2GgKH+Ebp/IiIiT8SKr4ncu3dPkiZNKgEBAbJlyxbthXn48KE+Nn36dIkfP36otpc+fXoZPXp0sOtgH0uXLtXvsV/cPnLkiN52PIawcurUKR3V+fTp0zDdLhEREbk3Bl8TGTRokNSoUUMDq6O6devK2bNnw3yfN2/elMqVKwf5WLFixfTxePHivXf4DkqOHDmkSJEiMmrUqA/eFhEREXkOBl+TePbsmUydOlWaN28e5OMxY8bUanBYS548ufj4+AT5WPTo0fXx8LhYRNOmTWXChAny5s2bMN82ERERuScGX5NYvXq1BlBUQoPiWG29cOGCVoeTJUsmsWPHlkKFCsmGDRveeR6ulFK/fn2JFSuWpEqVSsaPH++01cGRbasDvkdYffTokd6HpV+/fjJgwADx8/N757l58+aVPn36OD3f8uXLy/3792Xr1q3Bvi5ERERkHgy+JrF9+3YpUKCAy+s/efJEqlSpIhs3bpTDhw9LpUqVpFq1anLlyhW79f73v/9Jnjx5dJ1evXpJx44dZf369aE+PrQ9oF8Y8+ei/QFLt27dpFmzZnL69GnZv3+/dV3s69ixYxqUnUE1GeEY5+3My5cvddJr24WIiIg8F4OvSVy+fFmvYe0qhNlWrVpptTVz5swycOBAyZgxoyxfvtxuveLFi2vgzZIli7Rv316++uor+fnnn0N9fAiq6PVFpRftD1hQacYgtYoVK8q0adOs6+L7UqVKSYYMGYLdJs4X5+3MkCFDdJ/GwssVExEReTYGX5N4/vy5xIgRI1QVX1Rcs2fPri0QCKGovDpWfIsWLfrObawXllq0aCFz586VFy9eyKtXr2TOnDlaCQ4J+pbR2+xM7969tbXCWHCpYiIiIvJcnMfXJBInTiwPHjxweX2EXrQsjBgxQjJlyqQhEtVcBM+PDS0W6E9esmSJVoZfv36txxIS9PiiSu0Mtuls4B0RERF5HgZfk8iXL5/MmjXL5fV37twpTZo0kVq1alkrwJiH19GePXveuY0q8ftAqH379u0790eNGlUaN26sLQ5Yp169ehrEQ3LixAmXAjIRERGZA4OvSaBPFh/to+qbIEGCENdHX+/ixYu12oq+W8ygEBgYGGRAHj58uNSsWVMrxAsXLpRVq1a91zFifmEEbAyoQ4+xr6+vLvDNN99YAzX2GRKE9OvXr8vnn3/+XsdCREREnoc9viaRK1cuyZ8/vyxYsMCl9XHxBwRkzLaA8IvgjOc76tq1qxw4cEAryj/99JM+D+u+D+yrdevWejGNJEmSaKC2DeJ4PFu2bPLpp58GOS2abUUaPcEVKlSQdOnSvdexEBERkefxslgslog+CPo4UInt3r27tgB4e7vXex78M0X4bdOmjXTp0sXuMbRADB48WC9VHC1aNO1DxroYBIdZJ1yF6cwwuwMGumFaNSIiIor8QvP3m60OJuLv7y/nzp3TFgB3mrrrzp07Mm/ePLl161aQc/fi4hwIvgi9gJknvvvuu1CFXiIiIvJ8rPhSpIc2BsxKMWbMGPn666/DbT+s+BIREbkfVnzJo/C9GREREYUF92r0JCIiIiJ6Twy+RERERGQKDL5EREREZAoMvkRERERkCgy+RERERGQKDL5EREREZAoMvkRERERkCgy+RERERGQKvICFB7p3755kz55d9u3bJ+nTpw923SZNmsjDhw9l6dKlLm27X79+uu6RI0dcfj6OoVOnTrp8DL169ZKnT5/KuHHj3uv5fn3XibePr0RmAUP9I/oQiIiI3A6DrwcaNGiQ1KhRI8TQC7gMcGiujNatWzdp3769RGY4xgwZMkjnzp31KxERERGw1cHDPHv2TKZOnSrNmzd3aX1c2zp+/Pgubz927NiSKFEiicwSJ04sFStWlAkTJkT0oRAREVEkwuDrYVavXi0+Pj5SpEgR630nT56UqlWrSty4cSVOnDhSokQJuXDhgrVVoWbNmvr95MmTJWXKlBIYGGi3TVSPmzVrZm11yJs37wcd46hRoyRXrlwSK1YsSZMmjbRp00aePHmij6H6nCRJEvnzzz+t62N/KVKksN7esWOHniNCvjPVqlWTefPmfdBxEhERkWdh8PUw27dvlwIFClhvX79+XUqWLKlBcdOmTXLw4EENsW/evHnnubVr19b+4M2bN1vvu3//vqxdu1YaNGgQZsfo7e0tY8eO1UA+Y8YMPa4ePXroY15eXnq8W7Zs0dsPHjyQ06dPy/Pnz+Xvv//W+7Zu3SqFChUSX1/nfbiFCxeWa9euSUBAgNN1Xr58KY8fP7ZbiIiIyHMx+HqYy5cva9XWMH78eG1nQPWzYMGCkiVLFmnatKlkzZr1necmSJBAKleuLHPmzLHeh8orWgfKlCkTZseIQW7YHnqQy5YtKz/99JMsWLDA+njp0qWtwXfbtm2SL18+u/vwtVSpUsHuw3gN8Ho4M2TIEH1tjAXVZyIiIvJcDL4eBpXRGDFiWG9j9gW0NkSLFs2l56Oyu2jRIq2GwuzZs6VevXpapQ0J1kUPsLGg+hyUDRs2SLly5SRVqlTaetGwYUOtNButCwi1p06dkjt37mh1F6HXCL6vX7+WXbt26e3gxIwZU78G1w7Ru3dvefTokXW5evVqiOdIRERE7ovB18OgOov2AMcA6Cr0xqLPdtWqVRoEEV5dbXOoXr26Bm1jQYXZEVoP0G+cO3duDdhovUBVGl69eqVf0f+bMGFCDb22wRff79+/X8NvsWLFgj0WtGgA+oWdQfsH+p5tFyIiIvJcnM7Mw6AtYNasWdbbCJjoo0VYdKXqi2rxF198odXb8+fPa0tE/vz5Xdo3qrdYgoOgi8FzI0eOtFaRbdscjD5fVKmXLVumfcCfffaZ9vOiCj1p0iQN1BgYF5wTJ07o+ebMmdOlYyciIiLPx4qvh8E0XgiLRtW3Xbt2OmgL7QoHDhyQc+fOycyZM+XMmTNOt4EKLyq+v//+e5gOaoNMmTJpCMfFJS5evKjHMnHixHfWQ4V37ty5OqMD2iYQkjHoDYE8pP5eQKUa4Tm0FW8iIiLyXAy+HgZtAqjQGlVUzLmLWRMwXRgCI2Z8mDJlSrDVXww4Q6sBwvHXX38dpseXJ08enc5s2LBh4ufnp0EWg8wc4Vjfvn1r18uL7x3vM6Zkc7wPg/latGgRpsdORERE7s3LEprLdpFbQLW2e/fu+nG/K4PS3B1CMmaJwBzDsGbNGunatascO3ZMokZ1vZsHlXHM7oCBbuz3JSIicg+h+fvNHl8P5O/vry0NmMPX06fowj9yXIwDYd/w9OlTmTZtWqhCLxEREXk+VnyJ/sOKLxERkWf//fb8z8GJiIiIiBh8iYiIiMgsGHyJiIiIyBQYfImIiIjIFBh8iYiIiMgUGHyJiIiIyBQYfImIiIjIFBh8iYiIiMgUGHyJiIiIyBR4TVcPce/ePcmePbvs27dP0qdP/8Hb69evn0yYMEFu374tS5YskZo1a4q7ePXqlWTJkkX+/PNPKViwYKif79d3nXj7+Io7ChjqH9GHQEREFGmx4ushBg0aJDVq1AiT0Hv69Gnp37+/TJo0SW7evCmVK1fW7Y4ePVrcQfTo0aVbt27Ss2fPiD4UIiIiikQYfD3As2fPZOrUqdK8efMw2d6FCxf0K4J08uTJxcfHRyLC27dvJTAw8L2e26BBA9mxY4ecPHkyzI+LiIiI3BODrwdYvXq1htMiRYro7QcPHmjwS5IkicSMGVMyZ84s06ZNs65//PhxKVu2rD6WKFEiadmypTx58sTa4lCtWjX93tvbW7y8vKR06dJy+fJl6dy5s97GYrFYdPtoJzDkzZtXUqRIYb2N4InjQjCHUaNGSa5cuSRWrFiSJk0aadOmjXW/MH36dIkfP74sX75ccuTIoc+9cuWKvHz5Uiu4qVKl0ud++umnsmXLlmBfkwQJEkjx4sVl3rx5YfY6ExERkXtj8PUA27dvlwIFClhv9+nTR06dOiVr1qzRtgX06iZOnFgfe/r0qVSsWFGD4f79+2XhwoWyYcMGadeunT6OgGmEZLQ5YFm8eLGkTp1aBgwYYL0P4bdkyZLWAIqwjX09f/5c/v77b71v69atUqhQIfH19bUG6bFjx2oVdsaMGbJp0ybp0aOH3bkgJA8bNkx+++03XS9p0qR6bLt379YQe+zYMaldu7ZUqlRJzp07F+zrUrhwYX1tnEGgfvz4sd1CREREnouD2zwAqrEpU6a03kaVNF++fNaBXbZ9v3PmzJEXL17IH3/8odVT+OWXX7TKi8CZLFkyrboC2hwMUaJEkThx4tjdh0ow+oBh27Ztuk88jjCcLVs2/VqqVCnr+p06dbJ+j2P66aefpHXr1vLrr79a73/9+rXezpMnj/VcEMTx1ThHhPO1a9fq/YMHD3b6umB9vDbODBkyRHuZiYiIyBxY8fUAqLLGiBHDevvbb7/V6ihaD1BR3bVrl/UxVGURKo3QC2gJQC/tmTNnQrVfhFpUlu/cuaPVXQRhLAi8CLDYL24bUFkuV66ctiwgRDds2FBnozBaIYyBablz57Zry0CvL2ZpiB07tnXB/oxeZGfQymG7bUe9e/eWR48eWZerV6+G6vyJiIjIvbDi6wHQxoBWAwNmYUClE72/69ev17DZtm1bGTFiRJjuF/26CRMm1BCKBTNLoOKLyjHaKBB+ixUrpusGBARI1apVNZRjPTwPPcAYkIfpx4x2CIRVtFEY0AOMavPBgwf1qy0E4ODcv39f+5CdQQ9xRA3cIyIioo+PFV8PgBYDVF5tIfA1btxYZs2apdOQTZ48We/HXL9Hjx7VXl/Dzp07tf82a9asTveBSiwqr7YQUEuUKCHLli3TftzPPvtMq7XonUULBFotjMoygiuqyiNHjtRBeKjg3rhxw6Vzw34xn3CmTJnsFtu2i6CcOHFCn09EREQEDL4eAIPVEDyNqu+PP/6oYfT8+fN6/8qVKzXwAmZ7QFsEQjGC4ebNm6V9+/badoD+XmfQk4s+3uvXr8vdu3et96OVYe7cudpWgQosAjQGvc2ePduuvxdBFRXgcePGycWLF2XmzJkyceLEEM8NARnH3KhRIx1kd+nSJb1IB/pzV61aFexzMbCtQoUKLr2GRERE5PkYfD0AWg7y588vCxYssFZn0b+K6itCKFoEjGm90FKwbt06bQPAjAtfffWVtkJggFtwMKMD2hUyZsxo1z6AcIuKrG0vL753vA99xZjODG0Qfn5+GowRXl2BQWwIvl27dtWqNK4ih1aKtGnT2lWfMR2aAbNAoG8X50dEREQEXhZMyEpuD9XP7t27axUXVVczQRUYlWG0e2DOYqhbt66G7e+++87l7WA6s3jx4mlgjhs3bjgeMREREYWV0Pz95uA2D+Hv76/z2qIVAReHMBMM4sNFOIzQi8FyqILjghtEREREBlZ8if7Dii8REZFn//0212fiRERERGRaDL5EREREZAoMvkRERERkCgy+RERERGQKDL5EREREZAoMvkRERERkCgy+RERERGQKDL5EREREZAoMvkRERERkCrxksYe4d++eZM+eXfbt2ycBAQFSpkwZefDggcSPHz9c9tevXz9ZunSpHDlyRCKjevXqSaFChaRr166hfq5f33Xi7eMr7ihgqH9EHwIREVGkxYqvhxg0aJDUqFFD0qdP/1H2161bN9m4caNEVj/88IO+Jrh8IREREREw+HqAZ8+eydSpU6V58+YfbZ+xY8eWRIkSSWTl5+cnGTNmlFmzZkX0oRAREVEkweDrAVavXi0+Pj5SpEgRu/t37twpuXPnlhgxYuhjJ06csGtVyJs3r936o0ePtqsYb9myRQoXLiyxYsXSlonixYvL5cuXg3x+kyZNpGbNmjJixAhJkSKFhuK2bdvK69evreu8fPlSK8WpUqXSbX766ae6DwO2Xa1aNUmQIIE+njNnTj03QNtGgwYNJEmSJBIzZkzJnDmzTJs2LdjXBduaN2/ee7yiRERE5InY4+sBtm/fLgUKFHjn/u7du8uYMWMkefLk8t1332kQPHv2rESLFi3Ebb5580aDbIsWLWTu3Lny6tUr7R/28vJy+pzNmzdr6MXX8+fPS926dTUcYxvQrl07OXXqlIbRlClTypIlS6RSpUpy/PhxDbIIytjPtm3bNPhiXVSWoU+fPnp7zZo1kjhxYt3+8+fPgz0HhHa0OyBw440BERERmRuDrwdApRRB0lHfvn2lfPny+v2MGTMkderUGjbr1KkT4jYfP36s/bFVq1bVlgHA4LngoFL7yy+/SJQoUSRbtmzi7++vfcAIvleuXNEKLb4ax4rq79q1a/X+wYMH62Nffvml5MqVSx/PkCGDddt4LF++fFKwYEG97UovM/aDIH3r1i1Jly7dO48jEGOxPWciIiLyXGx18ACofKKdwVHRokWt3ydMmFCyZs0qp0+fdmmbWB/tCxUrVtRKMSrHN2/eDPY5aE1A6DWg+nv79m39HlXdt2/fSpYsWbSKayxbt26VCxcu6DodOnSQn376SVsqENqPHTtm3da3336rlWJUkHv06CG7du0K8RzQEmH0QAdlyJAhEi9ePOuSJk0al14bIiIick8Mvh4AH/2jBzY0vL29xWKx2N1n248LqMTu3r1bihUrJvPnz9fQumfPHqfbdGyhQFtEYGCgfv/kyRMNxQcPHtQp0IwFQRyhGr755hu5ePGiNGzYUIMyqrvjxo3TxypXrqyV7c6dO8uNGzekXLlyWjEOzv379/Ur+oKD0rt3b61qG8vVq1eD3R4RERG5NwZfD4AWAPS/OrINqQjG6O812hUQBtECYBt+g5qTF9tGQESFFTMlzJkz572PERVfVIAzZcpkt6AH2YCqa+vWrWXx4sU6B++UKVOsj+GYGzdurDM1YCDe5MmTg90nBvOhvQNvDIKCvt+4cePaLUREROS5GHw9ANoRTp48+U7Vd8CAAdpjiwCItgUEQAxYg9KlS8udO3dk+PDh2mowfvx4HThmuHTpkgZeVHxRaf3rr7/k3LlzIfb5OoNqMWZlaNSokYZabB+D5dBusGrVKl2nU6dOsm7dOn3s0KFDOkjO2N+PP/4oy5Yt00FtONeVK1eGeCwY9FehQoX3Ol4iIiLyPAy+HgCDwfLnzy8LFiywu3/o0KHSsWNHnfEB1d0VK1ZI9OjR9TGExl9//VUDb548eTSE2rYO+Pr6yt9//62DzRBaW7ZsqbMutGrV6r2PE60TCL6o5KLfGCF8//79kjZtWn0cFWHsA8eG2R6wXxwj4LgRxDE9W8mSJbVtwnaqMgR5hHvDixcv9MpyxowSRERERF4Wx0ZPckuommL6MlR30b9rNpi1oX///tbwO2HCBJ3BApVqV2FWBwxyQ78v2x6IiIjcQ2j+fnM6Mw+BqcPQinD9+nXTzU6A1gf8g0c12XagnTEwjoiIiAhY8SX6Dyu+REREnv3323yfiRMRERGRKTH4EhEREZEpMPgSERERkSkw+BIRERGRKTD4EhEREZEpMPgSERERkSkw+BIRERGRKTD4EhEREZEpMPgSERERkSnwksUe5t69e5I9e3bZt2+fpE+fXszo1atXkiVLFvnzzz+lYMGCoX6+X9914u3jK54uYKh/RB8CERHRR8WKr4cZNGiQ1KhR46OF3unTp0v8+PHlY8F5jR49Oth1okePLt26dZOePXt+tOMiIiKiyI/B14M8e/ZMpk6dKs2bN5fIWIX9mBo0aCA7duyQkydPftT9EhERUeTF4OtBVq9eLT4+PlKkSBHrfQh+VatWlbhx40qcOHGkRIkScuHCBX0sMDBQBgwYIKlTp9bn5c2bV9auXWt9bkBAgHh5ecnixYulTJky4uvrK3ny5JHdu3fr41u2bJGmTZvKo0ePdD0s/fr1s1ZmBw4cKI0aNdJ9t2zZUu9HFRZtCNhWhgwZpE+fPvL69Wu781ixYoUUKlRIYsSIIYkTJ5ZatWrp/aVLl5bLly9L586drftzJkGCBFK8eHGZN29emL7GRERE5L4YfD3I9u3bpUCBAtbb169fl5IlS2qo3bRpkxw8eFCaNWsmb9680cfHjBkjI0eOlBEjRsixY8ekYsWKUr16dTl37pzddr///nttHThy5IiG1vr16+s2ihUrpm0HCLY3b97UBesZsF0E5cOHD2vABYRvtEecOnVK9z9lyhT5+eefrc9ZtWqVBt0qVaro8zZu3CiFCxfWxxDAEdIR1o39BQfPw2tCREREBBzc5kFQDU2ZMqX19vjx4yVevHha9YwWLZreh+BqG0xRga1Xr57eHjZsmGzevFnDLJ5rQJj19///A6H69+8vOXPmlPPnz0u2bNl0+6i8Jk+e/J3jKVu2rHTt2tXuvh9++MH6ParC2DaOr0ePHtYeZRwP9mNAeIaECRNKlChRNDwHtT9HeC3wmjjz8uVLXQyPHz8OcZtERETkvljx9SDPnz/X9gADKrRobTBCry2EvBs3bmg7gC3cPn36tN19uXPntn6fIkUK/Xr79u0QjyeoGRXmz5+v+0BwjR07tgbhK1eu2B1zuXLlJCzEjBlT+56dGTJkiAZ3Y0mTJk2Y7JeIiIgiJwZfD4J+2AcPHtgFv7BgG5yNvlr0B4ckVqxYdrfRG4xBZ2hjWLlypbYyoI3CduBbWB0z3L9/X5IkSeL08d69e2t/srFcvXo1zPZNREREkQ+DrwfJly+f9s7aVmrR4+o4eAzQl4tWgJ07d9rdj9s5cuRweZ+YOuzt27curbtr1y5Jly6dhl1UgzNnzvxOKwKOGX29YbG/EydO6GviDHqf8TrYLkREROS5GHw9CAanYRYHo+rbrl07bWlAz+yBAwd00NrMmTPlzJkz+nj37t21rxftB7ivV69e2mrQsWNHl/eJPt0nT55oWL17926wrQUIumhrQE8vZpYYO3asLFmyxG6dvn37yty5c/UrWi6OHz+ux2i7v23btunAPewvOAj9FSpUcPlciIiIyLMx+HqQXLlySf78+WXBggV6O1GiRDqbA4JpqVKldMYHzKJgtC506NBBunTpogPQ8FxMZbZ8+XINqK7CzA6tW7eWunXralvB8OHDna6LGSMwFRkCOaZOQwXYmO3BgCnLFi5cqMeBdTBADlehM2BGB0yzljFjRrs2BrRgYLYI27YKtC989dVXLp8LEREReTYvi8ViieiDoLCD6cBQycXH/N7e5nhfc+nSJZ2tAm0eRmhHEMdsEN99953L20F1HIPcEJjZ9kBEROQeQvP3m9OZeRhMO4aWBrQCmGWWAly4AxfIMEIvBsuhgo3qMhEREZGBFV+i/7DiS0RE5Nl/v83xWTgRERERmR6DLxERERGZAoMvEREREZkCgy8RERERmQKDLxERERGZAoMvEREREZkCgy8RERERmQKDLxERERGZAoMvEREREZkCL1lM5MCv7zrx9vEVMwkY6h/Rh0BERBTuWPF1c/fu3ZOkSZNKQECAbNmyRby8vOThw4f62PTp0yV+/PgSWfTr10/y5s0b7vvp1auXtG/fPtz3Q0RERO6FwdfNDRo0SGrUqCHp06d/57G6devK2bNnJbLo1q2bbNy48aPsZ8aMGXLx4sVw3xcRERG5DwZfN/bs2TOZOnWqNG/ePMjHY8aMqdXgyCJ27NiSKFGicN9P4sSJpWLFijJhwoRw3xcRERG5DwZfN7Z69Wrx8fGRIkWKBPl4UK0OK1askEKFCkmMGDE0INaqVcv62MuXL7VamipVKokVK5Z8+umn2j7huL1169ZJ9uzZNchWqlRJbt68aV0H6xcuXFifj3WLFy8uly9fDrLVoUmTJlKzZk0ZMWKEpEiRQkNx27Zt5fXr19Z1sG1/f38N8Z988onMmTNHq9ujR48O9rWpVq2azJs3L1SvJxEREXk2Bl83tn37dilQoIDL669atUqDbpUqVeTw4cPadoCQamjXrp3s3r1bA+OxY8ekdu3aGmzPnTtnV2VGUJ05c6Zs27ZNrly5omEZ3rx5o0G2VKlS+nxsq2XLltp37MzmzZvlwoUL+hXtCQjXWAyNGjWSGzduaKBetGiRTJ48WW7fvh3iueK8rl27pr3PziDoP3782G4hIiIiz8VZHdwYKqkpU6YMVT9wvXr1pH///tb78uTJo18RYKdNm6ZfjW0i0K5du1bvHzx4sN6HauzEiRMlY8aM1rA8YMAA/R7B8dGjR1K1alXr46gMBydBggTyyy+/SJQoUSRbtmxa3UUgb9Gihfz999+yYcMG2b9/vxQsWFDX/+233yRz5swhnqtxDniNgup/hiFDhti9FkREROTZWPF1Y8+fP9eWBVcdOXJEypUrF+Rjx48fl7dv30qWLFm0hcFYtm7dqhVZg6+vrzXUAloUjApswoQJtX0B/bVoNRgzZoxdG0RQcubMqaE3qO2dOXNGokaNKvnz57c+nilTJg3LIUFrhFGhdqZ3794a1I3l6tWrIW6XiIiI3Bcrvm4MPboPHjxweX0jDAblyZMnGkAPHjxoF0QBAdgQLVo0u8fQxmCxWKy3UR3u0KGDVornz58vP/zwg6xfv95pH3JQ2wsMDJQPdf/+ff2aJEkSp+ugPxoLERERmQMrvm4sX758curUKZfXz507t9PpxLAtVHxRbUVV1XZJnjx5qI8L1dRdu3aJn5+fDkh7H1mzZtW+YfQjG86fP+9S2D9x4oSGalSUiYiIiIDB142hpeDkyZMuV3379u0rc+fO1a+nT5/W9oZhw4bpY2hxaNCggQ4mW7x4sVy6dEn27dunfbAYFOcKPAeBF4Pa0Fv7119/6cC4kPp8nUHP7+eff64D5HAsCMD4HpXr4AbMGQP/SpQoEWyVm4iIiMyFwdeN5cqVS/tfFyxY4NL6pUuXloULF8ry5ct1WrGyZctqoLRtU0Dw7dq1q1ZbMUMDBpalTZvWpe2j/xcD0r788ksN0gipmJ6sVatW732Of/zxhyRLlkxKliypM1Jg0FucOHHsepvRV4xzs4WZKbAuERERkcHLYtugSW4H1dju3bvrR/ve3p7/PgZTlKVJk0ZnezAG6mH6tDJlyug8wbBmzRoN75hSDYPjXIVZKeLFi6cD3eLGjRtu50BERERhJzR/vzm4zc1h+i+0E1y/fl0DoafZtGmTDrxDdRszRPTo0UOnJ0MFGPCPHLNO2LZjPH36VKvXoQm9RERE5PlY8aVIDVeJQ/X24sWL2uJQrFgxvWpbunTpwnxfrPgSERG5n9D8/WbwJfoPgy8REZFn//32/KZQIiIiIiIGXyIiIiIyCwZfIiIiIjIFBl8iIiIiMgUGXyIiIiIyBQZfIiIiIjIFBl8iIiIiMgVe2orIgV/fdeLt4ytmEjDUP6IPgYiIKNx5TMX33r17kjRpUgkICJDIqnTp0tKpU6dw2TbO28vLS44cOSIf0/Tp0yV+/PjBrtOkSROpWbPmRzumtWvXSt68eSUwMPCj7ZOIiIgiP48JvoMGDZIaNWpI+vTpI/pQZMuWLRpCHz58aHf/4sWLZeDAgdbbOFZcfje0ggqSadKkkZs3b4qfn5+YXaVKlSRatGgye/bsiD4UIiIiikQ8Ivg+e/ZMpk6dKs2bN5fILGHChBInTpxw2XaUKFEkefLkEjUqu1eMNwdjx46N6MMgIiKiSMQjgu/q1avFx8dHihQpYr3v5MmTUrVqVb1mM8JmiRIl5MKFC/oYPgIfMGCApE6dWp+Hj8Xx8bhj2wAqtGXKlBFfX1/JkyeP7N6927rO5cuXpVq1apIgQQKJFSuW5MyZU48Dz8VzAI9hOwhhjq0O+B7b6Ny5s66DBfr166fHYwtVYaOSjcdnzJghy5Ytsz4PFeagWh22bt0qhQsX1nNMkSKF9OrVS968eWN9HMfQoUMH6dGjh4ZyBGds39aoUaMkV65ceo6oKrdp00aePHnyQT8vvNafffaZtkgkSpRIf07Gzwa++uoradeunfU2XjOc299//623X716pcezYcMGp/vAz+bAgQN22yUiIiJz84jgu337dilQoID19vXr16VkyZIa+DZt2iQHDx6UZs2aWUPfmDFjZOTIkTJixAg5duyYVKxYUapXry7nzp2z2+73338v3bp10zCZJUsWqV+/vnUbbdu2lZcvX8q2bdvk+PHjMmzYMIkdO7aGw0WLFuk6Z86c0fYD7M8RQjWCNwI41sHiChxPnTp19ON843nFihV7Zz28BlWqVJFChQrJ0aNHZcKECVoV/+mnn+zWQ4hGiNy7d68MHz5cj2f9+vXWx729vbVyijcSWBevJ4Lyh3j69Kl06dJFg+nGjRt1H7Vq1bL25JYqVUrDvG2AT5w4sfW+/fv3y+vXr4M8b0PatGklWbJk+m/DGfz8Hj9+bLcQERGR5/KIz8VROU2ZMqX19vjx4yVevHgyb9487fUEBFcDAm/Pnj2lXr16ehuhdfPmzVpZxXNtQ6a///8f7d6/f3+t6p4/f16yZcsmV65ckS+//FKroZAhQwbr81A9BQy2czbwC+ugPQHVaFRaXYVwHTNmTA1twT3v119/1RD+yy+/aLUUx3zjxg097x9//FHDJuTOnVv69u2r32fOnFnXRxgtX7683mc7GA9VZwTn1q1b6/bfF143W7///rskSZJETp06pT3KqER37NhR7ty5o60buL9Pnz4afLFvfEWgRyU+OPg3gX8bzgwZMkR/rkRERGQOHlHxff78ucSIEcN6GxVatDYYodcWqnoIgMWLF7e7H7dPnz5tdx9CoQGtAnD79m39ihYBhEA8D8ERlePIBOdStGhRawsF4FjRpnDt2rUgz9E4T+McAe0E5cqVk1SpUmlIb9iwoc6ggb5qR3gzgGBuLIMHDw7y2FBZR/UcbxbQimK0ceD5gPCLNwao9KJimy9fPm2HwG3AV4TjkOANQlDHaejdu7c8evTIuly9ejXEbRIREZH78ojgi4/BHzx4YBd4woJtcDYCpPFx/DfffCMXL17UIIhWh4IFC8q4ceM+eJ+oxFosFrv78LF+eHF8c4DzNM4RfcMInAjHaN9Ay4hREUefbVAVVrzpMBZUZ531396/f1+mTJmiLRZYbLeJY0CrCiq7RsjFMaDKfeLECdm1a5e2Q4QE+0Al2Rm0wiB42y5ERETkuTwi+KIiiI/DDQhJqBQGFRgRbhDQdu7caXc/bufIkSNU+0UrAcId+nW7du2qQQ6iR4+uX9++fRvs87Ge4zoIardu3bILv45z8wb1PEfZs2fXwXi228E5omqL3mJXIOgiBKMfGgMH0S6CarkzaEvIlCmTdTFaPmyhWoze5x9++EEryThO2zctBqPPFwuCL94QIAz/73//0wDsWLF39OLFCx3Yhn8bRERERB4TfDE4DYOvjACFGQHQ0oAeXgygwkfrM2fO1MAF3bt3177e+fPn632Y7QDhEn2lrkLv67p16+TSpUty6NAh7RFGiIN06dJp1XLlypXap+psFgR8xI/BcRiIdvfuXb0PIQ/PwUAzBDdUWNesWfPO89BagWPH84IK+Jh9AR/dt2/fXmdDwCwQaMnAoDKjvzckCK/YNirZqG7jNZw4caJ8CMx0gZkcJk+erP3SGCyHY3KE1wFvZvBzxQwQxn2YmxfVdQzIC86ePXu0oot2DyIiIiKPCb4YYJY/f35ZsGCB3kawQqBC4ETlEDM+oBprfKyP/lyELVRp8VxMr7V8+XId3OUqVFwxswPCLmZYQDXUGPCFflgMmkKgxswCtlNz2cIMCmgnyJgxo/UjeWwP20HgxRRq+/bt00F2tlq0aCFZs2bVAIjnOVavjWPA9Gp4PraDyjTmOUal1VV4HqYzw5sE9N0idGJA2IdA6MagQ1STsU1M54YqriP8XDAwEFO7oV/YCL543R37e3H1ONteZpg7d640aNAgxAFwREREZB5eFseGUje1atUqreSiB9TViiZ5BlSy0QtsTHeGKjjeGKDa/8knn7i8HXxKgNlAMNCN/b5ERETuITR/vz1iOjPAtGNoaUDbAHpvyTzQCoJp2AyooqNqHprQS0RERJ7PYyq+RB+KFV8iIiLP/vvNngAiIiIiMgUGXyIiIiIyBQZfIiIiIjIFBl8iIiIiMgUGXyIiIiIyBQZfIiIiIjIFBl8iIiIiMgUGXyIiIiIyBY+5chtRWPHru068fXzF7AKG+kf0IRAREXlWxffevXuSNGlSvczsli1bxMvLSx4+fBju+23SpInUrFnTert06dLSqVMncQfp06eX0aNHR/RhRFqvXr3S1+jAgQMRfShEREQUiUR48B00aJDUqFFDg0qxYsXk5s2betm59w2wkYU7BenIAm9+8MbnyJEjH7Sd6NGjS7du3aRnz55hdmxERETk/iI0+D579kymTp0qzZs3twaW5MmTa/ihj8tiscibN28itEoblho0aCA7duyQkydPhul2iYiIyH1FaPBdvXq1+Pj4SJEiRfS2Y6vD9OnTJX78+LJu3TrJnj27xI4dWypVqqRVYejXr5/MmDFDli1bps/Dgm3A1atXpU6dOvr8hAkTalUZFUVXvXz5UquGqVKlklixYsmnn35q3bZh586dWtn19fWVBAkSSMWKFeXBgwdahd66dauMGTPGelzYt3E+tpYuXWoX9C9cuKDHmixZMj3fQoUKyYYNG96rcjpv3jytoseIEUP8/Pz0mAzGa71mzRopUKCA/hwQFHHeHTp00PYTPO+zzz6T/fv3v/O8VatWSe7cuXUd/PxOnDhhdwzYVokSJSRmzJiSJk0a3ebTp0+tj6PCP3DgQGnUqJHEjRtXWrZsKZ988ok+li9fPt0HXttt27ZJtGjR5NatW3bbRzUd23cGP4/ixYvra0BEREQU4cF3+/btGrpCqgqPGDFCZs6cqSHoypUrGkgBXxFujTCMBUHv9evXGkLjxImj+0BANUKzq5XFdu3aye7duzU4HTt2TGrXrq3PP3funD6Oj+PLlSsnOXLk0PUQ9KpVqyZv377VwFu0aFFp0aKF9bgQ/lzx5MkTqVKlimzcuFEOHz6s+8R2cd6h1b17d+natatuB8eD7aCn2lavXr1k6NChcvr0aQ2yPXr0kEWLFukbikOHDkmmTJn0tbx///472x45cqSG4iRJkui28bob4R3H/eWXX+prN3/+fH198Jraws81T548enx9+vSRffv26f0I+njNFi9eLCVLlpQMGTLoz9+A/cyePVuaNWsW7PkXLlxYf/7OIOQ/fvzYbiEiIiLPFaHB9/Lly5IyZcpg10HImThxohQsWFDy58+v4QmhEBBmUVFEtRItEljQLoGgFRgYKL/99pvkypVLq8XTpk3T8OhYtQ0K1sP6Cxcu1KpixowZNWSj+on7Yfjw4XpMv/76q4a3nDlz6rElTpxYe5RxHKgEG8cVJUoUl14TbKtVq1Zaoc2cObNWRbH/5cuXS2jheBA+cf4TJkzQ40Jria0BAwZI+fLldR94HbHe//73P6lcubKG+ilTpuhr7Pi8vn376vPw+iIk//PPP7JkyRJ9bMiQIdpqgKoszgFvRsaOHSt//PGHvHjxwrqNsmXLajDHvrEgQEOiRIn0NUOlHtAKY7zusGLFCt0O3vQEB/+28G/MGRwnXhNjcfXNCREREbmnCA2+z58/14/Kg4PwiFBkSJEihdy+fTvY5xw9elTOnz+vFV+EYywIUQhLqEaG5Pjx41q5zZIli/X5WNAqYDzfqPiGNVR8EbIRVtEWgf2iGuus4tu6dWu7Y7SFKq8hatSoGtSxLVu4z4BzwxsNtAgY0GaAyqnj82y3jdc2a9as1nXw+qOtw/a4UDXGm5FLly4Fue/goHUEP889e/bobWwboRctKMFBYMcnBs707t1bHj16ZF3QHkNERESeK0Ln8UV1FD2xwUHwsoXeTwzECik8ooUCH4c7MqqKIT0fFdqDBw++U6k1wiVCVWh5e3u/c+xGe4ABoXf9+vXaBoA2A+znq6++ctqigYqt0frxPkIKj+8Drx+q1ujrdZQ2bdpQ7xv9xmilQNUXfcDoS3alco/2jOB+3qhwYyEiIiJziNDgi0FMs2bN+qBtoKUA1VlbaIlAuwMCEwZOvc9xYZuoLDsbQIV+WLRc9O/f3+XjQgj7999/dZCXEfocp+5CPzIqnLVq1bKGyOAG5eEcsQQFFVL0yAJmbECQd+yztYXKOo4bx5AuXTprMEcfr+PUbNi2EWLx5uXs2bNapTZe/1OnTmlwDw3sGxxfN/jmm2+kfv36kjp1aj1O26q0Mxhwh58lERERUYS3OuDjb0w3FVLVNziYHQADqM6cOSN3797VoIb+UlSTMTsCBjfh43VUCFGBvHbtWojbRIsDtoEZBzDACs/HwCv0hGI2A+NjcgTCNm3a6P7//vtv7Y/FMRjHtXfvXg2tuA8f82NmCLRufPfdd9pWMGfOHP3Y3hZ6YrFPBGK0DHz99df63Pcxfvx47bvFsbVt21Zf5+AGhCGMf/vttzpwbe3atRpeMUAP7QLGlHO2lWYEf4RLBHW83sZ8ypg/d9euXRqycR4YEIiZN4IL3YAAjwo39o2eYbQf2P5bwZuYn376SZo2berS+eNnX6FCBZfWJSIiIs8XocEXA6NQHVywYMF7bwPBDP2l6BdFRRXVSoRLzACBiuQXX3yhlUgEN/T4uloBxsfqCL4YfIXtI9Qh6BpVToTjv/76S8MpemDR84pwh15aQPsB2iQwQAzHhR5d9MKiwo1p3HDuc+fO1SnZbI0aNUqn4sKAMHy8j8CH1+h9YLYGLBgwh1kVMEAOATWk52BAXMOGDXW/6K3FdHI4Jsf1OnbsqC0lmGoMA86Mii2q4eiHRhUYFXNUXX/88ccQBzLitcMguEmTJum6eONi2yaCgI1qMH4ujtACY/smAjNtIDijTYSIiIgIvCwhNcyGM1RQUWFE5RDhhj4cqszohcU0YXnz5g3TbaNyXqZMGa0eO85JHN7w5uXOnTvvzHCBijzeiKBCjYo51K1bVwM/quuuwnRmmN0Bgfl9WmSIiIjo4wvN3+8I7fEFf39//Sj8+vXrnE6KgoR/yJhpA60hQU3rhgo6LoBhhF4MBERFvXPnzhFwtERERBRZRXjwBceBU0S20PKAHmtM3Ya5gx2hf9kWWi5++OGHj3iERERE5JGtDhio9fvvv2sPpXEZWVxsAD2p6MF0ZbowosiIrQ5ERESe/fc7VE21GNyFXkoMQMIOMFUWFnyP+7JlyyYHDhz40OMnIiIiIorYim+RIkV0wBAuIYxR9LawGXwUjam9UA0mcjes+BIREbmfcBvcZlyK1jH0Au7DYCJeMICIiIiIIqNQtTqglxeDjJzBY8mSJQuL4yIiIiIiClOhqvjiogyYNgqXvi1Xrpw15OIqW7iK15QpU2TEiBFhe4RERERERB87+GLaKFz56+eff5Zff/1Vr6IFuEIZruCFNog6deqExXEREREREUWOK7e9fv1apzYDhOFo0aKF7ZERfWQc3EZEROR+PsqV2xB0U6RI8b5PJ4q0/PquE28f34g+DI8QMNQ/og+BiIjo/Qa3kXu7d++eJE2aVAICAj76vjHrx9KlSz/a/urVqycjR478aPsjIiKiyI/B10QGDRqkl/9Nnz69bNmyRcPow4cPxRPhksU4X3zsQURERAQMvibx7NkzmTp1qjRv3lzMwM/PTzJmzCizZs2K6EMhIiKiSILB1yRWr14tPj4+evU9tDqUKVNG70+QIIFWfps0aaK3165dK5999pnEjx9fEiVKJFWrVpULFy5Yt/PHH39I7Nix5dy5c9b72rRpo5erRrh2Vc+ePfXy176+vpIhQwbp06ePDpgEVGkxU4hx+evAwEBJmDChHrsBgTZNmjTB7qNatWoyb948l4+JiIiIPBuDr0ls375dp5wDBMZFixbp92fOnJGbN2/KmDFj9PbTp0+lS5cuGjoxN7O3t7fUqlVLwyc0atRIqlSpIg0aNJA3b97IqlWr5LfffpPZs2driHVVnDhxdPq7U6dO6b4xBzSmyQOMzMybN6+2Y8Dx48c1nB8+fFiePHmi923dulVKlSoV7D4KFy6sF1V5+fJlkI/jfowEtV2IiIjIczH4msTly5clZcqU+j2qqaigAga74Yp8CJvw5ZdfyhdffCGZMmXS8Pn7779r8ERANUyaNEnDcocOHbR1ol+/ftZQHZoe3GLFimm/MSqzuDjKggULrI+XLl3aGnzxtXz58pI9e3bZsWOH9b6Qgi/O99WrV3Lr1q0gHx8yZIiet7GEVEEmIiIi98bgaxLPnz+XGDFihLgeWhjq16+v7QeYCw/BFK5cuWJdB+0R6BeeMGGC9tH26tXL+tjgwYO1FcJYbJ9na/78+VK8eHEN3VgPQdh2XYRahFxcJAXVXQRhIwzfuHFDzp8/r7eDEzNmTP3qrAWjd+/e2lZhLFevXg3x9SEiIiL3xeBrErjIyIMHD0JcD9XX+/fva+vB3r17dQFUTm1t27ZNK8eo/KI9wtC6dWs5cuSIdTGqzLZ2796trRJomVi5cqW2MHz//fd2+yhZsqT8+++/cujQId2XbfBFEMZ2M2fOHOy54DwgSZIkQT6OnmeEe9uFiIiIPBeDr0nky5fPrl0hevTo+tW47LQxzy96flF9LVeunLYWBBWWd+3aJcOGDZMVK1ZotbZdu3bWx9BCgTYJY4kaNWqQz0+XLp2G3YIFC2qARSuGLQyuy507t/zyyy96sRQMnkMYRkhGWA6pzQFOnDghqVOn1tBPRERExOBrEhUrVpSTJ09agyyCJwaMIUTeuXNHB42hhQEzOUyePFlbCTZt2qQD3WyhCtuwYUPt761cubIOakPbwp9//unysSDooq0BMy5gxoixY8fKkiVL3lkPFV5s3wi5CNUI49ifK8EXA/oqVKjg8nERERGRZ2PwNYlcuXJJ/vz5rQPIUqVKJf3799f+3GTJkmnVFjM4IIwePHhQ58Ht3Lmz/O9//7PbTseOHSVWrFjay2tsF9+3atVKrl+/7tKxVK9eXbeNfWIAHSrAmM7MEcItKtK2vbz43vE+435jSjZ48eKFXimuRYsWoXyliIiIyFN5WSwWS0QfBH0cmHqse/fu2gKAkOtJUMFGkDfCLwbeoYr8119/ubwNTGeG2R0w0I39vkRERO4hNH+/323AJI/l7++vszagMutJU3ehhQP/4DHHsAF9wePGjYvQ4yIiIqLIhRVfov+w4ktEROTZf7896/NuIiIiIiInGHyJiIiIyBQYfImIiIjIFBh8iYiIiMgUGHyJiIiIyBQYfImIiIjIFBh8iYiIiMgUGHyJiIiIyBR45TYiB35914m3j29EH4bHCRjqH9GHQEREJseKr0ncu3dPkiZNKgEBAR+0nSZNmkjNmjUlMuvVq5e0b98+og+DiIiIIhlestgkunTpIv/++69MmTLlg7aDywHin0z8+PElsrp7965kyJBBjhw5ol9De8nDNJ0WsOIbDljxJSKi8MBLFpOdZ8+eydSpU6V58+bvvY23b99KYGCg/sOKzKEXEidOLBUrVpQJEyZE9KEQERFRJMLgawKrV68WHx8fKVKkiN7esmWLeHl5yapVqyR37twSI0YMfezEiRPW50yfPl0D7vLlyyVHjhz6/CtXrti1OkyePFlSpkypgdhWjRo1pFmzZvr9hQsX9HayZMkkduzYUqhQIdmwYYPd+i9fvpSePXtKmjRpdD+ZMmXSoI7KMr4fMWKE3fqo5OL4z58/7/Scq1WrJvPmzQuDV4+IiIg8BYOvCWzfvl0KFCjwzv3du3eXkSNHyv79+yVJkiQaFl+/fm1XKR42bJj89ttvcvLkSe0RtlW7dm3tHd68ebP1vvv378vatWulQYMGevvJkydSpUoV2bhxoxw+fFgqVaqk+0GINjRq1Ejmzp0rY8eOldOnT8ukSZM0JCPcIkBPmzbNbr+4XbJkSQ3FzhQuXFiuXbsWbE8zAjc+HrFdiIiIyHMx+JrA5cuXtTLrqG/fvlK+fHnJlSuXzJgxQ/755x9ZsmSJ9XGE4F9//VWKFSsmWbNmFV9f+77XBAkSSOXKlWXOnDnW+/78809tNShTpozezpMnj7Rq1Ur8/Pwkc+bMMnDgQMmYMaNWkuHs2bOyYMEC+f3336VWrVrak1uuXDmpW7euPo4K85kzZ2Tfvn3WY8L+jIqyM8b54tydGTJkiLZuGAsqzkREROS5GHxN4Pnz59rO4Kho0aLW7xMmTKjhFhVXQ/To0bUVIjio7C5atEirpzB79mypV6+eeHt7Wyu+3bp1k+zZs2vrBCq52IdR8UXbQpQoUaRUqVJOA6y/v78GY1ixYoXuC9Xm4MSMGdNatXamd+/e2ghvLFevXg12m0REROTeGHxNABXYBw8ehPp5CI9oNwgO2hbQi4t+YQRHtFUYbQ6A0Isq8uDBg/UxBF1UmF+9emXdR0i++eYb7ddFgEebA6rBjtVnR2i5ALRwOIN+Yoz+tF2IiIjIczH4mkC+fPnk1KlT79y/Z88e6/cIxmg7QGU2NFBJ/uKLL7TSiz5dVI3z589vfXznzp3aroA2BgTe5MmT2/Xd4j4Mjtu6davTfaBHOFasWDpLA/qHQ2pzAAzUixYtmuTMmTNU50NERESei8HXBDC1FwanOVZ9BwwYoIPOEBIRTlEZfp+LU6DCi4ov2hFsq72Avt7Fixdrpffo0aPy9ddf280CkT59emncuLGG2aVLl8qlS5d01gn0/RrQCoHjQ2sCtmfbouEMqsslSpRwqaJMRERE5sDgawKoqqIKaxsmYejQodKxY0ed8eHWrVvaP4u+3tAqW7as9ghjEBqCra1Ro0bpIDgMkENbBEK4bUUYUMn96quvpE2bNpItWzZp0aKFPH361G4dzEGM9oimTZu+s3+E4tKlS9vdh9YIbIeIiIjIwCu3mQQqspi+DNXdbdu26awLqABH9otR2FZwMdsD+ogxJ7AtDIzD+fTr109vr1mzRrp27SrHjh2TqFGjhsuVX4iIiChyCM3fb9dTAbk1zIxw7tw5uX79urgTzOBw584dDbWYycEx9OIfOS6SgWBvQLUYg+BCE3qJiIjI87HVwUQ6derkdnPVYsBcunTp5OHDhzJ8+PB3Hsc7PFyoAtOkGdA28emnn37kIyUiIqLIjq0ORP9hqwMREZFn//1mxZeIiIiITIHBl4iIiIhMgcGXiIiIiEyBwZeIiIiITIHBl4iIiIhMgcGXiIiIiEyBwZeIiIiITIGXtiJy4Nd3nXj7+Eb0YXi0gKH+EX0IRERkQqz4erB79+5J0qRJJSAgQLZs2SJeXl56BTRP9+rVK0mfPr0cOHAgog+FiIiIIhEGXw82aNAgqVGjhobAYsWKyc2bN/XKJq5q0qSJ1KxZUyITnMvo0aODXSd69OjSrVs36dmz50c7LiIiIor8GHw91LNnz2Tq1KnSvHlzaxhMnjy5Vn0jogL7sTVo0EB27NghJ0+e/Oj7JiIiosiJwddDrV69Wnx8fKRIkSJ627HVYfr06RI/fnxZt26dZM+eXWLHji2VKlXSqjD069dPZsyYIcuWLdPnYcE24OrVq1KnTh19fsKECbWqjHYKx0oxKs4pU6aUrFmz6uPYxuLFi6VMmTLi6+srefLkkd27d9sdN8JqiRIlJGbMmJImTRrp0KGDPH36VB8rXbq0XL58WTp37mw9JmcSJEggxYsXl3nz5oXDq0tERETuiMHXQ23fvl0KFCgQYlV4xIgRMnPmTNm2bZtcuXJFWwQAXxFujTCMBe0Sr1+/looVK0qcOHF0Hzt37rSGZtvK7saNG+XMmTOyfv16WblypfX+77//Xrd95MgRyZIli9SvX1/evHmjj124cEG38+WXX8qxY8dk/vz5GoTbtWunjyM0p06dWgYMGGA9puAULlxYj9GZly9fyuPHj+0WIiIi8lyc1cFDoTKKamtwEGInTpwoGTNm1NsImAiVgDCLqivCIVokDLNmzZLAwED57bffrBXXadOmafUXFeEKFSrofbFixdJ10GIBRkUYodff//+P6O/fv7/kzJlTzp8/L9myZZMhQ4Zoi0KnTp308cyZM8vYsWOlVKlSMmHCBK0uR4kSRUO37TE5g/PH6+AM9odjICIiInNgxddDPX/+XGLEiBHsOmg3MEIvpEiRQm7fvh3sc44ePapBFeET4RgLAumLFy+0YmvIlSuXNfTayp07t93+wNgnto0WDGO7WFBdRtC+dOmShBaCO6razvTu3VsePXpkXdDCQURERJ6LFV8PlThxYnnw4EGw60SLFs3uNiq4Fosl2Oc8efJEWyhmz579zmNJkiSxfo+Kb0j7NCrGCLbGtlu1aqV9vY7Spk0roXX//n27Y3KEHmgsREREZA4Mvh4qX7582pbwIVCxffv2rd19+fPn195bzA8cN25cCUvY9qlTpyRTpkyhOiZnTpw4oa8DEREREbDVwUOhRQBTeYVU9Q1pzlwMMsMgtbt372pPMHpwUU3GTA4YOIYWBPT2okp77dq1DzpmzLu7a9cu7TXG4Ldz587prBLG4DbjmDAQ7/r163pMwcHxGT3HRERERAy+Hgo9tqigLliw4L230aJFC52KrGDBgtoygBkc0BeM4InWgy+++EKnQsNcwejx/dAKMPp/t27dKmfPntUpzVCt/fHHH+0G6WHwHQbKoTfZto0BbRPoDzZgmjT07X711VcfdExERETkObwsITV1kttatWqVdO/eXT/y9/b23Pc4qDpjajS0SWAmCKhbt67OE/zdd9+5vB1MZ4Yr2yEwh3UbBxEREYWP0Pz9Zo+vB8O0YWgXQFsALgbhyRfraNmypTX0Yj5hVLxxoQsiIiIiAyu+RP9hxZeIiMiz/3577uffREREREQ2GHyJiIiIyBQYfImIiIjIFBh8iYiIiMgUGHyJiIiIyBQYfImIiIjIFBh8iYiIiMgUGHyJiIiIyBQYfImIiIjIFHjJ4nBy7949yZ49u+zbt0/Sp08fYccxffp06dSpkzx8+FDM4u7du5IjRw45dOiQpE6dOtTP9+u7Trx9fMPl2Mg1AUP9I/oQiIjIA7HiG04GDRokNWrUiNDQ64m8vLxk6dKlwa6TOHFiadSokfTt2/ejHRcRERFFfgy+4eDZs2cydepUad68eUQfimk1bdpUZs+eLffv34/oQyEiIqJIgsE3HKxevVp8fHykSJEi1vtOnDghlStXltixY0uyZMmkYcOG+pG8oXTp0tK+fXttS0iQIIGuM2XKFHn69KmGuDhx4kimTJlkzZo11uds2bJFK6CrVq2S3LlzS4wYMXSf2FdwJkyYIBkzZpTo0aNL1qxZZebMmdbHmjVrJlWrVrVb//Xr15I0aVIN8+97rK6+Bh06dJAePXpIwoQJJXny5NKvXz/r40b1vFatWnrewVXTc+bMKSlTppQlS5YE+1oQERGReTD4hoPt27dLgQIFrLfRX1u2bFnJly+fHDhwQNauXSv//POP1KlTx+55M2bM0I/p0ReMYPntt99K7dq1pVixYtqvWqFCBQ2LqCjb6t69u4wcOVL2798vSZIkkWrVqmlYDQqCYMeOHaVr164aRFu1aqVhdfPmzfr4N998o8d38+ZN63NWrlyp+6xbt+57H2toXoNYsWLJ3r17Zfjw4TJgwABZv369Pobzg2nTpunxGbedKVy4sP4snHn58qU8fvzYbiEiIiLP5WWxWCwRfRCepmbNmpIoUSJrhfSnn37SALZu3TrrOteuXZM0adLImTNnJEuWLFrtfPv2rTWo4ft48eLJF198IX/88Yfed+vWLUmRIoXs3r1bK7uo+JYpU0bmzZtnDaX4aB8DujCoDaHScXBb8eLFtRo6efJk67FgPVRrUTkGPN64cWOtvEL16tX1fBA44X2O9X1eAyO8IjAPHTpUb6PSi/CO1zgkXbp0kcOHD1tDvSNUk/v37//O/Wk6LeDgtgjGwW1EROQqFK6QQx49eiRx48YNdl1WfMPB8+fPte3AcPToUQ1f+IjfWLJly6aPXbhwwboe2hUMUaJE0bCZK1cu631oD4Dbt2/b7a9o0aLW79EigPaF06dPB3lsuB/h1xZu266Pqq8RclGVRcsCWiBshfZY3+c1AIRnx/N1VcyYMd+pjtvq3bu3/k9iLFevXn2v/RAREZF74HRm4QAtAA8ePLDefvLkibYfDBs27J11EewM0aJFs3sM1U3b+3AbAgMDJTxhRoRevXpptXbXrl3yySefSIkSJezWCe2xfshr8L7ni+o3Wj+cQR82FiIiIjIHBt9wgD7WWbNmWW/nz59fFi1apIOxokYN+5d8z549kjZtWv0egfvs2bM6h3BQcP/OnTu1lcGA25j31oDqLVoJUPVF+EUP8IcKq9cAwRjtEK5ADzPaJ4iIiIiArQ7hoGLFinLy5Elr1bdt27Zafaxfv74OyMJH++h1RaB0NcQFBwPANm7cqEGvSZMmWnF21gOLgXDo+8XMDufOnZNRo0bJ4sWLpVu3bnbrod0BA83QAmEbkt9XWL0GCM44V/QQ21bVHaHF4eDBgzrIjoiIiAgYfMMBel1R4VywYIHexrRaqKoi4CGI4XEMOIsfP754e3/4jwADvzBTA2aSQCBcsWKFTlUWFATiMWPGyIgRI3QQ26RJk7Sy61gZ/fzzz7UFASEex/+hwuo1wOwVmOUBg+JQWYeAgABticBgP8OyZcu0Cu7YokFERETmxVkdwglmSEB1FVXYsAi3QTFmdUDlEwEyLKEnN1WqVBqKMVtDZIZBczjGixcv6rzCgJkkMCfw119/HS6jQomIiChyCM3fb/b4hhN/f39tJbh+/bpWJ90FBpLhohKorCJMYyozd7hgyHfffWcNvTh+BGG0VRAREREZWPF1Y+FR8UXbAGZxMOYCLleunJgFK75ERETuhxVfk0Bfbli/b8HgMb4XIiIiIk/EwW1EREREZAoMvkRERERkCgy+RERERGQKDL5EREREZAoMvkRERERkCgy+RERERGQKDL5EREREZAoMvkRERERkCryARSRz7949yZ49u+zbt08vJhHecHW2Tp06ycOHDyUyMq4kd/jwYcmbN69Lz+nVq5c8ffpUxo0b91779Ou7Trx9fN/ruRT2Aob6R/QhEBGRh2DFN5IZNGiQ1KhRQ0MvLkns5eUVaUNpZNWtWzeZMWOGXLx4MaIPhYiIiCIRBt9I5NmzZzJ16lRp3rx5qJ736tUr8UTve16JEyeWihUryoQJE8L8mIiIiMh9MfhGIqtXrxYfHx8pUqSIfsRfpkwZvT9BggRa+W3SpIneLl26tLRr105bFIyQB6NGjZJcuXJJrFixJE2aNNKmTRt58uTJO60NadOmFV9fX6lVq5a2VjhatmyZ5M+fX2LEiCEZMmSQ/v37y5s3b6yP41gmTZokVatW1e2gNWP37t1y/vx5PTbsv1ixYnLhwgXrc/A9KtnJkiWT2LFjS6FChWTDhg12+0WVe+DAgdKoUSOJGzeutGzZ8p1je/v2rTRr1kyyZcsmV65ccfpaVqtWTebNmxeKV5+IiIg8HYNvJLJ9+3YpUKCAfo/gumjRIv3+zJkzcvPmTRkzZox1XXyUHz16dNm5c6dMnDhR7/P29paxY8fKyZMn9fFNmzZJjx49rM/Zu3evVpMRmo8cOaLB+qeffnrnGBA8O3bsKKdOndKAi7CMFgxbRkDFdhBCv/76a2nVqpX07t1bDhw4IBaLRfdjQACvUqWKbNy4Uft1K1WqpOHUMbyOGDFC8uTJo+v06dPH7rGXL19K7dq1dZ84TgR4ZwoXLizXrl3TNxDOYHuPHz+2W4iIiMhzeVmQUChSqFmzpiRKlEjbHQA9vginDx48kPjx41vXQ1UVIe3QoUPBbu/PP/+U1q1by927d/U2wumjR49k1apV1nXq1asna9eutfYRf/7551KuXDkNsIZZs2ZpgL5x44a14vvDDz9o+IU9e/ZI0aJF9bhRjQVUW5s2bSrPnz93enx+fn56fEZARsU3X758smTJkncGtyHo9uvXT8PqypUrJV68eMGeO14frIPXsFSpUkGug+2hmu0oTacFHNwWiXBwGxERufI3HxkHnxgHhxXfSAQhEe0FrjAqw7bQOoDQmipVKokTJ440bNhQWxnQOwynT5+WTz/91O45CKy2jh49KgMGDNB2BGNp0aKFVpyN7UDu3Lmt36N9AdBmYXvfixcvrFVUVHwx6AxtEQjx2C6Ox7HiW7BgwSDPt379+jpTw19//RVi6IWYMWPqV9tjdoRwj/9JjOXq1ashbpeIiIjcF4NvJIJ+XVR3XYE+WluojKLnFoEULRIHDx6U8ePHh3qQGAIqqqBoJzCW48ePy7lz5+xCebRo0azfowLs7L7AwED9itCLSu7gwYO1eovtIig7HpvjeRnQJnHs2DHtJXbF/fv39WuSJEmcroN+arwztF2IiIjIc3Ee30gEH/OjrcCAHl5jQFdIEHQRMkeOHKm9vrBgwQK7dVBtRZ+vLbQp2MKgNvQUZ8qUScISepExOA8D6oyAHVz/raNvv/1WWyOqV6+urRrO2hcMJ06c0CCeM2fODz52IiIi8gys+EYimJ0BA9OMqm+6dOm0coqe1jt37rwzQ4MtBNXXr1/rRRswf+3MmTOtg94MHTp00H5eDCBDBfeXX37R27Z+/PFH+eOPP7Tqi2NBOwL6ddHT+yEyZ84sixcv1kov2inQb2xUg13Vvn17HYyHyvaOHTuCXRdV5RIlSlhbHoiIiIgYfCMRfPSPiqtRqUWvLgIorkSGnlnbWRIcYSYETGc2bNgwrYzOnj1bhgwZYrcOpkmbMmWKzg6B9dEv6xhoEb4RtPEYphzDc37++WcN4R8Cx4Zp2TDNGWZzwH5wrqGFKdzwmqD1YdeuXXofKskY8GcLYR29yUREREQGzuoQyeBj/O7du+tH9UbLAgUPbQ+Y/QKzNMCaNWuka9eu2hMcNWrUcBkVSkRERJFDaP5+s8c3kvH399c2hOvXr+tcvhQ8/CPHxTFsp2jD7A/Tpk0LVeglIiIiz8eKL9F/WPElIiJyP5zHl4iIiIjIAYMvEREREZkCgy8RERERmQKDLxERERGZAoMvEREREZkCgy8RERERmQKDLxERERGZAoMvEREREZkCgy8RERERmQKv6RqO7t27J9mzZ5d9+/ZJ+vTpxaz69esnS5culSNHjnyU/Z06dUoqVKggZ86ckVixYoX6+X5914m3j2+4HBuR2QQM9Y/oQyAismLFNxwNGjRIatSo8dFC7/Tp0yV+/PjyseC8Ro8eLZFNjhw5pEiRIjJq1KiIPhQiIiKKRBh8w8mzZ89k6tSp0rx5c4lsXr16JZ6uadOmMmHCBHnz5k1EHwoRERFFEgy+4WT16tXi4+OjlUfDyZMnpWrVqhI3blyJEyeOlChRQi5cuKCPBQYGyoABAyR16tT6vLx588ratWutzw0ICBAvLy9ZvHixlClTRnx9fSVPnjyye/dufXzLli0a9h49eqTrYUGLgVGZHThwoDRq1Ej33bJlS72/Z8+ekiVLFt1WhgwZpE+fPvL69Wu781ixYoUUKlRIYsSIIYkTJ5ZatWrp/aVLl5bLly9L586drftz1f79+6V8+fK6vXjx4kmpUqXk0KFD1se7deumr5MBVWVs3/b1yJQpk/z2229O94Ht379/X7Zu3erycREREZFnY/ANJ9u3b5cCBQpYb1+/fl1KliypoXbTpk1y8OBBadasmbUiOWbMGBk5cqSMGDFCjh07JhUrVpTq1avLuXPn7Lb7/fffazBEvyxCa/369XUbxYoV04CIYHvz5k1dsJ4B20VQPnz4sAZcQPhGewR6YrH/KVOmyM8//2x9zqpVqzToVqlSRZ+3ceNGKVy4sD6GAI6QjrBu7M9V//77rzRu3Fh27Nghe/bskcyZM+s+cD8gCOOxt2/f6m2EV4RkhHvjtcQbBoRvZ6JHj65vHvBzICIiIgIObgsnqIamTJnSenv8+PFa3Zw3b55EixZN70NwtQ2mqMDWq1dPbw8bNkw2b96sYRbPNSDM+vv//8Ei/fv3l5w5c8r58+clW7Zsun1URpMnT/7O8ZQtW1a6du1qd98PP/xg/R5VYWwbx9ejRw9rjzKOB/sxIDxDwoQJJUqUKBqeg9pfcHAstiZPnqy9yQi4qPSiEo4QjLCNNw/btm2T7t276wA5QABOlSqVVn2Dg9cfPwdnXr58qYvh8ePHoToPIiIici+s+IaT58+fa3uAARVaBDoj9NpC4Lpx44YUL17c7n7cPn36tN19uXPntn6fIkUK/Xr79u0Qj6dgwYLv3Dd//nzdB4Jr7NixNQhfuXLF7pjLlSsnoYHtGEvr1q2DXOeff/6RFi1aaKUXYR1V6idPnlj3jRCMgI2Ae/z4ca3eoj0DQRjrISCjKhySmDFjaq+1M0OGDNH9G0uaNGlCda5ERETkXljxDSf4aP7Bgwd2ISws2AZno68W/cEhcZzWC73BDRo00Gou2iqMajTaLT7kmG2nLEOgDQraHDDVG9or0qVLp+0fRYsWtRt0hzYGBF88hpCLCjOmhkMLBIKvY/U6KOjxzZgxo9PHe/fuLV26dLF7A8LwS0RE5LlY8Q0n+fLl095Z20ot+k0dB48ZAREfy+/cudPuftzG1FyuQmXU6IsNya5duzR0omcY1WBUXx3bAnDM6OsNzf7QfmAsSZMmDfJ5OK8OHTpoXy9aNRBu7969a7eO0eeL/Ru9vPg6d+5cOXv2bLD9vYYTJ07oz8EZ7Bevve1CREREnovBN5ygiopZHIyqb7t27bSiiJ7ZAwcO6KC1mTNn6kUWAD2s6OtF+wHu69Wrl1ZPO3bs6PI+0aeLVgCERQTJ4D7mR9BFawGqvBgoNnbsWFmyZIndOn379tWgia9ouUDbAY7Rdn/ov8VgM8fgGhzsG+eObe7du1crz47VZQwERJ/vypUr7YLv7NmztcXDtj86KJgFA8f1+eefu3xcRERE5NkYfMNJrly5JH/+/LJgwQK9nShRIp3NAcEU1UwM2sIsCkbrAiqg+NgdH+HjuZi6a/ny5RoSXYWZHdBXW7duXUmSJIkMHz7c6bqYMQJTkSGQY/YDVICN2R4MCJoLFy7U48A6GJSGq9AZMKMDAibaCbA/V2F+Y7whwOvTsGFDPXfH6nCCBAn0dcB2MXDPCMNo63Ds70VLBNo+cCwGBHZcvQ1VbSIiIiLwslgsFr4U4QPTgaGSi4/cvb35HiO8TJs2TQYPHqytJXgjgV5hvGGYM2fOOwMGg4OKPHqdMRcy2x6IiIjcQ2j+fnNwWzjCtGNoacBH7hw0Fb4XC0HwNarnaOH47rvvQhV6iYiIyPOx4kv0H1Z8iYiIPPvvNz9/JyIiIiJTYPAlIiIiIlNg8CUiIiIiU2DwJSIiIiJTYPAlIiIiIlNg8CUiIiIiU2DwJSIiIiJTYPAlIiIiIlNg8CUiIiIiU+Aliz+Ce/fuSfbs2WXfvn2SPn3699oGntepUydd3le/fv1k6dKlcuTIEQkrW7ZskTJlysiDBw8kfvz4H22/wTl16pRUqFBBzpw5I7FixQr18/36rhNvH99wOTYishcw1D+iD4GITIQV349g0KBBUqNGjfcOvbB//35p2bLlBx1Ht27dZOPGjeLpcuTIIUWKFJFRo0ZF9KEQERFRJMLgG86ePXsmU6dOlebNm7/X81+9eqVfkyRJIr6+H1aFjB07tiRKlEjMoGnTpjJhwgR58+ZNRB8KERERRRIMvuFs9erV4uPjoxVIw8mTJ6Vq1aoSN25ciRMnjpQoUUIuXLigjzVp0kRq1qypVeKUKVNK1qxZ9X5Ui0ePHm3dhpeXl0yaNEm3g0CMVordu3fL+fPnpXTp0voRf7FixazbNVoO8ubNa71t7GvEiBGSIkUKDcVt27aV169fW9eZOXOmFCxYUI8zefLk8vXXX8vt27c/6DVB9bp8+fKSOHFiiRcvnpQqVUoOHTpkV5nGeRlw3jjftWvXWu/LlCmT/Pbbb073ge3fv39ftm7d+kHHSkRERJ6DwTecbd++XQoUKGC9ff36dSlZsqSG4U2bNsnBgwelWbNmdpVJtCOgP3X9+vWycuVKp9seOHCgNGrUSHtns2XLpqG0VatW0rt3bzlw4IBYLBZp165dsMe3efNmDcf4OmPGDJk+fbouBoRg7Ofo0aPapxsQEKCB+UP8+++/0rhxY9mxY4fs2bNHMmfOLFWqVNH7AUEYj719+1ZvI7wiJKOf2HgNccwI+M5Ejx5dQz5efyIiIiLg4LZwdvnyZa3cGsaPH69Vznnz5km0aNH0vixZstg9B9VaVDMR3kL6OL9OnTr6fc+ePaVo0aLSp08fqVixot7XsWNHXSc4CRIkkF9++UWiRImi4dnf31+Dd4sWLfRxhHJDhgwZZOzYsVKoUCF58uSJtk68j7Jly9rdnjx5sg6MQ8BFpRcVcITgw4cP65uGbdu2Sffu3TV4AwJwqlSptOobHLzueP2defnypS6Gx48fv9f5EBERkXtgxTecPX/+XGLEiGG9jeosgp0ReoOSK1euEEMv5M6d2/p9smTJrM+1ve/FixfBBrqcOXNq6DWg5cG2lQEV6WrVqknatGm13QHVWLhy5UqQ20MYNpbWrVsHuc4///yjwRqVXrwJQMsHgrSxTYTgPHnyaMA9fvy4vhYY2IcgjPUQkI3jCE7MmDG1x9qZIUOG6P6NJU2aNCFuk4iIiNwXK77hDB/RY6ov2zAWElen4LINz+iBdXZfYGCgS9swnmOs//TpU60eY5k9e7YOsEM4xW1j0J0j2ynLEGiDgjYHTPE2ZswYSZcunbZ9oFptu020MSD44jGE3IQJE2ofM1ogEHy7du0a4uuDHt+MGTM6fRwtIV26dLHexhsEhl8iIiLPxeAbzvLlyyezZs2yq9Kilxa9s8FVfSODv//+WwPq0KFDrYEQvcPBCan9AHbu3Cm//vqr9vXC1atX5e7du3brIOz+/vvvEjVqVKlUqZI1DM+dO1fOnj0bbH+v4cSJE/LVV185fRyhGgsRERGZA1sdwhmqo5jFwaj6YrAZKov16tXTEHnu3DmdOQGD2SIbtDegzWDcuHFy8eJFWb58uQ50+1BoccA5nz59Wvbu3SsNGjR4pxKOAYDo88XgPiPk4isqz2jHcOyLdoRBeBgE9/nnn3/w8RIREZFnYPANZ+i5zZ8/vyxYsEBvY8owzOaAXlVUNTF4a8qUKZGy+ovWBszwsHDhQr0oBCq/mPrsQ2FeY7wRwOvSsGFD6dChgyRNmvSdQXd47XAMGHRnhGG0YTj296IlAi0aCLsGVIZx9Ta0UhARERGBlwVzXlG4WrVqlc5KgI/evb35XiOsTZs2TQYPHqyXKsYbCPQKo6o8Z84cKV68uMvbQSUeg9wePXrktD+ZiIiIIpfQ/P1mj+9HgCnC0NKAj945eCp8LhKC4GtUzTEA77vvvgtV6CUiIiLPx4ov0X9Y8SUiIvLsv9/83J2IiIiITIHBl4iIiIhMgcGXiIiIiEyBwZeIiIiITIHBl4iIiIhMgcGXiIiIiEyBwZeIiIiITIHBl4iIiIhMgcGXiIiIiEyBlywmcuDXd514+/hG9GEQERF5lICh/hF9CKz4upt79+5J0qRJJSAgQCKzLVu2iJeXlzx8+NDl5/Tr10/y5s0bJvsvUqSILFq0KEy2RURERJ6BwdfNDBo0SGrUqCHp06eXyKxYsWJy8+ZNvXZ2WCpdurR06tQpxPV++OEH6dWrlwQGBobp/omIiMh9Mfi6kWfPnsnUqVOlefPmEpm9fv1aokePLsmTJ9eqb0SoXLmy/Pvvv7JmzZoI2T8RERFFPgy+bmT16tXi4+OjH+MbTp48KVWrVpW4ceNKnDhxpESJEnLhwgV9DNXOAQMGSOrUqfV5aCNYu3at9blol0AwnTdvnlZoY8SIIX5+frJ161Z93GKxSKZMmWTEiBF2x3HkyBF93vnz5/U2vp8wYYJUr15dYsWKpVXpoFodpkyZImnSpBFfX1+pVauWjBo1SuLHj//Oec6cOVMr2qgW16tXTwMsNGnSRI9tzJgxum0szlo+okSJIlWqVNFzIyIiIgIGXzeyfft2KVCggPX29evXpWTJkhpqN23aJAcPHpRmzZrJmzdv9HEExJEjR2pwPXbsmFSsWFHD6blz5+y22717d+natascPnxYihYtKtWqVdNeYgRLbG/atGl26+M29otQbNufizB7/PhxfY6jnTt3SuvWraVjx44anMuXL68B2RFC+9KlS2XlypW6IOgOHTrUej44vhYtWmgbBRYEaWcKFy6sr5kzL1++lMePH9stRERE5LkYfN3I5cuXJWXKlNbb48eP16ooqpoFCxaULFmySNOmTSVr1qz6OAJvz549tWqK+4YNG6ZV39GjR9ttt127dvLll19K9uzZtXKLbaKlwqiynjlzRvbt22dtY5gzZ8474fbrr7/WfWfIkEHSpk37zrGPGzdO2w+6deumx9mmTRu97QhV6unTp2vlGdXrhg0bysaNG/UxHBdaKFAxRhsFFlR2ncFrdfXqVad9vkOGDNFtGktwIZqIiIjcH4OvG3n+/Lm2IxhQOUU4jBYt2jvronp548YNKV68uN39uH369Gm7+1BFNUSNGlVDtLEOwqO/v7/8/vvvenvFihVaKa1du7bdNvCc4CA8owJry/E2oMUBLRuGFClSyO3bt+V9xIwZU0MvjjcovXv3lkePHlkXhGQiIiLyXAy+biRx4sTy4MEDu2D3MXzzzTdaVUbwRptD3bp1tepqC729YcExxKPd4n1nZrh//74el7PXCS0i6I22XYiIiMhzMfi6kXz58smpU6est3Pnzq09rGg/cIQQh2otemtt4XaOHDns7tuzZ4/1e/QHo1cYbQ8GDBJDgEQbBAbHBdXDGxK0Wuzfv9/uPsfbrkCrw9u3b11a98SJE/qaEREREQGDrxvB4DTM4mBUfdGbi5YG9PAeOHBAB61hRgS0FRiD1tDXO3/+fL0P89qiPQIDzGyhV3jJkiXy999/S9u2bXX7tuEWfbTo9UVrQObMme1aI1zVvn17nZUCMzngOCdNmqRTjYV2ujO0Quzdu1dnc7h7926w1WC8KahQoUKoj5WIiIg8E4OvG8mVK5fkz59fFixYoLcTJUqkszk8efJESpUqpTM+YMowo12gQ4cO0qVLF52xAc9FtXb58uUaXm1h1gQsefLkkR07dug6aKuwhbmDX716pQPY3gd6iydOnKjBF/vBsXTu3NmuZ9kVGByHII6qdZIkSeTKlSvWQIyZJWxnvNi1a9d7Hy8RERF5Hi8LJmslt7Fq1Sqt5OJjfG/vD3vfgqrpJ598otOYhXSpYFRPy5UrpwPAkiVLJmEB05KhyhzclGOuXtgDbwJQQcaV3QCzWaByPXnyZJe3g+o5ZnfAQDf2+xIREbmH0Pz9jvrRjorCBGZYQKsAKpofY/otzIhw584draZiJocPCb2YXg3z96JfGCF1xowZ8uuvv37wMW7evFnKli1rDb2QNGlSrXYTERERGVjxNTFXKr6YUxdtDngcLRCpUqV67/3VqVNHr+iGK7Fhvl/0/eKiFpEFK75ERETuJzR/vxl8if7D4EtEROR+2OpA9B6M94C8dDEREZH7MP5uu1LLZfAl+s+9e/f0Ky9dTERE5H7QSonKb3AYfIn+kzBhQv2KKdJC+h/HXd8RI9RjZg5PbOXg+bk3np974/m5t8dufn6o9CL04sJdIWHwJfqPMT0cQq87/o/vKk+/PDPPz73x/Nwbz8+9xXXj83O1YMULWBARERGRKTD4EhEREZEpMPgS/cfHx0f69u2rXz0Rz8+98fzcG8/PvfH8PAfn8SUiIiIiU2DFl4iIiIhMgcGXiIiIiEyBwZeIiIiITIHBl4iIiIhMgcGX6D/jx4+X9OnTS4wYMeTTTz+Vffv2iTsaMmSIFCpUSOLEiSNJkyaVmjVrypkzZ+zWefHihbRt21YSJUoksWPHli+//FL++ecfcTdDhw4VLy8v6dSpk8ec2/Xr1+X//u//9PhjxowpuXLlkgMHDlgfx3jkH3/8UVKkSKGPf/7553Lu3DlxB2/fvpU+ffrIJ598oseeMWNGGThwoJ6TO57ftm3bpFq1anq1KPw7XLp0qd3jrpzL/fv3pUGDBnrRgPjx40vz5s3lyZMnEtnP7/Xr19KzZ0/99xkrVixdp1GjRnLjxg2POD9HrVu31nVGjx7tUed3+vRpqV69ul78AT9H/O3A1Us95fdpUBh8iURk/vz50qVLF53O5dChQ5InTx6pWLGi3L59W9zN1q1b9RfVnj17ZP369foHqkKFCvL06VPrOp07d5YVK1bIwoULdX38sfriiy/Enezfv18mTZokuXPntrvfnc/twYMHUrx4cYkWLZqsWbNGTp06JSNHjpQECRJY1xk+fLiMHTtWJk6cKHv37tU/Vvi3ij9Qkd2wYcNkwoQJ8ssvv+gfXNzG+YwbN84tzw//T+F3Bd40B8WVc0FoOnnypP6/unLlSg0rLVu2lMh+fs+ePdPflXgjg6+LFy/WN9gIUbbc9fxsLVmyRH+fBnU5XHc+vwsXLshnn30m2bJlky1btsixY8f054nijyf8PnUK05kRmV3hwoUtbdu2td5++/atJWXKlJYhQ4ZY3N3t27dRTrNs3bpVbz98+NASLVo0y8KFC63rnD59WtfZvXu3xR38+++/lsyZM1vWr19vKVWqlKVjx44ecW49e/a0fPbZZ04fDwwMtCRPntzyv//9z3ofztnHx8cyd+5cS2Tn7+9vadasmd19X3zxhaVBgwZuf374N7ZkyRLrbVfO5dSpU/q8/fv3W9dZs2aNxcvLy3L9+nVLZD6/oOzbt0/Xu3z5ssec37Vr1yypUqWynDhxwpIuXTrLzz//bH3M3c+vbt26lv/7v/9z+hx3/33qDCu+ZHqvXr2SgwcP6seQBm9vb729e/ducXePHj3SrwkTJtSvOFdUgW3PF+/406ZN6zbni4q2v7+/3Tl4wrktX75cChYsKLVr19Y2lXz58smUKVOsj1+6dElu3bpld374iBKtOe5wfsWKFZONGzfK2bNn9fbRo0dlx44dUrlyZY84P1uunAu+4uNx/MwNWB+/f1AhdsffNfhIHefkCecXGBgoDRs2lO7du0vOnDnfedydzy8wMFBWrVolWbJk0U8h8PsG/zZt2yHc/fepMwy+ZHp3797V3sNkyZLZ3Y/b+MPlzvDLDf2v+Pjcz89P78M5RY8e3frHyd3Od968efrRKnqZHbn7uV28eFFbATJnzizr1q2Tb7/9Vjp06CAzZszQx41zcNd/q7169ZJ69erpH0+0cyDY498nPi72hPOz5cq54CsCh62oUaPqm1R3O1+0b6Dnt379+trv6gnnh1YcHC/+HwyKO5/f7du3tRcZ4yQqVaokf/31l9SqVUvbGNDS4Am/T52JGtEHQEThWxk9ceKEVtU8wdWrV6Vjx47aT2fbh+Yp8EYF1aPBgwfrbQRD/PzQI9q4cWNxdwsWLJDZs2fLnDlztIJ25MgRDb7onfSE8zMrVAXr1Kmjg/nwxs0ToNo5ZswYfZONKrYn/q6BGjVqaB8v5M2bV3bt2qW/b0qVKiWeihVfMr3EiRNLlChR3hmpitvJkycXd9WuXTsdbLF582ZJnTq19X6cE9o7Hj586Hbniz9GqFTkz59fKytYUJ3AACJ8j0qEu54bYPR/jhw57O7Lnj27dZS1cQ7u+m8VHxkbVV/MBoCPkfFH16jeu/v52XLlXPDVcQDtmzdvdKYAdzlfI/RevnxZ35Aa1V53P7/t27frseNjfeN3Dc6xa9euOvuPu59f4sSJ9ZxC+n3jzr9PnWHwJdPDRzkFChTQ3kPbd8O4XbRoUXE3qLog9GIk8qZNm3TqKFs4V3zMbHu+GI2NX3aR/XzLlSsnx48f10qhsaBCio/Kje/d9dwALSmOU8+hHzZdunT6PX6W+INje36PHz/WfkJ3OD/MBID+R1t402lUn9z9/Gy5ci74ilCBN3QG/D+L1wP9lu4SejFF24YNG3TKK1vufH54U4ZZDmx/1+CTCbx5QxuSu59f9OjRdeqy4H7fuPPfimBF9Og6oshg3rx5Otp6+vTpOlK3ZcuWlvjx41tu3bplcTfffvutJV68eJYtW7ZYbt68aV2ePXtmXad169aWtGnTWjZt2mQ5cOCApWjRorq4I9tZHdz93DAqPmrUqJZBgwZZzp07Z5k9e7bF19fXMmvWLOs6Q4cO1X+by5Ytsxw7dsxSo0YNyyeffGJ5/vy5JbJr3LixjpBfuXKl5dKlS5bFixdbEidObOnRo4dbnh9mFzl8+LAu+HM6atQo/d6Y1cCVc6lUqZIlX758lr1791p27Nihs5XUr1/fEtnP79WrV5bq1atbUqdObTly5Ijd75qXL1+6/fkFxXFWB3c/v8WLF+usDZMnT9bfN+PGjbNEiRLFsn37do/4feoMgy/Rf/A/Pf4Hjx49uk5vtmfPHos7wi+4oJZp06ZZ18Ef3jZt2lgSJEigwapWrVr6B8sTgq+7n9uKFSssfn5++kYsW7Zs+kfJFqbJ6tOnjyVZsmS6Trly5SxnzpyxuIPHjx/rzwr/n8WIEcOSIUMGy/fff28XlNzp/DZv3hzk/2sI+K6ey7179zQoxY4d2xI3blxL06ZNNbBE9vPDGxdnv2vwPHc/P1eDr7uf39SpUy2ZMmXS/x/z5MljWbp0qd023P33aVC88J+IrjoTEREREYU39vgSERERkSkw+BIRERGRKTD4EhEREZEpMPgSERERkSkw+BIRERGRKTD4EhEREZEpMPgSERERkSkw+BIRERGRKTD4EhEREZEpMPgSERERkSkw+BIRERGRKTD4EhEREZGYwf8DGMW+VMc0BrUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_labels = pd.DataFrame(dataset['train']['label']).value_counts()\n",
    "train_labels.plot(kind='barh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b613afaa-72c0-4ffb-aebf-dd6cbbd867c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2f04798ef514f13a12b99cc2b3ccb1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/319 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2492b426b54844d082a69364d7f6e1e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/319 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65b65c60d5684e0087df5de10b1eab50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/319 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8caa7eb3db294dfdb4fb7ee9d387bed1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/319 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# balance the dataset and take just 4 classes\n",
    "from datasets import concatenate_datasets\n",
    "\n",
    "NUM_TRAINING_SAMPLES = 30\n",
    "NUM_VAL_SAMPLES = 7\n",
    "NUM_TESTING_SAMPLES = 5\n",
    "\n",
    "# criminal-law \n",
    "label_6_train_data = dataset['train'].filter(lambda example: example[\"label\"] == 'criminal-law').select(range(NUM_TRAINING_SAMPLES))\n",
    "label_6_val_data = dataset['validation'].filter(lambda example: example[\"label\"] == 'criminal-law').select(range(NUM_VAL_SAMPLES))\n",
    "label_6_test_data = dataset['test'].filter(lambda example: example[\"label\"] == 'criminal-law').select(range(NUM_TESTING_SAMPLES))\n",
    "# employment (7)\n",
    "label_7_train_data = dataset['train'].filter(lambda example: example[\"label\"] == 'employment').select(range(NUM_TRAINING_SAMPLES))\n",
    "label_7_val_data = dataset['validation'].filter(lambda example: example[\"label\"] == 'employment').select(range(NUM_VAL_SAMPLES))\n",
    "label_7_test_data = dataset['test'].filter(lambda example: example[\"label\"] == 'employment').select(range(NUM_TESTING_SAMPLES))\n",
    "# tax-law (14)\n",
    "label_14_train_data = dataset['train'].filter(lambda example: example[\"label\"] == 'tax-law').select(range(NUM_TRAINING_SAMPLES))\n",
    "label_14_val_data = dataset['validation'].filter(lambda example: example[\"label\"] == 'tax-law').select(range(NUM_VAL_SAMPLES))\n",
    "label_14_test_data = dataset['test'].filter(lambda example: example[\"label\"] == 'tax-law').select(range(NUM_TESTING_SAMPLES))\n",
    "# trademark (15)\n",
    "label_15_train_data = dataset['train'].filter(lambda example: example[\"label\"] == 'trademark').select(range(NUM_TRAINING_SAMPLES))\n",
    "label_15_val_data = dataset['validation'].filter(lambda example: example[\"label\"] == 'trademark').select(range(NUM_VAL_SAMPLES))\n",
    "label_15_test_data = dataset['test'].filter(lambda example: example[\"label\"] == 'trademark').select(range(NUM_TESTING_SAMPLES))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c877736b-2666-4306-ad0f-5880442ed6c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label'],\n",
       "    num_rows: 120\n",
       "})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = concatenate_datasets([label_6_train_data, label_7_train_data, label_14_train_data, label_15_train_data])\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b7f2af0f-34f9-4d37-92a6-5016efe2676b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label'],\n",
       "    num_rows: 28\n",
       "})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_data = concatenate_datasets([label_6_val_data, label_7_val_data, label_14_val_data, label_15_val_data])\n",
    "val_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d98ba6e1-608c-45ee-88b7-2d4f74271a7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label'],\n",
       "    num_rows: 20\n",
       "})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data = concatenate_datasets([label_6_test_data, label_7_test_data, label_14_test_data, label_15_test_data])\n",
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "25519b72-a5aa-48d5-8a1a-dac20e1486cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_LABELS = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "39b7fd28-8a72-4c74-8c91-453bf8917824",
   "metadata": {},
   "outputs": [],
   "source": [
    "# need to properly encode the labels \n",
    "from datasets import Dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a30eee3c-b4ec-41df-9b0b-c3e0077f7513",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': Value(dtype='string', id=None),\n",
       " 'label': Value(dtype='string', id=None)}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "014cad7a-5248-4ee0-9cbc-577a680691c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_labels(dataset):\n",
    "    encoded_data = pd.DataFrame(dataset)\n",
    "    encoded_data = Dataset.from_pandas(encoded_data)\n",
    "    encoded_data = encoded_data.class_encode_column(\"label\")\n",
    "    return encoded_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "52efe5e7-04ea-4b58-8066-b0102824b2f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df99727ce629459089dca950d85add34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Casting to class labels:   0%|          | 0/120 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9ca29779bb74eb096741b2e674d7ad4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Casting to class labels:   0%|          | 0/28 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a6ad4a938ef44b38aa8b78371f3b9b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Casting to class labels:   0%|          | 0/20 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_data = encode_labels(train_data)\n",
    "val_data = encode_labels(val_data)\n",
    "test_data = encode_labels(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ec9672d2-3292-4cc0-aa9d-f183693a77f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': Value(dtype='string', id=None),\n",
       " 'label': ClassLabel(names=['criminal-law', 'employment', 'tax-law', 'trademark'], id=None)}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cb214468-1c32-4112-b288-a7279945a3e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create id2label and label2id\n",
    "labels = train_data.features['label'].names\n",
    "ids = range(NUM_LABELS)\n",
    "id2label = dict(zip(ids, labels))   \n",
    "label2id = dict(zip(labels, ids)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "132542f7-816f-47c2-a9dd-8b81412d6ccf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize\n",
    "from transformers import AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "279bc938-77c1-4e3d-9c87-2a27ff293035",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1e3989a1-52ec-49e7-aa1c-672c1fe4fca8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e4201ea2-b318-4e20-97d1-8c662c96370b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f80dd987f31349128c736587c746aab2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/120 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08aa1ad7eaf94348b5140687e58a0f41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/28 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f058edb85c34ddda25c2c60d27eda05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/20 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_train = train_data.map(tokenize_function, batched=True)\n",
    "tokenized_val = val_data.map(tokenize_function, batched=True)\n",
    "tokenized_test = test_data.map(tokenize_function, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7387a4f9-a0e6-4ce7-9bbc-a08e9846f735",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_train = tokenized_train.remove_columns([\"text\"])\n",
    "tokenized_val = tokenized_val.remove_columns([\"text\"])\n",
    "tokenized_test = tokenized_test.remove_columns([\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b2ab0aca-e487-4c04-91a6-0c2827701380",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'input_ids', 'attention_mask'],\n",
       "    num_rows: 120\n",
       "})"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48ade704-78ed-4aa0-84e5-ce32e3f92d42",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "96cf4c92-27bc-4ae8-a8b1-4f12ed42ce72",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "85621a17-0495-45c4-abaa-4add6fddc481",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=NUM_LABELS, label2id=label2id, id2label=id2label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "14af36c1-1334-4788-bb71-8fa4be930502",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "4f9a5095-0f33-48d3-9446-a50f691bfc34",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(output_dir=\"my_test_model\", eval_strategy=\"epoch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "a6fe87eb-8a0b-45eb-b327-34ce84b1cbcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "33ebc6b3-cd77-4d47-9551-8d4ef2fece1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "metric = evaluate.load(\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b72774f9-0c37-44b5-a3eb-c28a8db1e44b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "d12d4b04-1aec-493f-bdbd-e7e5628e9251",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "9cf083cb-e884-487a-aeb9-2fd1daac57e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_train,\n",
    "    eval_dataset=tokenized_val,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "4f363132-a392-47af-b5a5-37105c87940d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='45' max='45' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [45/45 08:11, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.140705</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.634312</td>\n",
       "      <td>0.928571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.461649</td>\n",
       "      <td>0.892857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=45, training_loss=0.8562598334418403, metrics={'train_runtime': 515.0012, 'train_samples_per_second': 0.699, 'train_steps_per_second': 0.087, 'total_flos': 47689964421120.0, 'train_loss': 0.8562598334418403, 'epoch': 3.0})"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0558a6b9-003c-4246-a409-e3559ec7f74a",
   "metadata": {},
   "source": [
    "## Evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "06ba6711-802f-44d1-a402-3ac9771b619c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.4440459609031677,\n",
       " 'eval_accuracy': 0.95,\n",
       " 'eval_runtime': 6.4454,\n",
       " 'eval_samples_per_second': 3.103,\n",
       " 'eval_steps_per_second': 0.465,\n",
       " 'epoch': 3.0}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate(tokenized_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebc2f0fc-28fb-4e72-adc6-f37b577ad806",
   "metadata": {},
   "source": [
    "## Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "91772fb8-bdb5-4be1-b9db-06949a9e68f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "1065c5b9-a195-42b8-94c5-31913c3d5c83",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialise a new model for each tuning run\n",
    "def model_init():\n",
    "    return AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=NUM_LABELS, label2id=label2id, id2label=id2label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "8062d12b-ed1a-47ed-8301-7892acab1fc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to control the search of hyperparams to tune\n",
    "def hp_space(trial):\n",
    "    return {\n",
    "        \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-5, 5e-4, log=True),\n",
    "        \"per_device_train_batch_size\": trial.suggest_categorical(\"per_device_train_batch_size\", [8, 16, 32]),\n",
    "        \"num_train_epochs\": trial.suggest_categorical(\"num_train_epochs\", [2, 3, 5])\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "e506ff48-cce6-44ad-9e4a-75a75f051c1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the training args for each tuning run\n",
    "tuning_training_args = TrainingArguments(\n",
    "    output_dir=\"model_tuning_output\",\n",
    "    eval_strategy=\"epoch\",\n",
    "    learning_rate=0.1,  # Placeholder; will be overridden by the hyperparameter search\n",
    "    per_device_train_batch_size=16,  # Placeholder; will be overridden by the hyperparameter search\n",
    "    num_train_epochs=3,  # Placeholder; will be overridden by the hyperparameter search\n",
    "    logging_dir=\"./logs\",\n",
    "    #push_to_hub=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "84ff80bb-528e-4cfd-9091-feb9b5368664",
   "metadata": {},
   "outputs": [],
   "source": [
    "#from huggingface_hub import notebook_login\n",
    "#notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "21e79a69-9cb4-4890-8105-ac971c8dcc56",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "tuning_trainer = Trainer(\n",
    "    model_init=model_init,\n",
    "    args=tuning_training_args,\n",
    "    train_dataset=tokenized_train,\n",
    "    eval_dataset=tokenized_val,\n",
    "    processing_class=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "fa0a0c8c-94e4-40cf-8704-940dc42ea0fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-01 16:08:23,814] A new study created in memory with name: no-name-2be3205c-c33e-4b02-8b56-2ae2b7ccd655\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='16' max='16' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [16/16 04:50, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.522721</td>\n",
       "      <td>0.892857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.209599</td>\n",
       "      <td>0.964286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-01 16:13:34,507] Trial 0 finished with value: 0.9642857142857143 and parameters: {'learning_rate': 0.0002728032165901902, 'per_device_train_batch_size': 16, 'num_train_epochs': 2}. Best is trial 0 with value: 0.9642857142857143.\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='30' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [30/30 05:05, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.373767</td>\n",
       "      <td>0.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.431960</td>\n",
       "      <td>0.892857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-01 16:18:50,407] Trial 1 finished with value: 0.8928571428571429 and parameters: {'learning_rate': 0.0004841251983791936, 'per_device_train_batch_size': 8, 'num_train_epochs': 2}. Best is trial 0 with value: 0.9642857142857143.\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [24/24 07:29, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.410274</td>\n",
       "      <td>0.928571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.218974</td>\n",
       "      <td>0.892857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.379487</td>\n",
       "      <td>0.928571</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-01 16:26:39,501] Trial 2 finished with value: 0.9285714285714286 and parameters: {'learning_rate': 0.0003169465104468718, 'per_device_train_batch_size': 16, 'num_train_epochs': 3}. Best is trial 0 with value: 0.9642857142857143.\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='20' max='20' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [20/20 14:08, Epoch 5/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.951409</td>\n",
       "      <td>0.892857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.300698</td>\n",
       "      <td>0.964286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.218479</td>\n",
       "      <td>0.964286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.174622</td>\n",
       "      <td>0.964286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.169968</td>\n",
       "      <td>0.964286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-01 16:41:47,202] Trial 3 finished with value: 0.9642857142857143 and parameters: {'learning_rate': 0.00021073447095197996, 'per_device_train_batch_size': 32, 'num_train_epochs': 5}. Best is trial 0 with value: 0.9642857142857143.\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='8' max='8' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [8/8 05:17, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.348463</td>\n",
       "      <td>0.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.309576</td>\n",
       "      <td>0.821429</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2025-02-01 16:47:52,344] Trial 4 finished with value: 0.8214285714285714 and parameters: {'learning_rate': 3.9687757840299386e-05, 'per_device_train_batch_size': 32, 'num_train_epochs': 2}. Best is trial 0 with value: 0.9642857142857143.\n"
     ]
    }
   ],
   "source": [
    "best_trial = tuning_trainer.hyperparameter_search(\n",
    "    direction=\"maximize\",\n",
    "    backend=\"optuna\",\n",
    "    hp_space=hp_space,\n",
    "    n_trials=5\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "bd6b1d40-9ff2-4a2e-90bb-aff655c21661",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Hyperparameters: {'learning_rate': 0.0002728032165901902, 'per_device_train_batch_size': 16, 'num_train_epochs': 2}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Hyperparameters:\", best_trial.hyperparameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "83d08274-d336-41bb-a0ba-126631b3b464",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('tokenizer\\\\tokenizer_config.json',\n",
       " 'tokenizer\\\\special_tokens_map.json',\n",
       " 'tokenizer\\\\vocab.txt',\n",
       " 'tokenizer\\\\added_tokens.json',\n",
       " 'tokenizer\\\\tokenizer.json')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuning_trainer.model.save_pretrained(\"my_tuned_model\") # not used when pushing model to hub\n",
    "#tuning_trainer.push_to_hub(\"my_tuned_model\") # to be used when push_to_hub = True\n",
    "tuning_trainer.processing_class.save_pretrained(\"tokenizer\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d2bed25-81d1-4326-8002-4a65694c7e8c",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "76878f97-4f2b-428e-99d4-d16230e64f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_model = AutoModelForSequenceClassification.from_pretrained(\"my_tuned_model\")\n",
    "#my_model = AutoModelForSequenceClassification.from_pretrained(\"lnewbould/my_tuned_model\") # to be used when push_to_hub = True\n",
    "my_tokenizer = AutoTokenizer.from_pretrained(\"tokenizer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "5180fe54-0f5c-4538-8ce7-1d6cb78d8ea3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "c5de2fa5-1446-440f-a443-399e18fe04da",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Device set to use cpu\n"
     ]
    }
   ],
   "source": [
    "my_text_classifier = pipeline(\"text-classification\", model=my_model, tokenizer=my_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "cbac1a91-5dfc-44dd-a0bc-7f0c66a7220e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random \n",
    "random_sample = random.randint(0, 20)\n",
    "sample = test_data[random_sample]\n",
    "sample_text = sample['text']\n",
    "sample_label = sample['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "4284ce7d-3947-47e6-a944-8efe04c67f73",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_model_result = my_text_classifier(sample_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "da86ccb3-4b67-4284-aff3-da13e6abb232",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text to classify:\n",
      " I was recently watching a video of a police officer who lied to a driver that \"a new law passed\" that allows police officers to stop someone from recording. The driver was an attorney and knew better.\n",
      "\n",
      "\n",
      "I was curious about this situation where a law changes that you may not be aware of because it only very recently passes. It seems wrong that someone who may have been not aware it was recently changed would be punished. I imagine it depends on the severity of the law, but is there some sort of grace period due to information not travelling fast enough?\n",
      "\n",
      "\n",
      "While I think the above scenario is a little extreme, any advice is appreciated.\n",
      "\n",
      "\n",
      " \n",
      "\n",
      "The correct label from the dataset: criminal-law \n",
      "Our model classification: criminal-law with a score of 0.2641546428203583\n"
     ]
    }
   ],
   "source": [
    "print(\"Text to classify:\\n\", sample_text, \n",
    "      \"\\n\\nThe correct label from the dataset:\", sample_label, \n",
    "      \"\\nOur model classification:\", my_model_result[0]['label'], \"with a score of\", my_model_result[0]['score'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d4cd395-5a77-4e18-888d-ced1b885e8ac",
   "metadata": {},
   "source": [
    "## Pushing model to hub"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ceab92c6-7b3e-4030-b6af-4259695e8ca2",
   "metadata": {},
   "source": [
    "Will need to move this section up in finished file so hyperparam tuning can be uploaded to hub. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "6cd9045d-39e1-405a-ae9a-2f05d271a565",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "117043b0eda241d2981cf25fe96d9d35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "14891e0b-48ef-4202-8c98-7e7507715582",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85ff16fabfb040a29cd9d33e3998032b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/263M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/lnewbould/my_final_model/commit/2bd5ea82b139fd02721ac1dae10c05fc49c59d99', commit_message='Upload DistilBertForSequenceClassification', commit_description='', oid='2bd5ea82b139fd02721ac1dae10c05fc49c59d99', pr_url=None, repo_url=RepoUrl('https://huggingface.co/lnewbould/my_final_model', endpoint='https://huggingface.co', repo_type='model', repo_id='lnewbould/my_final_model'), pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_model.push_to_hub(\"my_final_model\")\n",
    "my_tokenizer.push_to_hub(\"my_final_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6cbfdb8d-e8d4-40cb-8548-307c84a4f415",
   "metadata": {},
   "source": [
    "## Gradio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "579eb07e-9a7f-4bff-9de6-acf3a7f4bf02",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gradio as gr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "c7d5b5bd-68c2-411b-90bf-c5996ea0e0fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Running on local URL:  http://127.0.0.1:7863\n",
      "* Running on public URL: https://7ce527515988a9431e.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from the terminal in the working directory to deploy to Hugging Face Spaces (https://huggingface.co/spaces)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"https://7ce527515988a9431e.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "demo = gr.Interface.from_pipeline(my_text_classifier)\n",
    "demo.launch(share=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huggingface_course_env",
   "language": "python",
   "name": "huggingface_course_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}