tokenizers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import T5Tokenizer, BertTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/models/t5/tokenization_t5.py:217: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
      "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
      "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
      "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
      "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "text=\"This is an introduction to: tokenizers\"\n",
    "t5_tokenizer=T5Tokenizer.from_pretrained(\"t5-base\")\n",
    "bert_tokenizer=BertTokenizer.from_pretrained(\"bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "t5_tokens=t5_tokenizer.tokenize(text)\n",
    "bert_tokens=bert_tokenizer.tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tT5\n",
      " --------------\n",
      "\n",
      "['▁This', '▁is', '▁an', '▁introduction', '▁to', ':', '▁token', 'izer', 's']\n",
      "\n",
      "\n",
      "\tBert\n",
      " --------------\n",
      "\n",
      "['this', 'is', 'an', 'introduction', 'to', ':', 'token', '##izer', '##s']\n"
     ]
    }
   ],
   "source": [
    "print(\"\\tT5\\n --------------\\n\")\n",
    "print(t5_tokens)\n",
    "print(\"\\n\")\n",
    "print(\"\\tBert\\n --------------\\n\")\n",
    "print(bert_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "t5_ids=t5_tokenizer.convert_tokens_to_ids(t5_tokens)\n",
    "bert_ids=bert_tokenizer.convert_tokens_to_ids(bert_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tT5\n",
      " --------------\n",
      "\n",
      "[100, 19, 46, 5302, 12, 10, 14145, 8585, 7]\n",
      "\n",
      "\n",
      "\tBert\n",
      " --------------\n",
      "\n",
      "[2023, 2003, 2019, 4955, 2000, 1024, 19204, 17629, 2015]\n"
     ]
    }
   ],
   "source": [
    "print(\"\\tT5\\n --------------\\n\")\n",
    "print(t5_ids)\n",
    "print(\"\\n\")\n",
    "print(\"\\tBert\\n --------------\\n\")\n",
    "print(bert_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['this', 'is', 'an', 'introduction', 'to', ':', 'token', '##izer', '##s']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['▁This', '▁is', '▁an', '▁introduction', '▁to', ':', '▁token', 'izer', 's']"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(bert_tokenizer.convert_ids_to_tokens(bert_ids))\n",
    "t5_tokenizer.convert_ids_to_tokens(t5_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'this is an introduction to : tokenizers'"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t5_tokenizer.decode(t5_ids)\n",
    "bert_tokenizer.decode(bert_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[100, 19, 46, 5302, 12, 10, 14145, 8585, 7, 1]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t5_tokenizer.encode(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is an introduction to: tokenizers</s>'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t5_tokenizer.decode(t5_tokenizer.encode(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    }
   ],
   "source": [
    "source = t5_tokenizer.batch_encode_plus([text], max_length= 16, pad_to_max_length=False,return_tensors='pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[  100,    19,    46,  5302,    12,    10, 14145,  8585,     7,     1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "# Importing the T5 modules from huggingface/transformers\n",
    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
    "\n",
    "# WandB – Import the wandb library\n",
    "#import wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "\n",
    "    def __init__(self, dataframe, tokenizer, source_len, summ_len):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.data = dataframe\n",
    "        self.source_len = source_len\n",
    "        self.summ_len = summ_len\n",
    "        self.text = self.data.text\n",
    "        self.ctext = self.data.ctext\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.text)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        ctext = str(self.ctext[index])\n",
    "        ctext = ' '.join(ctext.split())\n",
    "\n",
    "        text = str(self.text[index])\n",
    "        text = ' '.join(text.split())\n",
    "\n",
    "        source = self.tokenizer.batch_encode_plus([ctext], max_length= self.source_len, pad_to_max_length=True,return_tensors='pt')\n",
    "        target = self.tokenizer.batch_encode_plus([text], max_length= self.summ_len, pad_to_max_length=True,return_tensors='pt')\n",
    "\n",
    "        source_ids = source['input_ids'].squeeze()\n",
    "        source_mask = source['attention_mask'].squeeze()\n",
    "        target_ids = target['input_ids'].squeeze()\n",
    "        target_mask = target['attention_mask'].squeeze()\n",
    "\n",
    "        return {\n",
    "            'source_ids': source_ids.to(dtype=torch.long), \n",
    "            'source_mask': source_mask.to(dtype=torch.long), \n",
    "            'target_ids': target_ids.to(dtype=torch.long),\n",
    "            'target_ids_y': target_ids.to(dtype=torch.long)\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text  \\\n",
      "0  The Administration of Union Territory Daman an...   \n",
      "1  Malaika Arora slammed an Instagram user who tr...   \n",
      "2  The Indira Gandhi Institute of Medical Science...   \n",
      "3  Lashkar-e-Taiba's Kashmir commander Abu Dujana...   \n",
      "4  Hotels in Maharashtra will train their staff t...   \n",
      "\n",
      "                                               ctext  \n",
      "0  summarize: The Daman and Diu administration on...  \n",
      "1  summarize: From her special numbers to TV?appe...  \n",
      "2  summarize: The Indira Gandhi Institute of Medi...  \n",
      "3  summarize: Lashkar-e-Taiba's Kashmir commander...  \n",
      "4  summarize: Hotels in Mumbai and other Indian c...  \n",
      "FULL Dataset: (4514, 2)\n",
      "TRAIN Dataset: (3611, 2)\n",
      "TEST Dataset: (903, 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/models/t5/tokenization_t5.py:217: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
      "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
      "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
      "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
      "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "TRAIN_BATCH_SIZE = 2    # input batch size for training (default: 64)\n",
    "VALID_BATCH_SIZE = 2    # input batch size for testing (default: 1000)\n",
    "TRAIN_EPOCHS = 2        # number of epochs to train (default: 10)\n",
    "VAL_EPOCHS = 1 \n",
    "LEARNING_RATE = 1e-4    # learning rate (default: 0.01)\n",
    "SEED = 42               # random seed (default: 42)\n",
    "MAX_LEN = 512\n",
    "SUMMARY_LEN = 150 \n",
    "# Set random seeds and deterministic pytorch for reproducibility\n",
    "torch.manual_seed(SEED) # pytorch random seed\n",
    "np.random.seed(SEED) # numpy random seed\n",
    "torch.backends.cudnn.deterministic = True\n",
    "# tokenzier for encoding the text\n",
    "tokenizer = T5Tokenizer.from_pretrained(\"t5-base\")\n",
    "\n",
    "# Importing and Pre-Processing the domain data\n",
    "# Selecting the needed columns only. \n",
    "# Adding the summarzie text in front of the text. This is to format the dataset similar to how T5 model was trained for summarization task. \n",
    "df = pd.read_csv('~/Downloads/news_summary.csv',encoding='latin-1')\n",
    "df = df[['text','ctext']]\n",
    "df.ctext = 'summarize: ' + df.ctext\n",
    "print(df.head())\n",
    "\n",
    "\n",
    "# Creation of Dataset and Dataloader\n",
    "# Defining the train size. So 80% of the data will be used for training and the rest will be used for validation. \n",
    "train_size = 0.8\n",
    "train_dataset=df.sample(frac=train_size,random_state = SEED)\n",
    "val_dataset=df.drop(train_dataset.index).reset_index(drop=True)\n",
    "train_dataset = train_dataset.reset_index(drop=True)\n",
    "\n",
    "print(\"FULL Dataset: {}\".format(df.shape))\n",
    "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "print(\"TEST Dataset: {}\".format(val_dataset.shape))\n",
    "\n",
    "\n",
    "  # Creating the Training and Validation dataset for further creation of Dataloader\n",
    "training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN, SUMMARY_LEN)\n",
    "val_set = CustomDataset(val_dataset, tokenizer, MAX_LEN, SUMMARY_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_params = {\n",
    "        'batch_size': TRAIN_BATCH_SIZE,\n",
    "        'shuffle': True,\n",
    "        'num_workers': 0\n",
    "        }\n",
    "\n",
    "val_params = {\n",
    "        'batch_size': VALID_BATCH_SIZE,\n",
    "        'shuffle': False,\n",
    "        'num_workers': 0\n",
    "        }\n",
    "\n",
    "    # Creation of Dataloaders for testing and validation. This will be used down for training and validation stage for the model.\n",
    "training_loader = DataLoader(training_set, **train_params)\n",
    "val_loader = DataLoader(val_set, **val_params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "itr=iter(training_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/tokenization_utils_base.py:2418: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "x=next(itr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=x['target_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_ids = y[:, :-1].contiguous()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 149])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 150])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Tensor"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}