lab-11_5_seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# main reference\n",
    "# https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(0)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw = [\"I feel hungry.\t나는 배가 고프다.\",\n",
    "       \"Pytorch is very easy.\t파이토치는 매우 쉽다.\",\n",
    "       \"Pytorch is a framework for deep learning.\t파이토치는 딥러닝을 위한 프레임워크이다.\",\n",
    "       \"Pytorch is very clear to use.\t파이토치는 사용하기 매우 직관적이다.\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fix token for \"start of sentence\" and \"end of sentence\"\n",
    "SOS_token = 0\n",
    "EOS_token = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class for vocabulary related information of data\n",
    "class Vocab:\n",
    "    def __init__(self):\n",
    "        self.vocab2index = {\"<SOS>\": SOS_token, \"<EOS>\": EOS_token}\n",
    "        self.index2vocab = {SOS_token: \"<SOS>\", EOS_token: \"<EOS>\"}\n",
    "        self.vocab_count = {}\n",
    "        self.n_vocab = len(self.vocab2index)\n",
    "\n",
    "    def add_vocab(self, sentence):\n",
    "        for word in sentence.split(\" \"):\n",
    "            if word not in self.vocab2index:\n",
    "                self.vocab2index[word] = self.n_vocab\n",
    "                self.vocab_count[word] = 1\n",
    "                self.index2vocab[self.n_vocab] = word\n",
    "                self.n_vocab += 1\n",
    "            else:\n",
    "                self.vocab_count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter out the long sentence from source and target data\n",
    "def filter_pair(pair, source_max_length, target_max_length):\n",
    "    return len(pair[0].split(\" \")) < source_max_length and len(pair[1].split(\" \")) < target_max_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read and preprocess the corpus data\n",
    "def preprocess(corpus, source_max_length, target_max_length):\n",
    "    print(\"reading corpus...\")\n",
    "    pairs = []\n",
    "    for line in corpus:\n",
    "        pairs.append([s for s in line.strip().lower().split(\"\\t\")])\n",
    "    print(\"Read {} sentence pairs\".format(len(pairs)))\n",
    "\n",
    "    pairs = [pair for pair in pairs if filter_pair(pair, source_max_length, target_max_length)]\n",
    "    print(\"Trimmed to {} sentence pairs\".format(len(pairs)))\n",
    "\n",
    "    source_vocab = Vocab()\n",
    "    target_vocab = Vocab()\n",
    "\n",
    "    print(\"Counting words...\")\n",
    "    for pair in pairs:\n",
    "        source_vocab.add_vocab(pair[0])\n",
    "        target_vocab.add_vocab(pair[1])\n",
    "    print(\"source vocab size =\", source_vocab.n_vocab)\n",
    "    print(\"target vocab size =\", target_vocab.n_vocab)\n",
    "\n",
    "    return pairs, source_vocab, target_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# declare simple encoder\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "\n",
    "    def forward(self, x, hidden):\n",
    "        x = self.embedding(x).view(1, 1, -1)\n",
    "        x, hidden = self.gru(x, hidden)\n",
    "        return x, hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# declare simple decoder\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self, hidden_size, output_size):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(output_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "\n",
    "    def forward(self, x, hidden):\n",
    "        x = self.embedding(x).view(1, 1, -1)\n",
    "        x, hidden = self.gru(x, hidden)\n",
    "        x = self.softmax(self.out(x[0]))\n",
    "        return x, hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert sentence to the index tensor\n",
    "def tensorize(vocab, sentence):\n",
    "    indexes = [vocab.vocab2index[word] for word in sentence.split(\" \")]\n",
    "    indexes.append(vocab.vocab2index[\"<EOS>\"])\n",
    "    return torch.Tensor(indexes).long().to(device).view(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training seq2seq\n",
    "def train(pairs, source_vocab, target_vocab, encoder, decoder, n_iter, print_every=1000, learning_rate=0.01):\n",
    "    loss_total = 0\n",
    "\n",
    "    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
    "    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
    "\n",
    "    training_batch = [random.choice(pairs) for _ in range(n_iter)]\n",
    "    training_source = [tensorize(source_vocab, pair[0]) for pair in training_batch]\n",
    "    training_target = [tensorize(target_vocab, pair[1]) for pair in training_batch]\n",
    "\n",
    "    criterion = nn.NLLLoss()\n",
    "\n",
    "    for i in range(1, n_iter + 1):\n",
    "        source_tensor = training_source[i - 1]\n",
    "        target_tensor = training_target[i - 1]\n",
    "\n",
    "        encoder_hidden = torch.zeros([1, 1, encoder.hidden_size]).to(device)\n",
    "\n",
    "        encoder_optimizer.zero_grad()\n",
    "        decoder_optimizer.zero_grad()\n",
    "\n",
    "        source_length = source_tensor.size(0)\n",
    "        target_length = target_tensor.size(0)\n",
    "\n",
    "        loss = 0\n",
    "\n",
    "        for enc_input in range(source_length):\n",
    "            _, encoder_hidden = encoder(source_tensor[enc_input], encoder_hidden)\n",
    "\n",
    "        decoder_input = torch.Tensor([[SOS_token]]).long().to(device)\n",
    "        decoder_hidden = encoder_hidden # connect encoder output to decoder input\n",
    "\n",
    "        for di in range(target_length):\n",
    "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n",
    "            loss += criterion(decoder_output, target_tensor[di])\n",
    "            decoder_input = target_tensor[di]  # teacher forcing\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        encoder_optimizer.step()\n",
    "        decoder_optimizer.step()\n",
    "\n",
    "        loss_iter = loss.item() / target_length\n",
    "        loss_total += loss_iter\n",
    "\n",
    "        if i % print_every == 0:\n",
    "            loss_avg = loss_total / print_every\n",
    "            loss_total = 0\n",
    "            print(\"[{} - {}%] loss = {:05.4f}\".format(i, i / n_iter * 100, loss_avg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# insert given sentence to check the training\n",
    "def evaluate(pairs, source_vocab, target_vocab, encoder, decoder, target_max_length):\n",
    "    for pair in pairs:\n",
    "        print(\">\", pair[0])\n",
    "        print(\"=\", pair[1])\n",
    "        source_tensor = tensorize(source_vocab, pair[0])\n",
    "        source_length = source_tensor.size()[0]\n",
    "        encoder_hidden = torch.zeros([1, 1, encoder.hidden_size]).to(device)\n",
    "\n",
    "        for ei in range(source_length):\n",
    "            _, encoder_hidden = encoder(source_tensor[ei], encoder_hidden)\n",
    "\n",
    "        decoder_input = torch.Tensor([[SOS_token]], device=device).long()\n",
    "        decoder_hidden = encoder_hidden\n",
    "        decoded_words = []\n",
    "\n",
    "        for di in range(target_max_length):\n",
    "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n",
    "            _, top_index = decoder_output.data.topk(1)\n",
    "            if top_index.item() == EOS_token:\n",
    "                decoded_words.append(\"<EOS>\")\n",
    "                break\n",
    "            else:\n",
    "                decoded_words.append(target_vocab.index2vocab[top_index.item()])\n",
    "\n",
    "            decoder_input = top_index.squeeze().detach()\n",
    "\n",
    "        predict_words = decoded_words\n",
    "        predict_sentence = \" \".join(predict_words)\n",
    "        print(\"<\", predict_sentence)\n",
    "        print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# declare max length for sentence\n",
    "SOURCE_MAX_LENGTH = 10\n",
    "TARGET_MAX_LENGTH = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading corpus...\n",
      "Read 4 sentence pairs\n",
      "Trimmed to 4 sentence pairs\n",
      "Counting words...\n",
      "source vocab size = 17\n",
      "target vocab size = 13\n",
      "['pytorch is a framework for deep learning.', '파이토치는 딥러닝을 위한 프레임워크이다.']\n"
     ]
    }
   ],
   "source": [
    "# preprocess the corpus\n",
    "load_pairs, load_source_vocab, load_target_vocab = preprocess(raw, SOURCE_MAX_LENGTH, TARGET_MAX_LENGTH)\n",
    "print(random.choice(load_pairs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# declare the encoder and the decoder\n",
    "enc_hidden_size = 16\n",
    "dec_hidden_size = enc_hidden_size\n",
    "enc = Encoder(load_source_vocab.n_vocab, enc_hidden_size).to(device)\n",
    "dec = Decoder(dec_hidden_size, load_target_vocab.n_vocab).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1000 - 20.0%] loss = 0.7401\n",
      "[2000 - 40.0%] loss = 0.1098\n",
      "[3000 - 60.0%] loss = 0.0344\n",
      "[4000 - 80.0%] loss = 0.0182\n",
      "[5000 - 100.0%] loss = 0.0124\n"
     ]
    }
   ],
   "source": [
    "# train seq2seq model\n",
    "train(load_pairs, load_source_vocab, load_target_vocab, enc, dec, 5000, print_every=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> i feel hungry.\n",
      "= 나는 배가 고프다.\n",
      "< 나는 배가 고프다. <EOS>\n",
      "\n",
      "> pytorch is very easy.\n",
      "= 파이토치는 매우 쉽다.\n",
      "< 파이토치는 매우 쉽다. <EOS>\n",
      "\n",
      "> pytorch is a framework for deep learning.\n",
      "= 파이토치는 딥러닝을 위한 프레임워크이다.\n",
      "< 파이토치는 딥러닝을 위한 프레임워크이다. <EOS>\n",
      "\n",
      "> pytorch is very clear to use.\n",
      "= 파이토치는 사용하기 매우 직관적이다.\n",
      "< 파이토치는 사용하기 매우 직관적이다. <EOS>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# check the model with given data\n",
    "evaluate(load_pairs, load_source_vocab, load_target_vocab, enc, dec, TARGET_MAX_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}