training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "training",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "70CYH2bUD-wx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cd /content/drive/My Drive/ITSP-2020"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pf-u73P1EBsY",
        "colab_type": "code",
        "outputId": "573f51dd-d062-406b-c67b-97521283ed60",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "source": [
        "pip show tensorflow"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Name: tensorflow\n",
            "Version: 1.10.0\n",
            "Summary: TensorFlow is an open source machine learning framework for everyone.\n",
            "Home-page: https://www.tensorflow.org/\n",
            "Author: Google Inc.\n",
            "Author-email: opensource@google.com\n",
            "License: Apache 2.0\n",
            "Location: /usr/local/lib/python3.6/dist-packages\n",
            "Requires: astor, numpy, tensorboard, absl-py, wheel, gast, termcolor, setuptools, grpcio, protobuf, six\n",
            "Required-by: fancyimpute\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xFGRQWr6jB10",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import random\n",
        "import json\n",
        "import os\n",
        "import time\n",
        "\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import tensorflow.contrib.legacy_seq2seq as seq2seq\n",
        "\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Jel8LfgijB2M",
        "colab": {}
      },
      "source": [
        "xo = np.load(\"iseq_n.npy\")\n",
        "yo = np.load(\"oseq_n.npy\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "czBqZDeCRook",
        "colab_type": "code",
        "outputId": "7e19472d-5091-4e50-bea1-01678cff5dfe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "xo.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(16879, 35, 35)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ALRBRrGIJOKa",
        "colab": {}
      },
      "source": [
        "x = xo[:,:26,:]\n",
        "y = yo[:,:26]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fQBQQL3ol68m",
        "outputId": "5ae90867-c064-4af1-fc6b-d1ba16bad10c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "nxchars = x.shape[2]\n",
        "print(nxchars)\n",
        "ltokens = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '+', '=', '#leq', '#neq', '#geq', '#alpha',\n",
        "                            '#beta', '#lambda', '#lt', '#gt', 'x', 'y', '^', '#frac', '{', '}' ,' ','#int','d','#sinx','#cosx','#tanx']\n",
        "print(\"#ltokens: \", len(ltokens))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "35\n",
            "#ltokens:  33\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fmU4uwY7jB2S",
        "outputId": "3af0de8c-bbae-42bf-c6ea-e85b9176b21a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "x_seq_length = len(x[0])\n",
        "y_seq_length = len(y[0])- 1\n",
        "print(x_seq_length, y_seq_length)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "26 25\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "DC02d5lwjB2X",
        "colab": {}
      },
      "source": [
        "def batch_data(x, y, batch_size):\n",
        "    shuffle = np.random.permutation(len(x))\n",
        "    start = 0\n",
        "#     from IPython.core.debugger import Tracer; Tracer()()\n",
        "    x = x[shuffle]\n",
        "    y = y[shuffle]\n",
        "    while start + batch_size <= len(x):\n",
        "        yield x[start:start+batch_size], y[start:start+batch_size]\n",
        "        start += batch_size"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Zk9tm_s1b6S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.reset_default_graph()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "T1Hfp3ojjB2b",
        "colab": {}
      },
      "source": [
        "batch_size = 512\n",
        "nodes = 256\n",
        "embed_size = 20\n",
        "\n",
        "# Tensor where we will feed the data into graph\n",
        "inputs = tf.placeholder(tf.float32, (None, x_seq_length, nxchars), 'inputs')\n",
        "outputs = tf.placeholder(tf.int32, (None, None), 'output')\n",
        "targets = tf.placeholder(tf.int32, (None, None), 'targets')\n",
        "\n",
        "# Embedding layers\n",
        "output_embedding = tf.Variable(tf.random_uniform((len(ltokens)+1, embed_size), -1.0, 1.0), name='dec_embedding')\n",
        "date_output_embed = tf.nn.embedding_lookup(output_embedding, outputs)\n",
        "\n",
        "with tf.variable_scope(\"encoding\") as encoding_scope:\n",
        "    lstm_enc = tf.contrib.rnn.BasicLSTMCell(nodes)\n",
        "    _, last_state = tf.nn.dynamic_rnn(lstm_enc, inputs=inputs, dtype=tf.float32)\n",
        "\n",
        "with tf.variable_scope(\"decoding\") as decoding_scope:\n",
        "    lstm_dec = tf.contrib.rnn.BasicLSTMCell(nodes)\n",
        "    dec_outputs, _ = tf.nn.dynamic_rnn(lstm_dec, inputs=date_output_embed, initial_state=last_state)\n",
        "#connect outputs to \n",
        "logits = tf.contrib.layers.fully_connected(dec_outputs, num_outputs=len(ltokens)+1, activation_fn=None) \n",
        "with tf.name_scope(\"optimization\"):\n",
        "    # Loss function\n",
        "    loss = tf.contrib.seq2seq.sequence_loss(logits, targets, tf.ones([batch_size, y_seq_length]))\n",
        "    # Optimizer\n",
        "    optimizer = tf.train.RMSPropOptimizer(1e-3).minimize(loss)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "43-L1cRKjB2u"
      },
      "source": [
        "Train the graph above:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qTnfrvy3jB2x",
        "outputId": "d44c3a18-edb9-44c2-f3a0-1812a6008c49",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.33, random_state=42)\n",
        "print(y_train[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[33 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32\n",
            " 32 32]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "1IaI_AYpzv2B",
        "colab": {}
      },
      "source": [
        "def save(sess):\n",
        "    saver = tf.train.Saver(None)\n",
        "\n",
        "    save_path = saver.save(sess, save_path=\"seq_mod/model\", global_step=None)\n",
        "    print('model saved at %s' % save_path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z3lVFTqUoed8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def restore(sess):\n",
        "    saver = tf.train.Saver(None)\n",
        "    path = \"seq_mod/model\"\n",
        "    saver.restore(sess, save_path=path)\n",
        "    print('model restored from %s' % path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gz89pWSJjB20",
        "outputId": "5baf1bf2-af0d-4fc8-c18d-000180113a45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "with tf.Session() as sess: \n",
        "    sess.run(tf.global_variables_initializer())\n",
        "    restore(sess)   ####remove this line if training for the first time !!!\n",
        "    epochs = 200\n",
        "    for epoch_i in range(epochs):\n",
        "        start_time = time.time()\n",
        "        for batch_i, (source_batch, target_batch) in enumerate(batch_data(X_train, y_train, batch_size)):\n",
        "            _, batch_loss, batch_logits = sess.run([optimizer, loss, logits],\n",
        "                feed_dict = {inputs: source_batch,\n",
        "                 outputs: target_batch[:, :-1],\n",
        "                 targets: target_batch[:, 1:]})\n",
        "            \n",
        "\n",
        "        accuracy = np.mean(batch_logits.argmax(axis=-1) == target_batch[:,1:])\n",
        "        print('Epoch {:3} Loss: {:>6.3f} Accuracy: {:>6.4f} Epoch duration: {:>6.3f}s'.format(epoch_i, batch_loss, \n",
        "                                                                          accuracy, time.time() - start_time))\n",
        "\n",
        "        source_batch, target_batch = next(batch_data(X_test, y_test, batch_size))\n",
        "\n",
        "        dec_input = np.zeros((len(source_batch), 1)) + len(ltokens)\n",
        "        for i in range(y_seq_length):\n",
        "            batch_logits = sess.run(logits,\n",
        "                        feed_dict = {inputs: source_batch,\n",
        "                         outputs: dec_input})\n",
        "            prediction = batch_logits[:,-1].argmax(axis=-1)\n",
        "            dec_input = np.hstack([dec_input, prediction[:,None]])\n",
        "\n",
        "        print('Accuracy on test set is: {:>6.3f}'.format(np.mean(dec_input == target_batch)))     \n",
        "        if epoch_i % 5 == 0:\n",
        "            save(sess)\n",
        "    \n",
        "    save(sess)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from seq_mod/model\n",
            "model restored from seq_mod/model\n",
            "Epoch   0 Loss:  0.223 Accuracy: 0.9287 Epoch duration: 45.191s\n",
            "Accuracy on test set is:  0.698\n",
            "model saved at seq_mod/model\n",
            "Epoch   1 Loss:  0.229 Accuracy: 0.9263 Epoch duration: 51.391s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch   2 Loss:  0.239 Accuracy: 0.9234 Epoch duration: 44.950s\n",
            "Accuracy on test set is:  0.681\n",
            "Epoch   3 Loss:  0.252 Accuracy: 0.9176 Epoch duration: 45.049s\n",
            "Accuracy on test set is:  0.691\n",
            "Epoch   4 Loss:  0.238 Accuracy: 0.9218 Epoch duration: 45.093s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch   5 Loss:  0.215 Accuracy: 0.9314 Epoch duration: 45.089s\n",
            "Accuracy on test set is:  0.681\n",
            "model saved at seq_mod/model\n",
            "Epoch   6 Loss:  0.230 Accuracy: 0.9287 Epoch duration: 45.275s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch   7 Loss:  0.226 Accuracy: 0.9251 Epoch duration: 45.223s\n",
            "Accuracy on test set is:  0.681\n",
            "Epoch   8 Loss:  0.227 Accuracy: 0.9266 Epoch duration: 45.256s\n",
            "Accuracy on test set is:  0.683\n",
            "Epoch   9 Loss:  0.208 Accuracy: 0.9370 Epoch duration: 45.139s\n",
            "Accuracy on test set is:  0.670\n",
            "Epoch  10 Loss:  0.212 Accuracy: 0.9323 Epoch duration: 45.159s\n",
            "Accuracy on test set is:  0.689\n",
            "model saved at seq_mod/model\n",
            "Epoch  11 Loss:  0.222 Accuracy: 0.9273 Epoch duration: 48.171s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch  12 Loss:  0.202 Accuracy: 0.9402 Epoch duration: 44.968s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch  13 Loss:  0.226 Accuracy: 0.9254 Epoch duration: 44.933s\n",
            "Accuracy on test set is:  0.668\n",
            "Epoch  14 Loss:  0.213 Accuracy: 0.9298 Epoch duration: 45.059s\n",
            "Accuracy on test set is:  0.681\n",
            "Epoch  15 Loss:  0.208 Accuracy: 0.9358 Epoch duration: 44.977s\n",
            "Accuracy on test set is:  0.686\n",
            "model saved at seq_mod/model\n",
            "Epoch  16 Loss:  0.183 Accuracy: 0.9450 Epoch duration: 44.954s\n",
            "Accuracy on test set is:  0.702\n",
            "Epoch  17 Loss:  0.197 Accuracy: 0.9368 Epoch duration: 45.140s\n",
            "Accuracy on test set is:  0.672\n",
            "Epoch  18 Loss:  0.171 Accuracy: 0.9521 Epoch duration: 44.934s\n",
            "Accuracy on test set is:  0.692\n",
            "Epoch  19 Loss:  0.207 Accuracy: 0.9330 Epoch duration: 45.151s\n",
            "Accuracy on test set is:  0.669\n",
            "Epoch  20 Loss:  0.193 Accuracy: 0.9390 Epoch duration: 44.886s\n",
            "Accuracy on test set is:  0.688\n",
            "model saved at seq_mod/model\n",
            "Epoch  21 Loss:  0.194 Accuracy: 0.9388 Epoch duration: 44.867s\n",
            "Accuracy on test set is:  0.681\n",
            "Epoch  22 Loss:  0.195 Accuracy: 0.9358 Epoch duration: 46.574s\n",
            "Accuracy on test set is:  0.692\n",
            "Epoch  23 Loss:  0.202 Accuracy: 0.9348 Epoch duration: 44.769s\n",
            "Accuracy on test set is:  0.672\n",
            "Epoch  24 Loss:  0.169 Accuracy: 0.9503 Epoch duration: 44.836s\n",
            "Accuracy on test set is:  0.682\n",
            "Epoch  25 Loss:  0.165 Accuracy: 0.9553 Epoch duration: 44.795s\n",
            "Accuracy on test set is:  0.685\n",
            "model saved at seq_mod/model\n",
            "Epoch  26 Loss:  0.173 Accuracy: 0.9486 Epoch duration: 45.082s\n",
            "Accuracy on test set is:  0.690\n",
            "Epoch  27 Loss:  0.175 Accuracy: 0.9487 Epoch duration: 44.801s\n",
            "Accuracy on test set is:  0.666\n",
            "Epoch  28 Loss:  0.164 Accuracy: 0.9523 Epoch duration: 44.748s\n",
            "Accuracy on test set is:  0.681\n",
            "Epoch  29 Loss:  0.174 Accuracy: 0.9489 Epoch duration: 44.824s\n",
            "Accuracy on test set is:  0.694\n",
            "Epoch  30 Loss:  0.178 Accuracy: 0.9473 Epoch duration: 44.844s\n",
            "Accuracy on test set is:  0.697\n",
            "model saved at seq_mod/model\n",
            "Epoch  31 Loss:  0.167 Accuracy: 0.9501 Epoch duration: 44.787s\n",
            "Accuracy on test set is:  0.659\n",
            "Epoch  32 Loss:  0.162 Accuracy: 0.9523 Epoch duration: 48.366s\n",
            "Accuracy on test set is:  0.689\n",
            "Epoch  33 Loss:  0.159 Accuracy: 0.9550 Epoch duration: 44.724s\n",
            "Accuracy on test set is:  0.686\n",
            "Epoch  34 Loss:  0.171 Accuracy: 0.9470 Epoch duration: 44.845s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch  35 Loss:  0.177 Accuracy: 0.9430 Epoch duration: 44.725s\n",
            "Accuracy on test set is:  0.687\n",
            "model saved at seq_mod/model\n",
            "Epoch  36 Loss:  0.170 Accuracy: 0.9484 Epoch duration: 44.758s\n",
            "Accuracy on test set is:  0.688\n",
            "Epoch  37 Loss:  0.163 Accuracy: 0.9510 Epoch duration: 44.586s\n",
            "Accuracy on test set is:  0.691\n",
            "Epoch  38 Loss:  0.192 Accuracy: 0.9376 Epoch duration: 44.647s\n",
            "Accuracy on test set is:  0.681\n",
            "Epoch  39 Loss:  0.187 Accuracy: 0.9386 Epoch duration: 44.490s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch  40 Loss:  0.180 Accuracy: 0.9453 Epoch duration: 44.794s\n",
            "Accuracy on test set is:  0.679\n",
            "model saved at seq_mod/model\n",
            "Epoch  41 Loss:  0.157 Accuracy: 0.9527 Epoch duration: 44.897s\n",
            "Accuracy on test set is:  0.667\n",
            "Epoch  42 Loss:  0.140 Accuracy: 0.9618 Epoch duration: 44.647s\n",
            "Accuracy on test set is:  0.674\n",
            "Epoch  43 Loss:  0.157 Accuracy: 0.9534 Epoch duration: 44.751s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch  44 Loss:  0.155 Accuracy: 0.9524 Epoch duration: 44.536s\n",
            "Accuracy on test set is:  0.683\n",
            "Epoch  45 Loss:  0.139 Accuracy: 0.9630 Epoch duration: 44.660s\n",
            "Accuracy on test set is:  0.679\n",
            "model saved at seq_mod/model\n",
            "Epoch  46 Loss:  0.198 Accuracy: 0.9327 Epoch duration: 44.716s\n",
            "Accuracy on test set is:  0.706\n",
            "Epoch  47 Loss:  0.144 Accuracy: 0.9602 Epoch duration: 44.647s\n",
            "Accuracy on test set is:  0.680\n",
            "Epoch  48 Loss:  0.155 Accuracy: 0.9524 Epoch duration: 45.561s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch  49 Loss:  0.154 Accuracy: 0.9546 Epoch duration: 44.949s\n",
            "Accuracy on test set is:  0.686\n",
            "Epoch  50 Loss:  0.130 Accuracy: 0.9652 Epoch duration: 45.098s\n",
            "Accuracy on test set is:  0.692\n",
            "model saved at seq_mod/model\n",
            "Epoch  51 Loss:  0.148 Accuracy: 0.9540 Epoch duration: 45.153s\n",
            "Accuracy on test set is:  0.699\n",
            "Epoch  52 Loss:  0.132 Accuracy: 0.9643 Epoch duration: 44.963s\n",
            "Accuracy on test set is:  0.665\n",
            "Epoch  53 Loss:  0.139 Accuracy: 0.9604 Epoch duration: 48.331s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch  54 Loss:  0.154 Accuracy: 0.9541 Epoch duration: 45.235s\n",
            "Accuracy on test set is:  0.665\n",
            "Epoch  55 Loss:  0.123 Accuracy: 0.9668 Epoch duration: 45.288s\n",
            "Accuracy on test set is:  0.671\n",
            "model saved at seq_mod/model\n",
            "Epoch  56 Loss:  0.135 Accuracy: 0.9590 Epoch duration: 45.302s\n",
            "Accuracy on test set is:  0.693\n",
            "Epoch  57 Loss:  0.143 Accuracy: 0.9580 Epoch duration: 45.288s\n",
            "Accuracy on test set is:  0.670\n",
            "Epoch  58 Loss:  0.111 Accuracy: 0.9725 Epoch duration: 45.267s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch  59 Loss:  0.111 Accuracy: 0.9716 Epoch duration: 45.275s\n",
            "Accuracy on test set is:  0.675\n",
            "Epoch  60 Loss:  0.139 Accuracy: 0.9596 Epoch duration: 45.371s\n",
            "Accuracy on test set is:  0.688\n",
            "model saved at seq_mod/model\n",
            "Epoch  61 Loss:  0.121 Accuracy: 0.9680 Epoch duration: 45.282s\n",
            "Accuracy on test set is:  0.664\n",
            "Epoch  62 Loss:  0.124 Accuracy: 0.9662 Epoch duration: 45.330s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch  63 Loss:  0.182 Accuracy: 0.9388 Epoch duration: 48.774s\n",
            "Accuracy on test set is:  0.670\n",
            "Epoch  64 Loss:  0.115 Accuracy: 0.9688 Epoch duration: 45.167s\n",
            "Accuracy on test set is:  0.686\n",
            "Epoch  65 Loss:  0.125 Accuracy: 0.9651 Epoch duration: 45.218s\n",
            "Accuracy on test set is:  0.675\n",
            "model saved at seq_mod/model\n",
            "Epoch  66 Loss:  0.105 Accuracy: 0.9727 Epoch duration: 45.232s\n",
            "Accuracy on test set is:  0.700\n",
            "Epoch  67 Loss:  0.114 Accuracy: 0.9680 Epoch duration: 45.237s\n",
            "Accuracy on test set is:  0.689\n",
            "Epoch  68 Loss:  0.108 Accuracy: 0.9717 Epoch duration: 45.142s\n",
            "Accuracy on test set is:  0.687\n",
            "Epoch  69 Loss:  0.084 Accuracy: 0.9834 Epoch duration: 45.141s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch  70 Loss:  0.121 Accuracy: 0.9652 Epoch duration: 45.239s\n",
            "Accuracy on test set is:  0.669\n",
            "model saved at seq_mod/model\n",
            "Epoch  71 Loss:  0.115 Accuracy: 0.9697 Epoch duration: 45.791s\n",
            "Accuracy on test set is:  0.669\n",
            "Epoch  72 Loss:  0.127 Accuracy: 0.9634 Epoch duration: 45.719s\n",
            "Accuracy on test set is:  0.692\n",
            "Epoch  73 Loss:  0.120 Accuracy: 0.9677 Epoch duration: 45.732s\n",
            "Accuracy on test set is:  0.666\n",
            "Epoch  74 Loss:  0.138 Accuracy: 0.9579 Epoch duration: 45.877s\n",
            "Accuracy on test set is:  0.682\n",
            "Epoch  75 Loss:  0.095 Accuracy: 0.9768 Epoch duration: 45.603s\n",
            "Accuracy on test set is:  0.686\n",
            "model saved at seq_mod/model\n",
            "Epoch  76 Loss:  0.100 Accuracy: 0.9741 Epoch duration: 45.374s\n",
            "Accuracy on test set is:  0.699\n",
            "Epoch  77 Loss:  0.132 Accuracy: 0.9616 Epoch duration: 45.352s\n",
            "Accuracy on test set is:  0.681\n",
            "Epoch  78 Loss:  0.110 Accuracy: 0.9696 Epoch duration: 45.361s\n",
            "Accuracy on test set is:  0.695\n",
            "Epoch  79 Loss:  0.071 Accuracy: 0.9870 Epoch duration: 45.591s\n",
            "Accuracy on test set is:  0.673\n",
            "Epoch  80 Loss:  0.140 Accuracy: 0.9565 Epoch duration: 45.509s\n",
            "Accuracy on test set is:  0.690\n",
            "model saved at seq_mod/model\n",
            "Epoch  81 Loss:  0.097 Accuracy: 0.9759 Epoch duration: 45.677s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch  82 Loss:  0.075 Accuracy: 0.9854 Epoch duration: 45.544s\n",
            "Accuracy on test set is:  0.691\n",
            "Epoch  83 Loss:  0.070 Accuracy: 0.9872 Epoch duration: 45.526s\n",
            "Accuracy on test set is:  0.694\n",
            "Epoch  84 Loss:  0.075 Accuracy: 0.9856 Epoch duration: 48.434s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch  85 Loss:  0.229 Accuracy: 0.9220 Epoch duration: 45.455s\n",
            "Accuracy on test set is:  0.670\n",
            "model saved at seq_mod/model\n",
            "Epoch  86 Loss:  0.124 Accuracy: 0.9617 Epoch duration: 45.774s\n",
            "Accuracy on test set is:  0.686\n",
            "Epoch  87 Loss:  0.097 Accuracy: 0.9758 Epoch duration: 45.463s\n",
            "Accuracy on test set is:  0.674\n",
            "Epoch  88 Loss:  0.159 Accuracy: 0.9499 Epoch duration: 45.466s\n",
            "Accuracy on test set is:  0.686\n",
            "Epoch  89 Loss:  0.065 Accuracy: 0.9903 Epoch duration: 45.468s\n",
            "Accuracy on test set is:  0.670\n",
            "Epoch  90 Loss:  0.068 Accuracy: 0.9866 Epoch duration: 45.440s\n",
            "Accuracy on test set is:  0.688\n",
            "model saved at seq_mod/model\n",
            "Epoch  91 Loss:  0.056 Accuracy: 0.9919 Epoch duration: 45.754s\n",
            "Accuracy on test set is:  0.664\n",
            "Epoch  92 Loss:  0.063 Accuracy: 0.9896 Epoch duration: 45.444s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch  93 Loss:  0.117 Accuracy: 0.9641 Epoch duration: 45.495s\n",
            "Accuracy on test set is:  0.665\n",
            "Epoch  94 Loss:  0.098 Accuracy: 0.9741 Epoch duration: 49.116s\n",
            "Accuracy on test set is:  0.668\n",
            "Epoch  95 Loss:  0.072 Accuracy: 0.9860 Epoch duration: 45.473s\n",
            "Accuracy on test set is:  0.686\n",
            "model saved at seq_mod/model\n",
            "Epoch  96 Loss:  0.056 Accuracy: 0.9924 Epoch duration: 45.653s\n",
            "Accuracy on test set is:  0.674\n",
            "Epoch  97 Loss:  0.066 Accuracy: 0.9862 Epoch duration: 45.383s\n",
            "Accuracy on test set is:  0.686\n",
            "Epoch  98 Loss:  0.053 Accuracy: 0.9930 Epoch duration: 45.386s\n",
            "Accuracy on test set is:  0.682\n",
            "Epoch  99 Loss:  0.052 Accuracy: 0.9930 Epoch duration: 45.389s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch 100 Loss:  0.069 Accuracy: 0.9871 Epoch duration: 45.284s\n",
            "Accuracy on test set is:  0.677\n",
            "model saved at seq_mod/model\n",
            "Epoch 101 Loss:  0.079 Accuracy: 0.9799 Epoch duration: 45.548s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch 102 Loss:  0.139 Accuracy: 0.9541 Epoch duration: 45.570s\n",
            "Accuracy on test set is:  0.676\n",
            "Epoch 103 Loss:  0.135 Accuracy: 0.9554 Epoch duration: 45.555s\n",
            "Accuracy on test set is:  0.671\n",
            "Epoch 104 Loss:  0.123 Accuracy: 0.9613 Epoch duration: 47.958s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 105 Loss:  0.115 Accuracy: 0.9647 Epoch duration: 45.413s\n",
            "Accuracy on test set is:  0.682\n",
            "model saved at seq_mod/model\n",
            "Epoch 106 Loss:  0.149 Accuracy: 0.9499 Epoch duration: 45.306s\n",
            "Accuracy on test set is:  0.680\n",
            "Epoch 107 Loss:  0.075 Accuracy: 0.9816 Epoch duration: 45.318s\n",
            "Accuracy on test set is:  0.670\n",
            "Epoch 108 Loss:  0.071 Accuracy: 0.9839 Epoch duration: 45.108s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch 109 Loss:  0.067 Accuracy: 0.9863 Epoch duration: 45.080s\n",
            "Accuracy on test set is:  0.688\n",
            "Epoch 110 Loss:  0.062 Accuracy: 0.9876 Epoch duration: 45.312s\n",
            "Accuracy on test set is:  0.665\n",
            "model saved at seq_mod/model\n",
            "Epoch 111 Loss:  0.049 Accuracy: 0.9945 Epoch duration: 45.547s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch 112 Loss:  0.072 Accuracy: 0.9836 Epoch duration: 45.714s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch 113 Loss:  0.046 Accuracy: 0.9950 Epoch duration: 46.232s\n",
            "Accuracy on test set is:  0.669\n",
            "Epoch 114 Loss:  0.044 Accuracy: 0.9955 Epoch duration: 46.311s\n",
            "Accuracy on test set is:  0.687\n",
            "Epoch 115 Loss:  0.041 Accuracy: 0.9966 Epoch duration: 49.953s\n",
            "Accuracy on test set is:  0.680\n",
            "model saved at seq_mod/model\n",
            "Epoch 116 Loss:  0.040 Accuracy: 0.9965 Epoch duration: 46.455s\n",
            "Accuracy on test set is:  0.688\n",
            "Epoch 117 Loss:  0.043 Accuracy: 0.9960 Epoch duration: 45.647s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch 118 Loss:  0.041 Accuracy: 0.9962 Epoch duration: 45.375s\n",
            "Accuracy on test set is:  0.674\n",
            "Epoch 119 Loss:  0.043 Accuracy: 0.9953 Epoch duration: 45.603s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 120 Loss:  0.055 Accuracy: 0.9905 Epoch duration: 45.484s\n",
            "Accuracy on test set is:  0.680\n",
            "model saved at seq_mod/model\n",
            "Epoch 121 Loss:  0.055 Accuracy: 0.9902 Epoch duration: 45.555s\n",
            "Accuracy on test set is:  0.682\n",
            "Epoch 122 Loss:  0.060 Accuracy: 0.9881 Epoch duration: 45.828s\n",
            "Accuracy on test set is:  0.666\n",
            "Epoch 123 Loss:  0.098 Accuracy: 0.9702 Epoch duration: 45.646s\n",
            "Accuracy on test set is:  0.680\n",
            "Epoch 124 Loss:  0.121 Accuracy: 0.9622 Epoch duration: 45.627s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch 125 Loss:  0.177 Accuracy: 0.9427 Epoch duration: 50.103s\n",
            "Accuracy on test set is:  0.691\n",
            "model saved at seq_mod/model\n",
            "Epoch 126 Loss:  0.111 Accuracy: 0.9662 Epoch duration: 45.175s\n",
            "Accuracy on test set is:  0.669\n",
            "Epoch 127 Loss:  0.090 Accuracy: 0.9735 Epoch duration: 45.167s\n",
            "Accuracy on test set is:  0.674\n",
            "Epoch 128 Loss:  0.098 Accuracy: 0.9714 Epoch duration: 45.048s\n",
            "Accuracy on test set is:  0.680\n",
            "Epoch 129 Loss:  0.084 Accuracy: 0.9758 Epoch duration: 45.084s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch 130 Loss:  0.104 Accuracy: 0.9690 Epoch duration: 45.057s\n",
            "Accuracy on test set is:  0.688\n",
            "model saved at seq_mod/model\n",
            "Epoch 131 Loss:  0.159 Accuracy: 0.9455 Epoch duration: 45.508s\n",
            "Accuracy on test set is:  0.673\n",
            "Epoch 132 Loss:  0.254 Accuracy: 0.9141 Epoch duration: 45.499s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch 133 Loss:  0.102 Accuracy: 0.9688 Epoch duration: 45.533s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch 134 Loss:  0.030 Accuracy: 0.9984 Epoch duration: 45.412s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 135 Loss:  0.028 Accuracy: 0.9991 Epoch duration: 52.365s\n",
            "Accuracy on test set is:  0.696\n",
            "model saved at seq_mod/model\n",
            "Epoch 136 Loss:  0.029 Accuracy: 0.9986 Epoch duration: 46.594s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch 137 Loss:  0.029 Accuracy: 0.9981 Epoch duration: 46.814s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch 138 Loss:  0.031 Accuracy: 0.9985 Epoch duration: 46.900s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 139 Loss:  0.032 Accuracy: 0.9972 Epoch duration: 46.569s\n",
            "Accuracy on test set is:  0.667\n",
            "Epoch 140 Loss:  0.062 Accuracy: 0.9858 Epoch duration: 47.375s\n",
            "Accuracy on test set is:  0.673\n",
            "model saved at seq_mod/model\n",
            "Epoch 141 Loss:  0.130 Accuracy: 0.9572 Epoch duration: 46.372s\n",
            "Accuracy on test set is:  0.672\n",
            "Epoch 142 Loss:  0.168 Accuracy: 0.9432 Epoch duration: 46.587s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch 143 Loss:  0.114 Accuracy: 0.9632 Epoch duration: 45.901s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 144 Loss:  0.027 Accuracy: 0.9984 Epoch duration: 45.477s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch 145 Loss:  0.024 Accuracy: 0.9994 Epoch duration: 48.561s\n",
            "Accuracy on test set is:  0.688\n",
            "model saved at seq_mod/model\n",
            "Epoch 146 Loss:  0.025 Accuracy: 0.9991 Epoch duration: 45.246s\n",
            "Accuracy on test set is:  0.685\n",
            "Epoch 147 Loss:  0.027 Accuracy: 0.9991 Epoch duration: 45.148s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 148 Loss:  0.036 Accuracy: 0.9959 Epoch duration: 45.299s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch 149 Loss:  0.199 Accuracy: 0.9348 Epoch duration: 45.206s\n",
            "Accuracy on test set is:  0.687\n",
            "Epoch 150 Loss:  0.217 Accuracy: 0.9266 Epoch duration: 45.231s\n",
            "Accuracy on test set is:  0.680\n",
            "model saved at seq_mod/model\n",
            "Epoch 151 Loss:  0.031 Accuracy: 0.9970 Epoch duration: 45.533s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 152 Loss:  0.022 Accuracy: 0.9996 Epoch duration: 45.572s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch 153 Loss:  0.021 Accuracy: 0.9993 Epoch duration: 45.702s\n",
            "Accuracy on test set is:  0.672\n",
            "Epoch 154 Loss:  0.024 Accuracy: 0.9991 Epoch duration: 45.710s\n",
            "Accuracy on test set is:  0.686\n",
            "Epoch 155 Loss:  0.038 Accuracy: 0.9952 Epoch duration: 45.589s\n",
            "Accuracy on test set is:  0.681\n",
            "model saved at seq_mod/model\n",
            "Epoch 156 Loss:  0.151 Accuracy: 0.9481 Epoch duration: 45.723s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch 157 Loss:  0.117 Accuracy: 0.9646 Epoch duration: 45.460s\n",
            "Accuracy on test set is:  0.673\n",
            "Epoch 158 Loss:  0.022 Accuracy: 0.9994 Epoch duration: 45.470s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch 159 Loss:  0.020 Accuracy: 0.9997 Epoch duration: 45.505s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch 160 Loss:  0.021 Accuracy: 0.9995 Epoch duration: 45.452s\n",
            "Accuracy on test set is:  0.688\n",
            "model saved at seq_mod/model\n",
            "Epoch 161 Loss:  0.038 Accuracy: 0.9936 Epoch duration: 45.679s\n",
            "Accuracy on test set is:  0.668\n",
            "Epoch 162 Loss:  0.112 Accuracy: 0.9630 Epoch duration: 45.600s\n",
            "Accuracy on test set is:  0.682\n",
            "Epoch 163 Loss:  0.024 Accuracy: 0.9986 Epoch duration: 45.727s\n",
            "Accuracy on test set is:  0.690\n",
            "Epoch 164 Loss:  0.019 Accuracy: 0.9996 Epoch duration: 45.723s\n",
            "Accuracy on test set is:  0.687\n",
            "Epoch 165 Loss:  0.021 Accuracy: 0.9991 Epoch duration: 45.562s\n",
            "Accuracy on test set is:  0.688\n",
            "model saved at seq_mod/model\n",
            "Epoch 166 Loss:  0.060 Accuracy: 0.9848 Epoch duration: 47.524s\n",
            "Accuracy on test set is:  0.665\n",
            "Epoch 167 Loss:  0.053 Accuracy: 0.9865 Epoch duration: 45.705s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch 168 Loss:  0.017 Accuracy: 1.0000 Epoch duration: 45.696s\n",
            "Accuracy on test set is:  0.689\n",
            "Epoch 169 Loss:  0.019 Accuracy: 0.9998 Epoch duration: 45.744s\n",
            "Accuracy on test set is:  0.680\n",
            "Epoch 170 Loss:  0.026 Accuracy: 0.9979 Epoch duration: 45.572s\n",
            "Accuracy on test set is:  0.677\n",
            "model saved at seq_mod/model\n",
            "Epoch 171 Loss:  0.260 Accuracy: 0.9200 Epoch duration: 46.252s\n",
            "Accuracy on test set is:  0.660\n",
            "Epoch 172 Loss:  0.017 Accuracy: 0.9997 Epoch duration: 45.627s\n",
            "Accuracy on test set is:  0.686\n",
            "Epoch 173 Loss:  0.017 Accuracy: 0.9998 Epoch duration: 45.758s\n",
            "Accuracy on test set is:  0.672\n",
            "Epoch 174 Loss:  0.031 Accuracy: 0.9959 Epoch duration: 46.091s\n",
            "Accuracy on test set is:  0.676\n",
            "Epoch 175 Loss:  0.250 Accuracy: 0.9227 Epoch duration: 46.522s\n",
            "Accuracy on test set is:  0.682\n",
            "model saved at seq_mod/model\n",
            "Epoch 176 Loss:  0.017 Accuracy: 0.9996 Epoch duration: 49.464s\n",
            "Accuracy on test set is:  0.673\n",
            "Epoch 177 Loss:  0.018 Accuracy: 0.9996 Epoch duration: 46.586s\n",
            "Accuracy on test set is:  0.675\n",
            "Epoch 178 Loss:  0.026 Accuracy: 0.9975 Epoch duration: 46.695s\n",
            "Accuracy on test set is:  0.661\n",
            "Epoch 179 Loss:  0.171 Accuracy: 0.9423 Epoch duration: 46.655s\n",
            "Accuracy on test set is:  0.679\n",
            "Epoch 180 Loss:  0.015 Accuracy: 0.9999 Epoch duration: 46.828s\n",
            "Accuracy on test set is:  0.674\n",
            "model saved at seq_mod/model\n",
            "Epoch 181 Loss:  0.017 Accuracy: 0.9996 Epoch duration: 46.739s\n",
            "Accuracy on test set is:  0.684\n",
            "Epoch 182 Loss:  0.033 Accuracy: 0.9949 Epoch duration: 46.575s\n",
            "Accuracy on test set is:  0.672\n",
            "Epoch 183 Loss:  0.182 Accuracy: 0.9405 Epoch duration: 46.761s\n",
            "Accuracy on test set is:  0.682\n",
            "Epoch 184 Loss:  0.014 Accuracy: 0.9999 Epoch duration: 46.593s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 185 Loss:  0.017 Accuracy: 0.9996 Epoch duration: 46.502s\n",
            "Accuracy on test set is:  0.671\n",
            "model saved at seq_mod/model\n",
            "Epoch 186 Loss:  0.109 Accuracy: 0.9645 Epoch duration: 50.919s\n",
            "Accuracy on test set is:  0.683\n",
            "Epoch 187 Loss:  0.016 Accuracy: 0.9995 Epoch duration: 46.576s\n",
            "Accuracy on test set is:  0.697\n",
            "Epoch 188 Loss:  0.015 Accuracy: 0.9998 Epoch duration: 46.616s\n",
            "Accuracy on test set is:  0.672\n",
            "Epoch 189 Loss:  0.021 Accuracy: 0.9984 Epoch duration: 46.616s\n",
            "Accuracy on test set is:  0.690\n",
            "Epoch 190 Loss:  0.283 Accuracy: 0.9133 Epoch duration: 46.603s\n",
            "Accuracy on test set is:  0.682\n",
            "model saved at seq_mod/model\n",
            "Epoch 191 Loss:  0.014 Accuracy: 0.9998 Epoch duration: 46.519s\n",
            "Accuracy on test set is:  0.677\n",
            "Epoch 192 Loss:  0.016 Accuracy: 0.9998 Epoch duration: 46.602s\n",
            "Accuracy on test set is:  0.669\n",
            "Epoch 193 Loss:  0.107 Accuracy: 0.9649 Epoch duration: 46.612s\n",
            "Accuracy on test set is:  0.678\n",
            "Epoch 194 Loss:  0.014 Accuracy: 0.9998 Epoch duration: 46.571s\n",
            "Accuracy on test set is:  0.676\n",
            "Epoch 195 Loss:  0.014 Accuracy: 0.9999 Epoch duration: 46.678s\n",
            "Accuracy on test set is:  0.688\n",
            "model saved at seq_mod/model\n",
            "Epoch 196 Loss:  0.040 Accuracy: 0.9914 Epoch duration: 51.757s\n",
            "Accuracy on test set is:  0.688\n",
            "Epoch 197 Loss:  0.026 Accuracy: 0.9965 Epoch duration: 46.719s\n",
            "Accuracy on test set is:  0.680\n",
            "Epoch 198 Loss:  0.013 Accuracy: 0.9998 Epoch duration: 46.772s\n",
            "Accuracy on test set is:  0.689\n",
            "Epoch 199 Loss:  0.017 Accuracy: 0.9993 Epoch duration: 46.595s\n",
            "Accuracy on test set is:  0.687\n",
            "model saved at seq_mod/model\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "-8PTFAcXjB29"
      },
      "source": [
        "Translate on test set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Nj5SqSmDjB2-",
        "outputId": "26a0f16b-e9e7-4847-c2f3-2cca8d07587d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "with tf.Session() as sess: \n",
        "    restore(sess)\n",
        "    batch_size = 512\n",
        "    source_batch, target_batch = next(batch_data(X_test, y_test, batch_size))\n",
        "\n",
        "    dec_input = np.zeros((len(source_batch), 1)) + len(ltokens)\n",
        "    for i in range(y_seq_length):\n",
        "        batch_logits = sess.run(logits,\n",
        "                    feed_dict = {inputs: source_batch,\n",
        "                     outputs: dec_input})\n",
        "        prediction = batch_logits[:,-1].argmax(axis=-1)\n",
        "        dec_input = np.hstack([dec_input, prediction[:,None]])\n",
        "\n",
        "    print('Accuracy on test set is: {:>6.3f}'.format(np.mean(dec_input == target_batch)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from seq_mod/model\n",
            "model restored from seq_mod/model\n",
            "Accuracy on test set is:  0.687\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "iPFJMJoyjB3B"
      },
      "source": [
        "Let's randomly take two from this test set and see what it spits out:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sGdJ8TtBoFOk",
        "outputId": "73f4dd10-1976-41e3-b23c-5aabf934a140",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "source": [
        "i = 100\n",
        "print(dec_input[i,:])\n",
        "print(ltokens)\n",
        "seq = \"\"\n",
        "for c in dec_input[i,1:]:\n",
        "    c = int(c)\n",
        "    if c != 28:\n",
        "        seq += ltokens[c] \n",
        "        \n",
        "print(seq)\n",
        "\n",
        "seq = \"\"\n",
        "for c in target_batch[i,1:]:\n",
        "    c = int(c)\n",
        "    \n",
        "    if c != 28:\n",
        "        seq += ltokens[c] \n",
        "\n",
        "print(\"Correct: \", seq)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[33. 28. 27. 24. 27. 25.  7.  6.  7. 26. 25.  1.  5.  0. 26. 21. 23.  4.\n",
            " 10. 30. 10. 31. 27. 11.  3.  8.]\n",
            "['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '+', '=', '#leq', '#neq', '#geq', '#alpha', '#beta', '#lambda', '#lt', '#gt', 'x', 'y', '^', '#frac', '{', '}', ' ', '#int', 'd', '#sinx', '#cosx', '#tanx']\n",
            " #frac {767}{150}x^4-#sinx-#cosx +38\n",
            "Correct:   #frac {846}{542}x^4-#cosx-#cosx +83\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}