SupervisedAssignment3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SupervisedAssignment3.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNii2OxzNCFyb+byzKyZpSH",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/esraa9322/CNN/blob/main/SupervisedAssignment3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#pip install tensorflow==2.7.0"
      ],
      "metadata": {
        "id": "OFTrWFvxDndm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4JNxBZnYihlO"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from keras.datasets import mnist \n",
        "from matplotlib import pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(train_X, train_labels), (test_X, test_labels) = mnist.load_data()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5rIpKnxJi1d_",
        "outputId": "04cbdf45-7c9b-41d7-b6eb-f1c4e1cbe4f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n",
            "11501568/11490434 [==============================] - 0s 0us/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_X.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vwGvXuyEjMO3",
        "outputId": "eb8473ba-40f8-4a4b-8d71-02bf448a9d62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60000, 28, 28)"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.imshow(train_X[0],cmap = plt.cm.binary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "xyjYvgOyjYTH",
        "outputId": "d2b01c01-0866-4983-ab1f-337ce3e941e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f61e4892c50>"
            ]
          },
          "metadata": {},
          "execution_count": 6
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOUElEQVR4nO3dX4xUdZrG8ecF8R+DCkuHtAyRGTQmHY1AStgEg+hk8U+iwI2BGERjxAuQmQTiolzAhRdGd2YyihnTqAE2IxPCSITErIMEY4iJoVC2BZVFTeNA+FOE6Dh6gTLvXvRh0mLXr5qqU3XKfr+fpNPV56nT502Fh1Ndp7t+5u4CMPQNK3oAAK1B2YEgKDsQBGUHgqDsQBAXtfJgY8eO9YkTJ7bykEAovb29OnXqlA2UNVR2M7tT0h8kDZf0krs/nbr/xIkTVS6XGzkkgIRSqVQ1q/tpvJkNl/SCpLskdUlaYGZd9X4/AM3VyM/s0yR96u6fu/sZSX+WNCefsQDkrZGyj5f0t35fH8m2/YCZLTazspmVK5VKA4cD0Iimvxrv7t3uXnL3UkdHR7MPB6CKRsp+VNKEfl//PNsGoA01UvY9kq4zs1+Y2cWS5kvals9YAPJW96U3d//ezJZKelN9l95ecfcDuU0GIFcNXWd39zckvZHTLACaiF+XBYKg7EAQlB0IgrIDQVB2IAjKDgRB2YEgKDsQBGUHgqDsQBCUHQiCsgNBUHYgCMoOBEHZgSAoOxAEZQeCoOxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQVB2IIiGVnFF+zt79mwy/+qrr5p6/LVr11bNvv322+S+Bw8eTOYvvPBCMl+xYkXVbNOmTcl9L7300mS+cuXKZL569epkXoSGym5mvZK+lnRW0vfuXspjKAD5y+PMfpu7n8rh+wBoIn5mB4JotOwu6a9mttfMFg90BzNbbGZlMytXKpUGDwegXo2W/RZ3nyrpLklLzGzm+Xdw9253L7l7qaOjo8HDAahXQ2V396PZ55OStkqalsdQAPJXd9nNbKSZjTp3W9JsSfvzGgxAvhp5NX6cpK1mdu77vOru/5PLVEPMF198kczPnDmTzN99991kvnv37qrZl19+mdx3y5YtybxIEyZMSOaPPfZYMt+6dWvVbNSoUcl9b7rppmR+6623JvN2VHfZ3f1zSelHBEDb4NIbEARlB4Kg7EAQlB0IgrIDQfAnrjn44IMPkvntt9+ezJv9Z6btavjw4cn8qaeeSuYjR45M5vfff3/V7Oqrr07uO3r06GR+/fXXJ/N2xJkdCIKyA0FQdiAIyg4EQdmBICg7EARlB4LgOnsOrrnmmmQ+duzYZN7O19mnT5+ezGtdj961a1fV7OKLL07uu3DhwmSOC8OZHQiCsgNBUHYgCMoOBEHZgSAoOxAEZQeC4Dp7DsaMGZPMn3322WS+ffv2ZD5lypRkvmzZsmSeMnny5GT+1ltvJfNaf1O+f3/1pQSee+655L7IF2d2IAjKDgRB2YEgKDsQBGUHgqDsQBCUHQiC6+wtMHfu3GRe633lay0v3NPTUzV76aWXkvuuWLEimde6jl7LDTfcUDXr7u5u6HvjwtQ8s5vZK2Z20sz299s2xsx2mNmh7HP6HQwAFG4wT+PXS7rzvG0rJe109+sk7cy+BtDGapbd3d+RdPq8zXMkbchub5CUfp4KoHD1vkA3zt2PZbePSxpX7Y5mttjMymZWrlQqdR4OQKMafjXe3V2SJ/Judy+5e6mjo6PRwwGoU71lP2FmnZKUfT6Z30gAmqHesm+TtCi7vUjS6/mMA6BZal5nN7NNkmZJGmtmRyStlvS0pM1m9rCkw5Lua+aQQ90VV1zR0P5XXnll3fvWug4/f/78ZD5sGL+X9VNRs+zuvqBK9KucZwHQRPy3DARB2YEgKDsQBGUHgqDsQBD8iesQsGbNmqrZ3r17k/u+/fbbybzWW0nPnj07maN9cGYHgqDsQBCUHQiCsgNBUHYgCMoOBEHZgSC4zj4EpN7ued26dcl9p06dmswfeeSRZH7bbbcl81KpVDVbsmRJcl8zS+a4MJzZgSAoOxAEZQeCoOxAEJQdCIKyA0FQdiAIrrMPcZMmTUrm69evT+YPPfRQMt+4cWPd+TfffJPc94EHHkjmnZ2dyRw/xJkdCIKyA0FQdiAIyg4EQdmBICg7EARlB4LgOntw8+bNS+bXXnttMl++fHkyT73v/BNPPJHc9/Dhw8l81apVyXz8+PHJPJqaZ3Yze8XMTprZ/n7b1pjZUTPbl33c3dwxATRqME/j10u6c4Dtv3f3ydnHG/mOBSBvNcvu7u9IOt2CWQA0USMv0C01s57saf7oancys8VmVjazcqVSaeBwABpRb9n/KGmSpMmSjkn6bbU7unu3u5fcvdTR0VHn4QA0qq6yu/sJdz/r7v+UtE7StHzHApC3uspuZv3/tnCepP3V7gugPdS8zm5mmyTNkjTWzI5IWi1plplNluSSeiU92sQZUaAbb7wxmW/evDmZb9++vWr24IMPJvd98cUXk/mhQ4eS+Y4dO5J5NDXL7u4LBtj8chNmAdBE/LosEARlB4Kg7EAQlB0IgrIDQZi7t+xgpVLJy+Vyy46H9nbJJZck8++++y6ZjxgxIpm/+eabVbNZs2Yl9/2pKpVKKpfLA651zZkdCIKyA0FQdiAIyg4EQdmBICg7EARlB4LgraSR1NPTk8y3bNmSzPfs2VM1q3UdvZaurq5kPnPmzIa+/1DDmR0IgrIDQVB2IAjKDgRB2YEgKDsQBGUHguA6+xB38ODBZP78888n89deey2ZHz9+/IJnGqyLLkr/8+zs7Ezmw4ZxLuuPRwMIgrIDQVB2IAjKDgRB2YEgKDsQBGUHguA6+09ArWvZr776atVs7dq1yX17e3vrGSkXN998czJftWpVMr/33nvzHGfIq3lmN7MJZrbLzD4yswNm9uts+xgz22Fmh7LPo5s/LoB6DeZp/PeSlrt7l6R/l7TEzLokrZS0092vk7Qz+xpAm6pZdnc/5u7vZ7e/lvSxpPGS5kjakN1tg6S5zRoSQOMu6AU6M5soaYqk9ySNc/djWXRc0rgq+yw2s7KZlSuVSgOjAmjEoMtuZj+T9BdJv3H3v/fPvG91yAFXiHT3bncvuXupo6OjoWEB1G9QZTezEeor+p/c/dyfQZ0ws84s75R0sjkjAshDzUtvZmaSXpb0sbv/rl+0TdIiSU9nn19vyoRDwIkTJ5L5gQMHkvnSpUuT+SeffHLBM+Vl+vTpyfzxxx+vms2ZMye5L3+imq/BXGefIWmhpA/NbF+27Un1lXyzmT0s6bCk+5ozIoA81Cy7u++WNODi7pJ+le84AJqF50lAEJQdCIKyA0FQdiAIyg4EwZ+4DtLp06erZo8++mhy33379iXzzz77rK6Z8jBjxoxkvnz58mR+xx13JPPLLrvsgmdCc3BmB4Kg7EAQlB0IgrIDQVB2IAjKDgRB2YEgwlxnf++995L5M888k8z37NlTNTty5EhdM+Xl8ssvr5otW7YsuW+tt2seOXJkXTOh/XBmB4Kg7EAQlB0IgrIDQVB2IAjKDgRB2YEgwlxn37p1a0N5I7q6upL5Pffck8yHDx+ezFesWFE1u+qqq5L7Ig7O7EAQlB0IgrIDQVB2IAjKDgRB2YEgKDsQhLl7+g5mEyRtlDROkkvqdvc/mNkaSY9IqmR3fdLd30h9r1Kp5OVyueGhAQysVCqpXC4PuOryYH6p5ntJy939fTMbJWmvme3Ist+7+3/lNSiA5hnM+uzHJB3Lbn9tZh9LGt/swQDk64J+ZjeziZKmSDr3Hk9LzazHzF4xs9FV9llsZmUzK1cqlYHuAqAFBl12M/uZpL9I+o27/13SHyVNkjRZfWf+3w60n7t3u3vJ3UsdHR05jAygHoMqu5mNUF/R/+Tur0mSu59w97Pu/k9J6yRNa96YABpVs+xmZpJelvSxu/+u3/bOfnebJ2l//uMByMtgXo2fIWmhpA/N7Nzaw09KWmBmk9V3Oa5XUnrdYgCFGsyr8bslDXTdLnlNHUB74TfogCAoOxAEZQeCoOxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQdR8K+lcD2ZWkXS436axkk61bIAL066ztetcErPVK8/ZrnH3Ad//raVl/9HBzcruXipsgIR2na1d55KYrV6tmo2n8UAQlB0Iouiydxd8/JR2na1d55KYrV4tma3Qn9kBtE7RZ3YALULZgSAKKbuZ3WlmB83sUzNbWcQM1ZhZr5l9aGb7zKzQ9aWzNfROmtn+ftvGmNkOMzuUfR5wjb2CZltjZkezx26fmd1d0GwTzGyXmX1kZgfM7NfZ9kIfu8RcLXncWv4zu5kNl/R/kv5D0hFJeyQtcPePWjpIFWbWK6nk7oX/AoaZzZT0D0kb3f2GbNszkk67+9PZf5Sj3f0/22S2NZL+UfQy3tlqRZ39lxmXNFfSgyrwsUvMdZ9a8LgVcWafJulTd//c3c9I+rOkOQXM0fbc/R1Jp8/bPEfShuz2BvX9Y2m5KrO1BXc/5u7vZ7e/lnRumfFCH7vEXC1RRNnHS/pbv6+PqL3We3dJfzWzvWa2uOhhBjDO3Y9lt49LGlfkMAOouYx3K523zHjbPHb1LH/eKF6g+7Fb3H2qpLskLcmerrYl7/sZrJ2unQ5qGe9WGWCZ8X8p8rGrd/nzRhVR9qOSJvT7+ufZtrbg7kezzyclbVX7LUV94twKutnnkwXP8y/ttIz3QMuMqw0euyKXPy+i7HskXWdmvzCziyXNl7StgDl+xMxGZi+cyMxGSpqt9luKepukRdntRZJeL3CWH2iXZbyrLTOugh+7wpc/d/eWf0i6W32vyH8maVURM1SZ65eS/jf7OFD0bJI2qe9p3Xfqe23jYUn/JmmnpEOS3pI0po1m+29JH0rqUV+xOgua7Rb1PUXvkbQv+7i76McuMVdLHjd+XRYIghfogCAoOxAEZQeCoOxAEJQdCIKyA0FQdiCI/wfvpjt5Q0mdXQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_X = train_X / 255.0\n",
        "test_X = test_X / 255.0\n"
      ],
      "metadata": {
        "id": "dRtrUG8HlW_J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.imshow(train_X[0],cmap = plt.cm.binary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "XL_rbOL1mNfd",
        "outputId": "ab2b9f35-e476-447e-a1f1-12b5d0bfe00f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f61e4378d90>"
            ]
          },
          "metadata": {},
          "execution_count": 8
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOUElEQVR4nO3dX4xUdZrG8ecF8R+DCkuHtAyRGTQmHY1AStgEg+hk8U+iwI2BGERjxAuQmQTiolzAhRdGd2YyihnTqAE2IxPCSITErIMEY4iJoVC2BZVFTeNA+FOE6Dh6gTLvXvRh0mLXr5qqU3XKfr+fpNPV56nT502Fh1Ndp7t+5u4CMPQNK3oAAK1B2YEgKDsQBGUHgqDsQBAXtfJgY8eO9YkTJ7bykEAovb29OnXqlA2UNVR2M7tT0h8kDZf0krs/nbr/xIkTVS6XGzkkgIRSqVQ1q/tpvJkNl/SCpLskdUlaYGZd9X4/AM3VyM/s0yR96u6fu/sZSX+WNCefsQDkrZGyj5f0t35fH8m2/YCZLTazspmVK5VKA4cD0Iimvxrv7t3uXnL3UkdHR7MPB6CKRsp+VNKEfl//PNsGoA01UvY9kq4zs1+Y2cWS5kvals9YAPJW96U3d//ezJZKelN9l95ecfcDuU0GIFcNXWd39zckvZHTLACaiF+XBYKg7EAQlB0IgrIDQVB2IAjKDgRB2YEgKDsQBGUHgqDsQBCUHQiCsgNBUHYgCMoOBEHZgSAoOxAEZQeCoOxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQVB2IIiGVnFF+zt79mwy/+qrr5p6/LVr11bNvv322+S+Bw8eTOYvvPBCMl+xYkXVbNOmTcl9L7300mS+cuXKZL569epkXoSGym5mvZK+lnRW0vfuXspjKAD5y+PMfpu7n8rh+wBoIn5mB4JotOwu6a9mttfMFg90BzNbbGZlMytXKpUGDwegXo2W/RZ3nyrpLklLzGzm+Xdw9253L7l7qaOjo8HDAahXQ2V396PZ55OStkqalsdQAPJXd9nNbKSZjTp3W9JsSfvzGgxAvhp5NX6cpK1mdu77vOru/5PLVEPMF198kczPnDmTzN99991kvnv37qrZl19+mdx3y5YtybxIEyZMSOaPPfZYMt+6dWvVbNSoUcl9b7rppmR+6623JvN2VHfZ3f1zSelHBEDb4NIbEARlB4Kg7EAQlB0IgrIDQfAnrjn44IMPkvntt9+ezJv9Z6btavjw4cn8qaeeSuYjR45M5vfff3/V7Oqrr07uO3r06GR+/fXXJ/N2xJkdCIKyA0FQdiAIyg4EQdmBICg7EARlB4LgOnsOrrnmmmQ+duzYZN7O19mnT5+ezGtdj961a1fV7OKLL07uu3DhwmSOC8OZHQiCsgNBUHYgCMoOBEHZgSAoOxAEZQeC4Dp7DsaMGZPMn3322WS+ffv2ZD5lypRkvmzZsmSeMnny5GT+1ltvJfNaf1O+f3/1pQSee+655L7IF2d2IAjKDgRB2YEgKDsQBGUHgqDsQBCUHQiC6+wtMHfu3GRe633lay0v3NPTUzV76aWXkvuuWLEimde6jl7LDTfcUDXr7u5u6HvjwtQ8s5vZK2Z20sz299s2xsx2mNmh7HP6HQwAFG4wT+PXS7rzvG0rJe109+sk7cy+BtDGapbd3d+RdPq8zXMkbchub5CUfp4KoHD1vkA3zt2PZbePSxpX7Y5mttjMymZWrlQqdR4OQKMafjXe3V2SJ/Judy+5e6mjo6PRwwGoU71lP2FmnZKUfT6Z30gAmqHesm+TtCi7vUjS6/mMA6BZal5nN7NNkmZJGmtmRyStlvS0pM1m9rCkw5Lua+aQQ90VV1zR0P5XXnll3fvWug4/f/78ZD5sGL+X9VNRs+zuvqBK9KucZwHQRPy3DARB2YEgKDsQBGUHgqDsQBD8iesQsGbNmqrZ3r17k/u+/fbbybzWW0nPnj07maN9cGYHgqDsQBCUHQiCsgNBUHYgCMoOBEHZgSC4zj4EpN7ued26dcl9p06dmswfeeSRZH7bbbcl81KpVDVbsmRJcl8zS+a4MJzZgSAoOxAEZQeCoOxAEJQdCIKyA0FQdiAIrrMPcZMmTUrm69evT+YPPfRQMt+4cWPd+TfffJPc94EHHkjmnZ2dyRw/xJkdCIKyA0FQdiAIyg4EQdmBICg7EARlB4LgOntw8+bNS+bXXnttMl++fHkyT73v/BNPPJHc9/Dhw8l81apVyXz8+PHJPJqaZ3Yze8XMTprZ/n7b1pjZUTPbl33c3dwxATRqME/j10u6c4Dtv3f3ydnHG/mOBSBvNcvu7u9IOt2CWQA0USMv0C01s57saf7oancys8VmVjazcqVSaeBwABpRb9n/KGmSpMmSjkn6bbU7unu3u5fcvdTR0VHn4QA0qq6yu/sJdz/r7v+UtE7StHzHApC3uspuZv3/tnCepP3V7gugPdS8zm5mmyTNkjTWzI5IWi1plplNluSSeiU92sQZUaAbb7wxmW/evDmZb9++vWr24IMPJvd98cUXk/mhQ4eS+Y4dO5J5NDXL7u4LBtj8chNmAdBE/LosEARlB4Kg7EAQlB0IgrIDQZi7t+xgpVLJy+Vyy46H9nbJJZck8++++y6ZjxgxIpm/+eabVbNZs2Yl9/2pKpVKKpfLA651zZkdCIKyA0FQdiAIyg4EQdmBICg7EARlB4LgraSR1NPTk8y3bNmSzPfs2VM1q3UdvZaurq5kPnPmzIa+/1DDmR0IgrIDQVB2IAjKDgRB2YEgKDsQBGUHguA6+xB38ODBZP78888n89deey2ZHz9+/IJnGqyLLkr/8+zs7Ezmw4ZxLuuPRwMIgrIDQVB2IAjKDgRB2YEgKDsQBGUHguA6+09ArWvZr776atVs7dq1yX17e3vrGSkXN998czJftWpVMr/33nvzHGfIq3lmN7MJZrbLzD4yswNm9uts+xgz22Fmh7LPo5s/LoB6DeZp/PeSlrt7l6R/l7TEzLokrZS0092vk7Qz+xpAm6pZdnc/5u7vZ7e/lvSxpPGS5kjakN1tg6S5zRoSQOMu6AU6M5soaYqk9ySNc/djWXRc0rgq+yw2s7KZlSuVSgOjAmjEoMtuZj+T9BdJv3H3v/fPvG91yAFXiHT3bncvuXupo6OjoWEB1G9QZTezEeor+p/c/dyfQZ0ws84s75R0sjkjAshDzUtvZmaSXpb0sbv/rl+0TdIiSU9nn19vyoRDwIkTJ5L5gQMHkvnSpUuT+SeffHLBM+Vl+vTpyfzxxx+vms2ZMye5L3+imq/BXGefIWmhpA/NbF+27Un1lXyzmT0s6bCk+5ozIoA81Cy7u++WNODi7pJ+le84AJqF50lAEJQdCIKyA0FQdiAIyg4EwZ+4DtLp06erZo8++mhy33379iXzzz77rK6Z8jBjxoxkvnz58mR+xx13JPPLLrvsgmdCc3BmB4Kg7EAQlB0IgrIDQVB2IAjKDgRB2YEgwlxnf++995L5M888k8z37NlTNTty5EhdM+Xl8ssvr5otW7YsuW+tt2seOXJkXTOh/XBmB4Kg7EAQlB0IgrIDQVB2IAjKDgRB2YEgwlxn37p1a0N5I7q6upL5Pffck8yHDx+ezFesWFE1u+qqq5L7Ig7O7EAQlB0IgrIDQVB2IAjKDgRB2YEgKDsQhLl7+g5mEyRtlDROkkvqdvc/mNkaSY9IqmR3fdLd30h9r1Kp5OVyueGhAQysVCqpXC4PuOryYH6p5ntJy939fTMbJWmvme3Ist+7+3/lNSiA5hnM+uzHJB3Lbn9tZh9LGt/swQDk64J+ZjeziZKmSDr3Hk9LzazHzF4xs9FV9llsZmUzK1cqlYHuAqAFBl12M/uZpL9I+o27/13SHyVNkjRZfWf+3w60n7t3u3vJ3UsdHR05jAygHoMqu5mNUF/R/+Tur0mSu59w97Pu/k9J6yRNa96YABpVs+xmZpJelvSxu/+u3/bOfnebJ2l//uMByMtgXo2fIWmhpA/N7Nzaw09KWmBmk9V3Oa5XUnrdYgCFGsyr8bslDXTdLnlNHUB74TfogCAoOxAEZQeCoOxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQdR8K+lcD2ZWkXS436axkk61bIAL066ztetcErPVK8/ZrnH3Ad//raVl/9HBzcruXipsgIR2na1d55KYrV6tmo2n8UAQlB0Iouiydxd8/JR2na1d55KYrV4tma3Qn9kBtE7RZ3YALULZgSAKKbuZ3WlmB83sUzNbWcQM1ZhZr5l9aGb7zKzQ9aWzNfROmtn+ftvGmNkOMzuUfR5wjb2CZltjZkezx26fmd1d0GwTzGyXmX1kZgfM7NfZ9kIfu8RcLXncWv4zu5kNl/R/kv5D0hFJeyQtcPePWjpIFWbWK6nk7oX/AoaZzZT0D0kb3f2GbNszkk67+9PZf5Sj3f0/22S2NZL+UfQy3tlqRZ39lxmXNFfSgyrwsUvMdZ9a8LgVcWafJulTd//c3c9I+rOkOQXM0fbc/R1Jp8/bPEfShuz2BvX9Y2m5KrO1BXc/5u7vZ7e/lnRumfFCH7vEXC1RRNnHS/pbv6+PqL3We3dJfzWzvWa2uOhhBjDO3Y9lt49LGlfkMAOouYx3K523zHjbPHb1LH/eKF6g+7Fb3H2qpLskLcmerrYl7/sZrJ2unQ5qGe9WGWCZ8X8p8rGrd/nzRhVR9qOSJvT7+ufZtrbg7kezzyclbVX7LUV94twKutnnkwXP8y/ttIz3QMuMqw0euyKXPy+i7HskXWdmvzCziyXNl7StgDl+xMxGZi+cyMxGSpqt9luKepukRdntRZJeL3CWH2iXZbyrLTOugh+7wpc/d/eWf0i6W32vyH8maVURM1SZ65eS/jf7OFD0bJI2qe9p3Xfqe23jYUn/JmmnpEOS3pI0po1m+29JH0rqUV+xOgua7Rb1PUXvkbQv+7i76McuMVdLHjd+XRYIghfogCAoOxAEZQeCoOxAEJQdCIKyA0FQdiCI/wfvpjt5Q0mdXQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(train_X[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hyVKx9vsmTST",
        "outputId": "082e3561-ed5b-4be1-f8fc-9374646aaa08"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.01176471 0.07058824 0.07058824 0.07058824 0.49411765 0.53333333\n",
            "  0.68627451 0.10196078 0.65098039 1.         0.96862745 0.49803922\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.11764706 0.14117647 0.36862745 0.60392157\n",
            "  0.66666667 0.99215686 0.99215686 0.99215686 0.99215686 0.99215686\n",
            "  0.88235294 0.6745098  0.99215686 0.94901961 0.76470588 0.25098039\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.19215686 0.93333333 0.99215686 0.99215686 0.99215686\n",
            "  0.99215686 0.99215686 0.99215686 0.99215686 0.99215686 0.98431373\n",
            "  0.36470588 0.32156863 0.32156863 0.21960784 0.15294118 0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.07058824 0.85882353 0.99215686 0.99215686 0.99215686\n",
            "  0.99215686 0.99215686 0.77647059 0.71372549 0.96862745 0.94509804\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.31372549 0.61176471 0.41960784 0.99215686\n",
            "  0.99215686 0.80392157 0.04313725 0.         0.16862745 0.60392157\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.05490196 0.00392157 0.60392157\n",
            "  0.99215686 0.35294118 0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.54509804\n",
            "  0.99215686 0.74509804 0.00784314 0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.04313725\n",
            "  0.74509804 0.99215686 0.2745098  0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.1372549  0.94509804 0.88235294 0.62745098 0.42352941 0.00392157\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.31764706 0.94117647 0.99215686 0.99215686 0.46666667\n",
            "  0.09803922 0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.17647059 0.72941176 0.99215686 0.99215686\n",
            "  0.58823529 0.10588235 0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.0627451  0.36470588 0.98823529\n",
            "  0.99215686 0.73333333 0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.97647059\n",
            "  0.99215686 0.97647059 0.25098039 0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.18039216 0.50980392 0.71764706 0.99215686\n",
            "  0.99215686 0.81176471 0.00784314 0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.15294118 0.58039216 0.89803922 0.99215686 0.99215686 0.99215686\n",
            "  0.98039216 0.71372549 0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.09411765 0.44705882\n",
            "  0.86666667 0.99215686 0.99215686 0.99215686 0.99215686 0.78823529\n",
            "  0.30588235 0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.09019608 0.25882353 0.83529412 0.99215686\n",
            "  0.99215686 0.99215686 0.99215686 0.77647059 0.31764706 0.00784314\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.07058824 0.67058824 0.85882353 0.99215686 0.99215686 0.99215686\n",
            "  0.99215686 0.76470588 0.31372549 0.03529412 0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.21568627 0.6745098\n",
            "  0.88627451 0.99215686 0.99215686 0.99215686 0.99215686 0.95686275\n",
            "  0.52156863 0.04313725 0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.53333333 0.99215686\n",
            "  0.99215686 0.99215686 0.83137255 0.52941176 0.51764706 0.0627451\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.        ]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(train_labels[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1F6RRJqkmbNq",
        "outputId": "47c7890b-f7a6-4f51-cf10-43731711301e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "train_X = np.array(train_X).reshape(-1,28,28,1)\n",
        "test_X = np.array(test_X).reshape(-1,28,28,1)"
      ],
      "metadata": {
        "id": "HTz6YcgSprFf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense,Dropout,Activation,Flatten,Conv2D,MaxPooling2D,Softmax"
      ],
      "metadata": {
        "id": "UO-uYBgtMGHx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "GlA1MM0tFbSW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "THE INITAIL MODEL 2 CNN LAYERS , 2 FC LAYERS , SGD **bold text**\n",
        "\n"
      ],
      "metadata": {
        "id": "uNY-wvEeF6L4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.python.keras.activations import softmax\n",
        "from tensorflow.python.keras.backend import conv2d\n",
        "model1 =Sequential()\n",
        "#first convolution layer\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))"
      ],
      "metadata": {
        "id": "UPIaX5EGMbnu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FwAJ6ahlbuXz",
        "outputId": "7d2de478-d820-481d-f779-7c385b428850"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d (Conv2D)             (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d (MaxPooling2D  (None, 13, 13, 32)       0         \n",
            " )                                                               \n",
            "                                                                 \n",
            " conv2d_1 (Conv2D)           (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_1 (MaxPooling  (None, 5, 5, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 1600)              0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 121,930\n",
            "Trainable params: 121,930\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.optimizers import SGD \n",
        "sgd = SGD(learning_rate=0.001, momentum=0.0)"
      ],
      "metadata": {
        "id": "TtRZ6rVfdUS9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "history = model1.fit(train_X, train_labels, epochs=10, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cZS3pJy2bxH3",
        "outputId": "d90476b2-c453-4611-b2ae-e0354755a842"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1875/1875 [==============================] - 16s 4ms/step - loss: 2.1509 - accuracy: 0.3941 - val_loss: 1.7631 - val_accuracy: 0.7074\n",
            "Epoch 2/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.9498 - accuracy: 0.7906 - val_loss: 0.5131 - val_accuracy: 0.8688\n",
            "Epoch 3/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.4294 - accuracy: 0.8808 - val_loss: 0.3451 - val_accuracy: 0.9044\n",
            "Epoch 4/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.3248 - accuracy: 0.9079 - val_loss: 0.2746 - val_accuracy: 0.9217\n",
            "Epoch 5/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.2718 - accuracy: 0.9218 - val_loss: 0.2330 - val_accuracy: 0.9329\n",
            "Epoch 6/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.2368 - accuracy: 0.9319 - val_loss: 0.2035 - val_accuracy: 0.9417\n",
            "Epoch 7/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.2111 - accuracy: 0.9390 - val_loss: 0.1850 - val_accuracy: 0.9485\n",
            "Epoch 8/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.1905 - accuracy: 0.9447 - val_loss: 0.1661 - val_accuracy: 0.9525\n",
            "Epoch 9/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.1741 - accuracy: 0.9495 - val_loss: 0.1562 - val_accuracy: 0.9554\n",
            "Epoch 10/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.1609 - accuracy: 0.9534 - val_loss: 0.1441 - val_accuracy: 0.9572\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ebrHgA84h7J6",
        "outputId": "1bff37d3-6017-4f3d-833e-78730b68220f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "313/313 [==============================] - 1s 4ms/step - loss: 0.1583 - accuracy: 0.9540\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I77CgAq0eu1z",
        "outputId": "a61a46aa-e428-4063-b518-f4c66e4195d7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d (Conv2D)             (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d (MaxPooling2D  (None, 13, 13, 32)       0         \n",
            " )                                                               \n",
            "                                                                 \n",
            " conv2d_1 (Conv2D)           (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_1 (MaxPooling  (None, 5, 5, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 1600)              0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 121,930\n",
            "Trainable params: 121,930\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#############model2\n",
        "model1 =Sequential()\n",
        "#first convolution layer\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.001, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "history = model1.fit(train_X, train_labels, epochs=25, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32)\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lCArrjxciVrc",
        "outputId": "3e3f3154-a252-4294-c7f9-04b4fc7719ac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/25\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 2.0716 - accuracy: 0.4557 - val_loss: 1.4673 - val_accuracy: 0.7564\n",
            "Epoch 2/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.7506 - accuracy: 0.8267 - val_loss: 0.4291 - val_accuracy: 0.8834\n",
            "Epoch 3/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.3755 - accuracy: 0.8927 - val_loss: 0.3154 - val_accuracy: 0.9091\n",
            "Epoch 4/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.3016 - accuracy: 0.9117 - val_loss: 0.2641 - val_accuracy: 0.9248\n",
            "Epoch 5/25\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.2623 - accuracy: 0.9231 - val_loss: 0.2322 - val_accuracy: 0.9323\n",
            "Epoch 6/25\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.2340 - accuracy: 0.9300 - val_loss: 0.2097 - val_accuracy: 0.9386\n",
            "Epoch 7/25\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.2117 - accuracy: 0.9375 - val_loss: 0.1925 - val_accuracy: 0.9431\n",
            "Epoch 8/25\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1933 - accuracy: 0.9424 - val_loss: 0.1739 - val_accuracy: 0.9507\n",
            "Epoch 9/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1781 - accuracy: 0.9470 - val_loss: 0.1597 - val_accuracy: 0.9538\n",
            "Epoch 10/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1643 - accuracy: 0.9516 - val_loss: 0.1468 - val_accuracy: 0.9588\n",
            "Epoch 11/25\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.1530 - accuracy: 0.9550 - val_loss: 0.1359 - val_accuracy: 0.9613\n",
            "Epoch 12/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1427 - accuracy: 0.9583 - val_loss: 0.1303 - val_accuracy: 0.9616\n",
            "Epoch 13/25\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.1345 - accuracy: 0.9613 - val_loss: 0.1216 - val_accuracy: 0.9654\n",
            "Epoch 14/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1267 - accuracy: 0.9633 - val_loss: 0.1151 - val_accuracy: 0.9660\n",
            "Epoch 15/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1202 - accuracy: 0.9652 - val_loss: 0.1137 - val_accuracy: 0.9652\n",
            "Epoch 16/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1142 - accuracy: 0.9667 - val_loss: 0.1057 - val_accuracy: 0.9685\n",
            "Epoch 17/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1090 - accuracy: 0.9680 - val_loss: 0.1018 - val_accuracy: 0.9699\n",
            "Epoch 18/25\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.1048 - accuracy: 0.9689 - val_loss: 0.0937 - val_accuracy: 0.9720\n",
            "Epoch 19/25\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.1001 - accuracy: 0.9705 - val_loss: 0.0924 - val_accuracy: 0.9716\n",
            "Epoch 20/25\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0968 - accuracy: 0.9712 - val_loss: 0.0877 - val_accuracy: 0.9745\n",
            "Epoch 21/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0932 - accuracy: 0.9724 - val_loss: 0.0863 - val_accuracy: 0.9742\n",
            "Epoch 22/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0902 - accuracy: 0.9736 - val_loss: 0.0821 - val_accuracy: 0.9754\n",
            "Epoch 23/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0875 - accuracy: 0.9739 - val_loss: 0.0824 - val_accuracy: 0.9750\n",
            "Epoch 24/25\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0851 - accuracy: 0.9748 - val_loss: 0.0770 - val_accuracy: 0.9769\n",
            "Epoch 25/25\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0827 - accuracy: 0.9752 - val_loss: 0.0789 - val_accuracy: 0.9764\n",
            "313/313 [==============================] - 1s 4ms/step - loss: 0.0789 - accuracy: 0.9764\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uZlIxm9gjAo5",
        "outputId": "67f7a76c-f132-46cd-817c-9ed5bb9a11e4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_2 (Conv2D)           (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_2 (MaxPooling  (None, 13, 13, 32)       0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_3 (Conv2D)           (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_3 (MaxPooling  (None, 5, 5, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten_1 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 121,930\n",
            "Trainable params: 121,930\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import keras \n",
        "import time\n",
        "class TimeHistory(keras.callbacks.Callback):\n",
        "    def on_train_begin(self, logs={}):\n",
        "        self.times = []\n",
        "\n",
        "    def on_epoch_begin(self, batch, logs={}):\n",
        "        self.epoch_time_start = time.time()\n",
        "\n",
        "    def on_epoch_end(self, batch, logs={}):\n",
        "        self.times.append(time.time() - self.epoch_time_start)"
      ],
      "metadata": {
        "id": "VpeJ-QNdgmtr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from keras import callbacks"
      ],
      "metadata": {
        "id": "s1LA3T9Eg6Ny"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "#first convolution layer\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.001, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zbNEsoCygkTz",
        "outputId": "19291760-6049-49d4-b396-f56bfb90ccf1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "1875/1875 [==============================] - 13s 7ms/step - loss: 1.9742 - accuracy: 0.4747 - val_loss: 1.2076 - val_accuracy: 0.7127\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.7109 - accuracy: 0.8074 - val_loss: 0.4496 - val_accuracy: 0.8745\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.4003 - accuracy: 0.8854 - val_loss: 0.3318 - val_accuracy: 0.9051\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 12s 6ms/step - loss: 0.3158 - accuracy: 0.9082 - val_loss: 0.2719 - val_accuracy: 0.9229\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 12s 6ms/step - loss: 0.2705 - accuracy: 0.9207 - val_loss: 0.2422 - val_accuracy: 0.9303\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.2394 - accuracy: 0.9291 - val_loss: 0.2093 - val_accuracy: 0.9396\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.2158 - accuracy: 0.9369 - val_loss: 0.1879 - val_accuracy: 0.9457\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1953 - accuracy: 0.9430 - val_loss: 0.1703 - val_accuracy: 0.9515\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1791 - accuracy: 0.9476 - val_loss: 0.1582 - val_accuracy: 0.9543\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1650 - accuracy: 0.9523 - val_loss: 0.1459 - val_accuracy: 0.9582\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.1532 - accuracy: 0.9557 - val_loss: 0.1352 - val_accuracy: 0.9612\n",
            "Epoch 12/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1433 - accuracy: 0.9586 - val_loss: 0.1287 - val_accuracy: 0.9637\n",
            "Epoch 13/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1346 - accuracy: 0.9611 - val_loss: 0.1167 - val_accuracy: 0.9657\n",
            "Epoch 14/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1270 - accuracy: 0.9624 - val_loss: 0.1154 - val_accuracy: 0.9663\n",
            "Epoch 15/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1207 - accuracy: 0.9648 - val_loss: 0.1045 - val_accuracy: 0.9698\n",
            "Epoch 16/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1147 - accuracy: 0.9657 - val_loss: 0.1012 - val_accuracy: 0.9701\n",
            "Epoch 17/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1096 - accuracy: 0.9681 - val_loss: 0.0998 - val_accuracy: 0.9697\n",
            "Epoch 18/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1053 - accuracy: 0.9683 - val_loss: 0.0981 - val_accuracy: 0.9715\n",
            "Epoch 19/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.1010 - accuracy: 0.9704 - val_loss: 0.0889 - val_accuracy: 0.9736\n",
            "Epoch 20/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0973 - accuracy: 0.9706 - val_loss: 0.0855 - val_accuracy: 0.9739\n",
            "Epoch 21/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0942 - accuracy: 0.9720 - val_loss: 0.0841 - val_accuracy: 0.9741\n",
            "Epoch 22/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0910 - accuracy: 0.9731 - val_loss: 0.0800 - val_accuracy: 0.9751\n",
            "Epoch 23/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0883 - accuracy: 0.9738 - val_loss: 0.0824 - val_accuracy: 0.9744\n",
            "Epoch 24/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0854 - accuracy: 0.9745 - val_loss: 0.0756 - val_accuracy: 0.9770\n",
            "Epoch 25/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0830 - accuracy: 0.9754 - val_loss: 0.0755 - val_accuracy: 0.9769\n",
            "Epoch 26/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0811 - accuracy: 0.9753 - val_loss: 0.0722 - val_accuracy: 0.9775\n",
            "Epoch 27/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0787 - accuracy: 0.9763 - val_loss: 0.0728 - val_accuracy: 0.9765\n",
            "Epoch 28/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0768 - accuracy: 0.9768 - val_loss: 0.0703 - val_accuracy: 0.9775\n",
            "Epoch 29/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0750 - accuracy: 0.9775 - val_loss: 0.0693 - val_accuracy: 0.9785\n",
            "Epoch 30/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0732 - accuracy: 0.9784 - val_loss: 0.0673 - val_accuracy: 0.9803\n",
            "Epoch 31/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0716 - accuracy: 0.9783 - val_loss: 0.0655 - val_accuracy: 0.9798\n",
            "Epoch 32/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0703 - accuracy: 0.9792 - val_loss: 0.0672 - val_accuracy: 0.9799\n",
            "Epoch 33/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0688 - accuracy: 0.9794 - val_loss: 0.0632 - val_accuracy: 0.9808\n",
            "Epoch 34/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0677 - accuracy: 0.9794 - val_loss: 0.0643 - val_accuracy: 0.9806\n",
            "Epoch 35/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0660 - accuracy: 0.9799 - val_loss: 0.0603 - val_accuracy: 0.9812\n",
            "Epoch 36/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0648 - accuracy: 0.9807 - val_loss: 0.0626 - val_accuracy: 0.9799\n",
            "Epoch 37/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0633 - accuracy: 0.9814 - val_loss: 0.0603 - val_accuracy: 0.9810\n",
            "Epoch 38/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0626 - accuracy: 0.9810 - val_loss: 0.0605 - val_accuracy: 0.9809\n",
            "Epoch 39/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0611 - accuracy: 0.9815 - val_loss: 0.0603 - val_accuracy: 0.9816\n",
            "Epoch 40/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0602 - accuracy: 0.9818 - val_loss: 0.0565 - val_accuracy: 0.9827\n",
            "313/313 [==============================] - 1s 4ms/step - loss: 0.0565 - accuracy: 0.9827\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nwAyFepNhpl3",
        "outputId": "7003b821-7584-4b7b-ff91-a6abb65c2762"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_4 (Conv2D)           (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_4 (MaxPooling  (None, 13, 13, 32)       0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_5 (Conv2D)           (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_5 (MaxPooling  (None, 5, 5, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten_2 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 121,930\n",
            "Trainable params: 121,930\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "[13.28525161743164, 10.632070064544678, 10.308592081069946, 11.849564552307129, 12.000101089477539, 10.950177907943726, 10.19032907485962, 9.944242477416992, 10.437545776367188, 10.010447025299072, 10.574686527252197, 10.394061803817749, 10.083669185638428, 9.943841934204102, 10.371716260910034, 10.339104890823364, 10.3578941822052, 10.079188823699951, 10.77367901802063, 10.375660419464111, 10.312971591949463, 10.345721960067749, 10.087323665618896, 10.092099905014038, 10.476072311401367, 10.338266134262085, 10.10399079322815, 10.353189706802368, 10.408555746078491, 10.375573873519897, 10.026330709457397, 9.979304075241089, 10.386029243469238, 10.372284412384033, 10.53375768661499, 10.353919267654419, 10.202246904373169, 10.498692274093628, 10.572432041168213, 10.520873308181763]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "TESTING LEARNING RATES"
      ],
      "metadata": {
        "id": "NiY2oMrVhxpT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.01\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.01, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FPt2TmN-jbvN",
        "outputId": "045bb307-558e-4471-8ce9-d382f451bc75"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "   1/1875 [..............................] - ETA: 12:36 - loss: 2.3184 - accuracy: 0.1250WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0027s vs `on_train_batch_end` time: 0.0028s). Check your callbacks.\n",
            "1875/1875 [==============================] - 11s 5ms/step - loss: 0.5620 - accuracy: 0.8344 - val_loss: 0.1737 - val_accuracy: 0.9492\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1492 - accuracy: 0.9554 - val_loss: 0.1009 - val_accuracy: 0.9699\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1030 - accuracy: 0.9688 - val_loss: 0.0723 - val_accuracy: 0.9765\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0827 - accuracy: 0.9742 - val_loss: 0.0676 - val_accuracy: 0.9791\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0701 - accuracy: 0.9786 - val_loss: 0.0528 - val_accuracy: 0.9831\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0615 - accuracy: 0.9814 - val_loss: 0.0508 - val_accuracy: 0.9828\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0552 - accuracy: 0.9833 - val_loss: 0.0465 - val_accuracy: 0.9845\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0500 - accuracy: 0.9845 - val_loss: 0.0450 - val_accuracy: 0.9859\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0457 - accuracy: 0.9859 - val_loss: 0.0418 - val_accuracy: 0.9858\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0426 - accuracy: 0.9866 - val_loss: 0.0409 - val_accuracy: 0.9867\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0392 - accuracy: 0.9880 - val_loss: 0.0380 - val_accuracy: 0.9883\n",
            "Epoch 12/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0365 - accuracy: 0.9887 - val_loss: 0.0365 - val_accuracy: 0.9889\n",
            "Epoch 13/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0346 - accuracy: 0.9888 - val_loss: 0.0350 - val_accuracy: 0.9887\n",
            "Epoch 14/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0325 - accuracy: 0.9902 - val_loss: 0.0348 - val_accuracy: 0.9895\n",
            "Epoch 15/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0308 - accuracy: 0.9905 - val_loss: 0.0329 - val_accuracy: 0.9886\n",
            "Epoch 16/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0287 - accuracy: 0.9911 - val_loss: 0.0345 - val_accuracy: 0.9892\n",
            "Epoch 17/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0267 - accuracy: 0.9917 - val_loss: 0.0328 - val_accuracy: 0.9893\n",
            "Epoch 18/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0253 - accuracy: 0.9920 - val_loss: 0.0331 - val_accuracy: 0.9887\n",
            "Epoch 19/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0242 - accuracy: 0.9925 - val_loss: 0.0364 - val_accuracy: 0.9882\n",
            "Epoch 20/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0231 - accuracy: 0.9928 - val_loss: 0.0323 - val_accuracy: 0.9896\n",
            "Epoch 21/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0218 - accuracy: 0.9933 - val_loss: 0.0315 - val_accuracy: 0.9893\n",
            "Epoch 22/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0204 - accuracy: 0.9937 - val_loss: 0.0334 - val_accuracy: 0.9894\n",
            "Epoch 23/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0198 - accuracy: 0.9938 - val_loss: 0.0331 - val_accuracy: 0.9892\n",
            "Epoch 24/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0184 - accuracy: 0.9941 - val_loss: 0.0346 - val_accuracy: 0.9883\n",
            "Epoch 25/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0172 - accuracy: 0.9947 - val_loss: 0.0366 - val_accuracy: 0.9892\n",
            "Epoch 26/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0170 - accuracy: 0.9948 - val_loss: 0.0354 - val_accuracy: 0.9894\n",
            "313/313 [==============================] - 1s 4ms/step - loss: 0.0315 - accuracy: 0.9893\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "etkiq0hBhhWS",
        "outputId": "990cfacf-2f0c-486a-acf5-0e1d2c79f37b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_6 (Conv2D)           (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_6 (MaxPooling  (None, 13, 13, 32)       0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_7 (Conv2D)           (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_7 (MaxPooling  (None, 5, 5, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten_3 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_7 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 121,930\n",
            "Trainable params: 121,930\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 4ms/step - loss: 0.0315 - accuracy: 0.9893\n",
            "[10.66630744934082, 10.488824844360352, 10.095211267471313, 10.666344404220581, 10.631511688232422, 10.509984016418457, 10.522321939468384, 10.4005765914917, 10.063866376876831, 10.131177425384521, 10.495679378509521, 10.164026021957397, 10.49898624420166, 10.532265424728394, 10.149816751480103, 10.598557233810425, 10.082297086715698, 10.162006616592407, 10.152749061584473, 10.657643795013428, 10.105109691619873, 10.536373138427734, 10.449999809265137, 10.120105981826782, 10.465376615524292, 10.19348406791687]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZujR94KNj6ss",
        "outputId": "0a0ea591-1314-481d-8e1f-0ff92ab63b90"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.1779 - accuracy: 0.9438 - val_loss: 0.0575 - val_accuracy: 0.9803\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0553 - accuracy: 0.9830 - val_loss: 0.0391 - val_accuracy: 0.9871\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0384 - accuracy: 0.9879 - val_loss: 0.0394 - val_accuracy: 0.9864\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0296 - accuracy: 0.9911 - val_loss: 0.0359 - val_accuracy: 0.9883\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0237 - accuracy: 0.9924 - val_loss: 0.0302 - val_accuracy: 0.9907\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0185 - accuracy: 0.9942 - val_loss: 0.0299 - val_accuracy: 0.9907\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0147 - accuracy: 0.9955 - val_loss: 0.0306 - val_accuracy: 0.9904\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0124 - accuracy: 0.9959 - val_loss: 0.0342 - val_accuracy: 0.9900\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0106 - accuracy: 0.9966 - val_loss: 0.0308 - val_accuracy: 0.9907\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0079 - accuracy: 0.9977 - val_loss: 0.0346 - val_accuracy: 0.9904\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0060 - accuracy: 0.9982 - val_loss: 0.0337 - val_accuracy: 0.9912\n",
            "313/313 [==============================] - 1s 4ms/step - loss: 0.0299 - accuracy: 0.9907\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dpO9g3X5maDR",
        "outputId": "d27b073c-2333-4ab6-8a4c-21280746e449"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_4\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_8 (Conv2D)           (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_8 (MaxPooling  (None, 13, 13, 32)       0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_9 (Conv2D)           (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_9 (MaxPooling  (None, 5, 5, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten_4 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_8 (Dense)             (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_9 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 121,930\n",
            "Trainable params: 121,930\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 4ms/step - loss: 0.0299 - accuracy: 0.9907\n",
            "[10.947302103042603, 10.109866380691528, 11.027601480484009, 10.799046277999878, 10.092986822128296, 10.398900985717773, 10.477643251419067, 10.040547370910645, 10.456888198852539, 10.107140302658081, 10.492445945739746]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.003\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.003, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lva3df6Dn_x3",
        "outputId": "49348dc3-6f0d-4394-acac-8ebf474cb01f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "1875/1875 [==============================] - 11s 5ms/step - loss: 1.1887 - accuracy: 0.6259 - val_loss: 0.3600 - val_accuracy: 0.8965\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.3083 - accuracy: 0.9075 - val_loss: 0.2367 - val_accuracy: 0.9315\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.2218 - accuracy: 0.9345 - val_loss: 0.1715 - val_accuracy: 0.9505\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1720 - accuracy: 0.9491 - val_loss: 0.1393 - val_accuracy: 0.9578\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1413 - accuracy: 0.9584 - val_loss: 0.1137 - val_accuracy: 0.9679\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.1220 - accuracy: 0.9641 - val_loss: 0.1031 - val_accuracy: 0.9693\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.1073 - accuracy: 0.9677 - val_loss: 0.1018 - val_accuracy: 0.9706\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0970 - accuracy: 0.9708 - val_loss: 0.0809 - val_accuracy: 0.9752\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0889 - accuracy: 0.9732 - val_loss: 0.0763 - val_accuracy: 0.9754\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0824 - accuracy: 0.9754 - val_loss: 0.0677 - val_accuracy: 0.9787\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0765 - accuracy: 0.9770 - val_loss: 0.0663 - val_accuracy: 0.9784\n",
            "Epoch 12/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0721 - accuracy: 0.9780 - val_loss: 0.0658 - val_accuracy: 0.9784\n",
            "Epoch 13/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0685 - accuracy: 0.9797 - val_loss: 0.0581 - val_accuracy: 0.9819\n",
            "Epoch 14/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0648 - accuracy: 0.9804 - val_loss: 0.0559 - val_accuracy: 0.9818\n",
            "Epoch 15/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0618 - accuracy: 0.9811 - val_loss: 0.0595 - val_accuracy: 0.9811\n",
            "Epoch 16/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0590 - accuracy: 0.9820 - val_loss: 0.0563 - val_accuracy: 0.9815\n",
            "Epoch 17/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0563 - accuracy: 0.9830 - val_loss: 0.0548 - val_accuracy: 0.9818\n",
            "Epoch 18/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0540 - accuracy: 0.9836 - val_loss: 0.0519 - val_accuracy: 0.9825\n",
            "Epoch 19/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0519 - accuracy: 0.9845 - val_loss: 0.0541 - val_accuracy: 0.9821\n",
            "Epoch 20/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0504 - accuracy: 0.9848 - val_loss: 0.0486 - val_accuracy: 0.9848\n",
            "Epoch 21/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0491 - accuracy: 0.9852 - val_loss: 0.0482 - val_accuracy: 0.9840\n",
            "Epoch 22/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0469 - accuracy: 0.9855 - val_loss: 0.0475 - val_accuracy: 0.9852\n",
            "Epoch 23/40\n",
            "1875/1875 [==============================] - 13s 7ms/step - loss: 0.0462 - accuracy: 0.9862 - val_loss: 0.0453 - val_accuracy: 0.9856\n",
            "Epoch 24/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0441 - accuracy: 0.9868 - val_loss: 0.0432 - val_accuracy: 0.9863\n",
            "Epoch 25/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0425 - accuracy: 0.9872 - val_loss: 0.0411 - val_accuracy: 0.9860\n",
            "Epoch 26/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0419 - accuracy: 0.9871 - val_loss: 0.0425 - val_accuracy: 0.9848\n",
            "Epoch 27/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0400 - accuracy: 0.9882 - val_loss: 0.0415 - val_accuracy: 0.9864\n",
            "Epoch 28/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0392 - accuracy: 0.9883 - val_loss: 0.0411 - val_accuracy: 0.9865\n",
            "Epoch 29/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0381 - accuracy: 0.9886 - val_loss: 0.0415 - val_accuracy: 0.9856\n",
            "Epoch 30/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0369 - accuracy: 0.9891 - val_loss: 0.0388 - val_accuracy: 0.9874\n",
            "Epoch 31/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0359 - accuracy: 0.9893 - val_loss: 0.0367 - val_accuracy: 0.9876\n",
            "Epoch 32/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0354 - accuracy: 0.9892 - val_loss: 0.0393 - val_accuracy: 0.9875\n",
            "Epoch 33/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0343 - accuracy: 0.9897 - val_loss: 0.0368 - val_accuracy: 0.9872\n",
            "Epoch 34/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0331 - accuracy: 0.9905 - val_loss: 0.0437 - val_accuracy: 0.9860\n",
            "Epoch 35/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0331 - accuracy: 0.9905 - val_loss: 0.0379 - val_accuracy: 0.9878\n",
            "Epoch 36/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0322 - accuracy: 0.9905 - val_loss: 0.0357 - val_accuracy: 0.9881\n",
            "Epoch 37/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0311 - accuracy: 0.9908 - val_loss: 0.0394 - val_accuracy: 0.9858\n",
            "Epoch 38/40\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0305 - accuracy: 0.9911 - val_loss: 0.0371 - val_accuracy: 0.9874\n",
            "Epoch 39/40\n",
            "1875/1875 [==============================] - 10s 6ms/step - loss: 0.0296 - accuracy: 0.9912 - val_loss: 0.0406 - val_accuracy: 0.9868\n",
            "Epoch 40/40\n",
            "1875/1875 [==============================] - 11s 6ms/step - loss: 0.0292 - accuracy: 0.9913 - val_loss: 0.0368 - val_accuracy: 0.9881\n",
            "313/313 [==============================] - 1s 4ms/step - loss: 0.0368 - accuracy: 0.9881\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5ab18wUsoTJW",
        "outputId": "8f3061f3-18a1-4dbe-b61a-388ba8dcd364"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_5\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_10 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_10 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_11 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_11 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_5 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_10 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_11 (Dense)            (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 121,930\n",
            "Trainable params: 121,930\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 4ms/step - loss: 0.0368 - accuracy: 0.9881\n",
            "[10.648336410522461, 10.470914840698242, 10.317973613739014, 10.250239372253418, 10.331140041351318, 10.368586778640747, 9.954809427261353, 10.307901382446289, 10.501755475997925, 9.986394882202148, 10.352505922317505, 10.800374746322632, 10.25293779373169, 10.060513973236084, 10.007049322128296, 10.453199863433838, 10.130411148071289, 10.090394258499146, 9.967403888702393, 10.494374513626099, 9.993850231170654, 10.417911529541016, 12.686509132385254, 10.030638933181763, 9.98386836051941, 10.323788404464722, 9.921845197677612, 10.339212656021118, 9.969069719314575, 9.973387718200684, 10.396514892578125, 10.425524234771729, 10.17065167427063, 10.054506063461304, 10.036363363265991, 10.488706111907959, 10.535224199295044, 10.08469271659851, 10.498995542526245, 10.511680364608765]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.005\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.005, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HLghtSzZP86c",
        "outputId": "af5c7b09-c318-492a-d302-b89325439fab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.7580 - accuracy: 0.7807 - val_loss: 0.3076 - val_accuracy: 0.9096\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.2151 - accuracy: 0.9357 - val_loss: 0.1513 - val_accuracy: 0.9544\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.1489 - accuracy: 0.9556 - val_loss: 0.1075 - val_accuracy: 0.9667\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.1177 - accuracy: 0.9650 - val_loss: 0.0930 - val_accuracy: 0.9723\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0997 - accuracy: 0.9708 - val_loss: 0.0787 - val_accuracy: 0.9759\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0880 - accuracy: 0.9736 - val_loss: 0.0687 - val_accuracy: 0.9785\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0795 - accuracy: 0.9760 - val_loss: 0.0694 - val_accuracy: 0.9780\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0727 - accuracy: 0.9779 - val_loss: 0.0680 - val_accuracy: 0.9778\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0670 - accuracy: 0.9797 - val_loss: 0.0588 - val_accuracy: 0.9819\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0624 - accuracy: 0.9810 - val_loss: 0.0538 - val_accuracy: 0.9831\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0583 - accuracy: 0.9824 - val_loss: 0.0553 - val_accuracy: 0.9824\n",
            "Epoch 12/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0552 - accuracy: 0.9831 - val_loss: 0.0494 - val_accuracy: 0.9836\n",
            "Epoch 13/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0523 - accuracy: 0.9838 - val_loss: 0.0484 - val_accuracy: 0.9849\n",
            "Epoch 14/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0492 - accuracy: 0.9852 - val_loss: 0.0437 - val_accuracy: 0.9865\n",
            "Epoch 15/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0473 - accuracy: 0.9859 - val_loss: 0.0451 - val_accuracy: 0.9849\n",
            "Epoch 16/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0441 - accuracy: 0.9868 - val_loss: 0.0433 - val_accuracy: 0.9858\n",
            "Epoch 17/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0432 - accuracy: 0.9871 - val_loss: 0.0399 - val_accuracy: 0.9868\n",
            "Epoch 18/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0411 - accuracy: 0.9874 - val_loss: 0.0439 - val_accuracy: 0.9858\n",
            "Epoch 19/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0395 - accuracy: 0.9881 - val_loss: 0.0388 - val_accuracy: 0.9882\n",
            "Epoch 20/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0379 - accuracy: 0.9886 - val_loss: 0.0382 - val_accuracy: 0.9880\n",
            "Epoch 21/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0367 - accuracy: 0.9885 - val_loss: 0.0408 - val_accuracy: 0.9868\n",
            "Epoch 22/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0354 - accuracy: 0.9894 - val_loss: 0.0423 - val_accuracy: 0.9862\n",
            "Epoch 23/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0343 - accuracy: 0.9895 - val_loss: 0.0365 - val_accuracy: 0.9877\n",
            "Epoch 24/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0330 - accuracy: 0.9902 - val_loss: 0.0397 - val_accuracy: 0.9861\n",
            "Epoch 25/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0319 - accuracy: 0.9901 - val_loss: 0.0389 - val_accuracy: 0.9865\n",
            "Epoch 26/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0307 - accuracy: 0.9904 - val_loss: 0.0361 - val_accuracy: 0.9879\n",
            "Epoch 27/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0299 - accuracy: 0.9910 - val_loss: 0.0412 - val_accuracy: 0.9862\n",
            "Epoch 28/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0288 - accuracy: 0.9916 - val_loss: 0.0370 - val_accuracy: 0.9880\n",
            "Epoch 29/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0279 - accuracy: 0.9914 - val_loss: 0.0389 - val_accuracy: 0.9876\n",
            "Epoch 30/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0273 - accuracy: 0.9917 - val_loss: 0.0339 - val_accuracy: 0.9893\n",
            "Epoch 31/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0260 - accuracy: 0.9922 - val_loss: 0.0391 - val_accuracy: 0.9873\n",
            "Epoch 32/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0257 - accuracy: 0.9925 - val_loss: 0.0348 - val_accuracy: 0.9889\n",
            "Epoch 33/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0246 - accuracy: 0.9929 - val_loss: 0.0332 - val_accuracy: 0.9887\n",
            "Epoch 34/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0238 - accuracy: 0.9930 - val_loss: 0.0340 - val_accuracy: 0.9894\n",
            "Epoch 35/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0231 - accuracy: 0.9931 - val_loss: 0.0357 - val_accuracy: 0.9887\n",
            "Epoch 36/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0222 - accuracy: 0.9934 - val_loss: 0.0381 - val_accuracy: 0.9878\n",
            "Epoch 37/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0216 - accuracy: 0.9941 - val_loss: 0.0357 - val_accuracy: 0.9893\n",
            "Epoch 38/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0208 - accuracy: 0.9938 - val_loss: 0.0312 - val_accuracy: 0.9897\n",
            "Epoch 39/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0204 - accuracy: 0.9940 - val_loss: 0.0323 - val_accuracy: 0.9892\n",
            "Epoch 40/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0200 - accuracy: 0.9941 - val_loss: 0.0326 - val_accuracy: 0.9890\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0326 - accuracy: 0.9890\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0jQE-fhMP9HP",
        "outputId": "dc7b3527-5eac-49b6-a968-a8db7133bd62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_18\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_39 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_39 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_40 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_40 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_18 (Flatten)        (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_51 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_52 (Dense)            (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 121,930\n",
            "Trainable params: 121,930\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0326 - accuracy: 0.9890\n",
            "[6.4069600105285645, 5.524645805358887, 6.771639585494995, 5.5406951904296875, 5.589333772659302, 5.54652214050293, 5.541147708892822, 5.558661460876465, 5.595424175262451, 5.544501066207886, 5.569594860076904, 5.555826902389526, 5.561650991439819, 5.577897071838379, 5.6011059284210205, 5.523564100265503, 5.623640537261963, 5.718517303466797, 5.6122376918792725, 5.6008710861206055, 5.5956079959869385, 5.583516597747803, 5.628449440002441, 5.507747650146484, 5.574894428253174, 5.632693290710449, 5.52852988243103, 5.612900495529175, 5.74003005027771, 5.595170736312866, 5.553348779678345, 5.582526206970215, 5.595680236816406, 5.547945022583008, 5.68924880027771, 5.60068154335022, 5.749489784240723, 5.590587139129639, 5.681508302688599, 5.658182621002197]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "TESTING DIFFERNT ARCHITECTURES"
      ],
      "metadata": {
        "id": "mKJuNQQGo_pL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "we will use 40 epoches with 0.1 learning rate"
      ],
      "metadata": {
        "id": "3A_I4I0NqfG8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#1CNN AND 1 FC\n",
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9qH3acYkopM-",
        "outputId": "4600ddf7-b9e3-4e50-cbf3-f14d1164013a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "1875/1875 [==============================] - 8s 4ms/step - loss: 0.2722 - accuracy: 0.9195 - val_loss: 0.1197 - val_accuracy: 0.9660\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0975 - accuracy: 0.9725 - val_loss: 0.0758 - val_accuracy: 0.9757\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0733 - accuracy: 0.9784 - val_loss: 0.0609 - val_accuracy: 0.9810\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0611 - accuracy: 0.9822 - val_loss: 0.0622 - val_accuracy: 0.9800\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 8s 4ms/step - loss: 0.0536 - accuracy: 0.9838 - val_loss: 0.0590 - val_accuracy: 0.9806\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 8s 4ms/step - loss: 0.0478 - accuracy: 0.9858 - val_loss: 0.0518 - val_accuracy: 0.9824\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 8s 4ms/step - loss: 0.0434 - accuracy: 0.9873 - val_loss: 0.0659 - val_accuracy: 0.9786\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0395 - accuracy: 0.9880 - val_loss: 0.0483 - val_accuracy: 0.9845\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 8s 4ms/step - loss: 0.0364 - accuracy: 0.9892 - val_loss: 0.0547 - val_accuracy: 0.9834\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0335 - accuracy: 0.9902 - val_loss: 0.0535 - val_accuracy: 0.9828\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 8s 4ms/step - loss: 0.0305 - accuracy: 0.9907 - val_loss: 0.0636 - val_accuracy: 0.9798\n",
            "Epoch 12/40\n",
            "1875/1875 [==============================] - 8s 4ms/step - loss: 0.0286 - accuracy: 0.9914 - val_loss: 0.0523 - val_accuracy: 0.9844\n",
            "Epoch 13/40\n",
            "1875/1875 [==============================] - 8s 4ms/step - loss: 0.0264 - accuracy: 0.9921 - val_loss: 0.0492 - val_accuracy: 0.9841\n",
            "313/313 [==============================] - 1s 3ms/step - loss: 0.0483 - accuracy: 0.9845\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e7NUsQO8opXv",
        "outputId": "2a7b5cc8-7985-4c27-ac28-2c111dbfccd7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_6\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_12 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_12 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_6 (Flatten)         (None, 5408)              0         \n",
            "                                                                 \n",
            " dense_12 (Dense)            (None, 10)                54090     \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 54,410\n",
            "Trainable params: 54,410\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 3ms/step - loss: 0.0483 - accuracy: 0.9845\n",
            "[8.484246253967285, 7.416532516479492, 7.487035036087036, 7.415877103805542, 7.5231146812438965, 7.9372618198394775, 7.915063381195068, 7.440031051635742, 7.917072534561157, 7.481266021728516, 7.968287706375122, 7.526801347732544, 8.035681009292603]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Klp0sGgpqawJ",
        "outputId": "bdfc6b88-8abc-41a6-efed-955cfee674ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.2088 - accuracy: 0.9340 - val_loss: 0.0618 - val_accuracy: 0.9802\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0580 - accuracy: 0.9822 - val_loss: 0.0420 - val_accuracy: 0.9871\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0390 - accuracy: 0.9882 - val_loss: 0.0343 - val_accuracy: 0.9895\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0295 - accuracy: 0.9906 - val_loss: 0.0355 - val_accuracy: 0.9886\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0243 - accuracy: 0.9922 - val_loss: 0.0297 - val_accuracy: 0.9898\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0185 - accuracy: 0.9940 - val_loss: 0.0315 - val_accuracy: 0.9915\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0142 - accuracy: 0.9957 - val_loss: 0.0317 - val_accuracy: 0.9905\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0123 - accuracy: 0.9959 - val_loss: 0.0294 - val_accuracy: 0.9907\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0087 - accuracy: 0.9976 - val_loss: 0.0294 - val_accuracy: 0.9918\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0087 - accuracy: 0.9971 - val_loss: 0.0305 - val_accuracy: 0.9914\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0070 - accuracy: 0.9977 - val_loss: 0.0424 - val_accuracy: 0.9897\n",
            "Epoch 12/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0058 - accuracy: 0.9981 - val_loss: 0.0352 - val_accuracy: 0.9914\n",
            "Epoch 13/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0061 - accuracy: 0.9979 - val_loss: 0.0362 - val_accuracy: 0.9904\n",
            "Epoch 14/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0048 - accuracy: 0.9984 - val_loss: 0.0377 - val_accuracy: 0.9912\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0294 - accuracy: 0.9918\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dH2K6GToopZS",
        "outputId": "92d91ced-dd84-4bf0-f4e7-f27df62f0800"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_4\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_11 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_11 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_12 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_12 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_4 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_10 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_11 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_12 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0294 - accuracy: 0.9918\n",
            "[6.328695058822632, 5.764863014221191, 5.6650331020355225, 5.74968147277832, 5.780593156814575, 5.768248081207275, 5.766294717788696, 5.742184400558472, 5.763494253158569, 5.757632493972778, 5.755081653594971, 5.807462215423584, 5.757145643234253, 5.727740287780762]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 , trying 3CNN and 3FC\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q8F90ezuopcu",
        "outputId": "1cff14f8-574c-44a8-e631-07839d4bbc36"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.3402 - accuracy: 0.8899 - val_loss: 0.0895 - val_accuracy: 0.9743\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0864 - accuracy: 0.9741 - val_loss: 0.0806 - val_accuracy: 0.9752\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0609 - accuracy: 0.9813 - val_loss: 0.0530 - val_accuracy: 0.9854\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0492 - accuracy: 0.9849 - val_loss: 0.0613 - val_accuracy: 0.9808\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0412 - accuracy: 0.9872 - val_loss: 0.0562 - val_accuracy: 0.9831\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0341 - accuracy: 0.9889 - val_loss: 0.0488 - val_accuracy: 0.9859\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0294 - accuracy: 0.9908 - val_loss: 0.0828 - val_accuracy: 0.9789\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0251 - accuracy: 0.9919 - val_loss: 0.0589 - val_accuracy: 0.9846\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0213 - accuracy: 0.9929 - val_loss: 0.0510 - val_accuracy: 0.9857\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0205 - accuracy: 0.9936 - val_loss: 0.0530 - val_accuracy: 0.9858\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0188 - accuracy: 0.9937 - val_loss: 0.0572 - val_accuracy: 0.9852\n",
            "313/313 [==============================] - 1s 3ms/step - loss: 0.0488 - accuracy: 0.9859\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dgkFYo4FopeC",
        "outputId": "fcc85bc6-50a3-4169-eadb-b17b985e1a85"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_5 (Conv2D)           (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_5 (MaxPooling  (None, 13, 13, 32)       0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_6 (Conv2D)           (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_6 (MaxPooling  (None, 5, 5, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_7 (Conv2D)           (None, 3, 3, 32)          18464     \n",
            "                                                                 \n",
            " max_pooling2d_7 (MaxPooling  (None, 1, 1, 32)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten_2 (Flatten)         (None, 32)                0         \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 64)                2112      \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_7 (Dense)             (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 41,802\n",
            "Trainable params: 41,802\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 3ms/step - loss: 0.0488 - accuracy: 0.9859\n",
            "[6.6867616176605225, 6.2012739181518555, 6.1085498332977295, 6.056229114532471, 6.767364740371704, 6.054199695587158, 6.107012987136841, 6.086791038513184, 6.102833032608032, 6.042593002319336, 6.016862630844116]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 , trying 3CNN and 2FC\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 32 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "paPBRAIaophM",
        "outputId": "0a81a37e-d1fd-4add-e647-e3e90a9eea7c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "1875/1875 [==============================] - 9s 5ms/step - loss: 0.3089 - accuracy: 0.9020 - val_loss: 0.1018 - val_accuracy: 0.9687\n",
            "Epoch 2/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0844 - accuracy: 0.9740 - val_loss: 0.0607 - val_accuracy: 0.9811\n",
            "Epoch 3/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0614 - accuracy: 0.9814 - val_loss: 0.0586 - val_accuracy: 0.9834\n",
            "Epoch 4/40\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.0483 - accuracy: 0.9850 - val_loss: 0.0603 - val_accuracy: 0.9841\n",
            "Epoch 5/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0414 - accuracy: 0.9872 - val_loss: 0.0607 - val_accuracy: 0.9832\n",
            "Epoch 6/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0336 - accuracy: 0.9895 - val_loss: 0.0651 - val_accuracy: 0.9799\n",
            "Epoch 7/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0299 - accuracy: 0.9904 - val_loss: 0.0489 - val_accuracy: 0.9858\n",
            "Epoch 8/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0250 - accuracy: 0.9921 - val_loss: 0.0617 - val_accuracy: 0.9831\n",
            "Epoch 9/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0234 - accuracy: 0.9925 - val_loss: 0.0585 - val_accuracy: 0.9839\n",
            "Epoch 10/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0205 - accuracy: 0.9932 - val_loss: 0.0562 - val_accuracy: 0.9847\n",
            "Epoch 11/40\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.0176 - accuracy: 0.9942 - val_loss: 0.0625 - val_accuracy: 0.9832\n",
            "Epoch 12/40\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.0154 - accuracy: 0.9948 - val_loss: 0.0542 - val_accuracy: 0.9868\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0489 - accuracy: 0.9858\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "63rsEO1Lopi2",
        "outputId": "8c61abd7-339f-49f3-8107-f2f5dd653543"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_8 (Conv2D)           (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_8 (MaxPooling  (None, 13, 13, 32)       0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_9 (Conv2D)           (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_9 (MaxPooling  (None, 5, 5, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_10 (Conv2D)          (None, 3, 3, 32)          18464     \n",
            "                                                                 \n",
            " max_pooling2d_10 (MaxPoolin  (None, 1, 1, 32)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_3 (Flatten)         (None, 32)                0         \n",
            "                                                                 \n",
            " dense_8 (Dense)             (None, 32)                1056      \n",
            "                                                                 \n",
            " dense_9 (Dense)             (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 38,666\n",
            "Trainable params: 38,666\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0489 - accuracy: 0.9858\n",
            "[9.056811332702637, 6.00836968421936, 5.844502925872803, 6.525131464004517, 5.908543109893799, 5.901589393615723, 5.83287239074707, 5.832604169845581, 5.852531909942627, 5.828242301940918, 6.560439586639404, 6.220177412033081]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "TRYING DIFFERENT BATCH SIZE "
      ],
      "metadata": {
        "id": "LSNggOtM6Win"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WlPwLprMopm4",
        "outputId": "89d7e557-e9c5-4fcc-d360-343e5404653f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.2903 - accuracy: 0.9075 - val_loss: 0.0708 - val_accuracy: 0.9785\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0686 - accuracy: 0.9784 - val_loss: 0.0447 - val_accuracy: 0.9858\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0479 - accuracy: 0.9852 - val_loss: 0.0413 - val_accuracy: 0.9865\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0379 - accuracy: 0.9879 - val_loss: 0.0328 - val_accuracy: 0.9885\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0299 - accuracy: 0.9905 - val_loss: 0.0310 - val_accuracy: 0.9886\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0245 - accuracy: 0.9926 - val_loss: 0.0382 - val_accuracy: 0.9879\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0204 - accuracy: 0.9937 - val_loss: 0.0425 - val_accuracy: 0.9868\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0176 - accuracy: 0.9944 - val_loss: 0.0417 - val_accuracy: 0.9864\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0150 - accuracy: 0.9952 - val_loss: 0.0289 - val_accuracy: 0.9908\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0118 - accuracy: 0.9962 - val_loss: 0.0364 - val_accuracy: 0.9889\n",
            "Epoch 11/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0105 - accuracy: 0.9968 - val_loss: 0.0345 - val_accuracy: 0.9894\n",
            "Epoch 12/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0081 - accuracy: 0.9977 - val_loss: 0.0304 - val_accuracy: 0.9914\n",
            "Epoch 13/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0078 - accuracy: 0.9976 - val_loss: 0.0278 - val_accuracy: 0.9919\n",
            "Epoch 14/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0066 - accuracy: 0.9979 - val_loss: 0.0327 - val_accuracy: 0.9908\n",
            "Epoch 15/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0060 - accuracy: 0.9981 - val_loss: 0.0313 - val_accuracy: 0.9914\n",
            "Epoch 16/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0042 - accuracy: 0.9988 - val_loss: 0.0345 - val_accuracy: 0.9908\n",
            "Epoch 17/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0035 - accuracy: 0.9990 - val_loss: 0.0406 - val_accuracy: 0.9887\n",
            "Epoch 18/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0028 - accuracy: 0.9993 - val_loss: 0.0336 - val_accuracy: 0.9918\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0278 - accuracy: 0.9919\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oTTIJRdPoppI",
        "outputId": "b9fc49a1-f280-4def-b880-b148461ff2ea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_6\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_15 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_15 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_16 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_16 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_6 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_15 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_16 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_17 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0278 - accuracy: 0.9919\n",
            "[4.016432762145996, 3.4067440032958984, 3.7209880352020264, 3.364039897918701, 3.3387527465820312, 3.359880208969116, 3.801670789718628, 3.6276445388793945, 3.6455094814300537, 3.5915675163269043, 3.55423903465271, 3.357398748397827, 3.322221040725708, 3.3238909244537354, 3.3913235664367676, 3.455738067626953, 3.7189836502075195, 3.399812698364258]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 96\n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 96 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d1bwAEd9ops0",
        "outputId": "b7c249f7-9b81-48ee-cf44-ceb6ce825165"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.3860 - accuracy: 0.8763 - val_loss: 0.1010 - val_accuracy: 0.9678\n",
            "Epoch 2/40\n",
            "625/625 [==============================] - 2s 4ms/step - loss: 0.0862 - accuracy: 0.9729 - val_loss: 0.0884 - val_accuracy: 0.9701\n",
            "Epoch 3/40\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.0609 - accuracy: 0.9807 - val_loss: 0.0547 - val_accuracy: 0.9820\n",
            "Epoch 4/40\n",
            "625/625 [==============================] - 2s 4ms/step - loss: 0.0464 - accuracy: 0.9859 - val_loss: 0.0449 - val_accuracy: 0.9850\n",
            "Epoch 5/40\n",
            "625/625 [==============================] - 2s 3ms/step - loss: 0.0374 - accuracy: 0.9881 - val_loss: 0.0329 - val_accuracy: 0.9894\n",
            "Epoch 6/40\n",
            "625/625 [==============================] - 2s 4ms/step - loss: 0.0319 - accuracy: 0.9900 - val_loss: 0.0473 - val_accuracy: 0.9849\n",
            "Epoch 7/40\n",
            "625/625 [==============================] - 2s 4ms/step - loss: 0.0274 - accuracy: 0.9912 - val_loss: 0.0305 - val_accuracy: 0.9894\n",
            "Epoch 8/40\n",
            "625/625 [==============================] - 2s 4ms/step - loss: 0.0226 - accuracy: 0.9927 - val_loss: 0.0282 - val_accuracy: 0.9903\n",
            "Epoch 9/40\n",
            "625/625 [==============================] - 2s 4ms/step - loss: 0.0194 - accuracy: 0.9937 - val_loss: 0.0296 - val_accuracy: 0.9894\n",
            "Epoch 10/40\n",
            "625/625 [==============================] - 2s 3ms/step - loss: 0.0166 - accuracy: 0.9947 - val_loss: 0.0305 - val_accuracy: 0.9901\n",
            "Epoch 11/40\n",
            "625/625 [==============================] - 2s 3ms/step - loss: 0.0153 - accuracy: 0.9954 - val_loss: 0.0340 - val_accuracy: 0.9887\n",
            "Epoch 12/40\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.0130 - accuracy: 0.9958 - val_loss: 0.0325 - val_accuracy: 0.9885\n",
            "Epoch 13/40\n",
            "625/625 [==============================] - 2s 4ms/step - loss: 0.0110 - accuracy: 0.9967 - val_loss: 0.0343 - val_accuracy: 0.9900\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0282 - accuracy: 0.9903\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8XWYsvJqopuk",
        "outputId": "bb959a3d-a4f5-49e4-835f-9671e9116ef1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_7\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_17 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_17 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_18 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_18 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_7 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_18 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_19 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_20 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0282 - accuracy: 0.9903\n",
            "[2.959050178527832, 2.2978005409240723, 2.601702928543091, 2.311624765396118, 2.168905019760132, 2.2645761966705322, 2.32275128364563, 2.289583683013916, 2.210627317428589, 2.1708009243011475, 2.188220739364624, 2.693422794342041, 2.355470895767212]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "rqceHfXGopyQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "WE WILL GO WITH BATCH SIZE = 64 "
      ],
      "metadata": {
        "id": "skmx7ueg8qOv"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "WE WILL TEST 3 ACTIVATION FUNCTIONS"
      ],
      "metadata": {
        "id": "SDkS4RRI83Ck"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , activation function will be sigmoid \n",
        "model1.add(Conv2D(32, (3, 3), activation='sigmoid', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='sigmoid'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='sigmoid'))\n",
        "model1.add(Dense(32, activation='sigmoid'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XIqLQAjU7mII",
        "outputId": "cacd8d08-ab59-411e-b2a9-83121d778334"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 2.3064 - accuracy: 0.1068 - val_loss: 2.3096 - val_accuracy: 0.1028\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 2.3038 - accuracy: 0.1078 - val_loss: 2.3020 - val_accuracy: 0.1135\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 2.3020 - accuracy: 0.1124 - val_loss: 2.3026 - val_accuracy: 0.1135\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 2.2940 - accuracy: 0.1297 - val_loss: 2.2776 - val_accuracy: 0.2497\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 1.7290 - accuracy: 0.4425 - val_loss: 0.9315 - val_accuracy: 0.7372\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.6303 - accuracy: 0.8290 - val_loss: 0.4413 - val_accuracy: 0.8789\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3635 - accuracy: 0.8970 - val_loss: 0.2925 - val_accuracy: 0.9162\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.2592 - accuracy: 0.9266 - val_loss: 0.2127 - val_accuracy: 0.9407\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1992 - accuracy: 0.9423 - val_loss: 0.1751 - val_accuracy: 0.9512\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1627 - accuracy: 0.9526 - val_loss: 0.1508 - val_accuracy: 0.9540\n",
            "Epoch 11/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1378 - accuracy: 0.9594 - val_loss: 0.1213 - val_accuracy: 0.9637\n",
            "Epoch 12/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1204 - accuracy: 0.9648 - val_loss: 0.1280 - val_accuracy: 0.9628\n",
            "Epoch 13/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1080 - accuracy: 0.9681 - val_loss: 0.1043 - val_accuracy: 0.9687\n",
            "Epoch 14/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0984 - accuracy: 0.9705 - val_loss: 0.0977 - val_accuracy: 0.9689\n",
            "Epoch 15/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0902 - accuracy: 0.9732 - val_loss: 0.0905 - val_accuracy: 0.9722\n",
            "Epoch 16/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0841 - accuracy: 0.9753 - val_loss: 0.0731 - val_accuracy: 0.9785\n",
            "Epoch 17/40\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0785 - accuracy: 0.9767 - val_loss: 0.0807 - val_accuracy: 0.9769\n",
            "Epoch 18/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0738 - accuracy: 0.9779 - val_loss: 0.0787 - val_accuracy: 0.9767\n",
            "Epoch 19/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0694 - accuracy: 0.9798 - val_loss: 0.0663 - val_accuracy: 0.9802\n",
            "Epoch 20/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0662 - accuracy: 0.9800 - val_loss: 0.0641 - val_accuracy: 0.9814\n",
            "Epoch 21/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0633 - accuracy: 0.9809 - val_loss: 0.0620 - val_accuracy: 0.9813\n",
            "Epoch 22/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0600 - accuracy: 0.9820 - val_loss: 0.0579 - val_accuracy: 0.9814\n",
            "Epoch 23/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0572 - accuracy: 0.9827 - val_loss: 0.0540 - val_accuracy: 0.9828\n",
            "Epoch 24/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0547 - accuracy: 0.9839 - val_loss: 0.0544 - val_accuracy: 0.9834\n",
            "Epoch 25/40\n",
            "938/938 [==============================] - 5s 5ms/step - loss: 0.0528 - accuracy: 0.9846 - val_loss: 0.0554 - val_accuracy: 0.9842\n",
            "Epoch 26/40\n",
            "938/938 [==============================] - 5s 5ms/step - loss: 0.0505 - accuracy: 0.9849 - val_loss: 0.0501 - val_accuracy: 0.9857\n",
            "Epoch 27/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0486 - accuracy: 0.9856 - val_loss: 0.0498 - val_accuracy: 0.9843\n",
            "Epoch 28/40\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0468 - accuracy: 0.9862 - val_loss: 0.0512 - val_accuracy: 0.9839\n",
            "Epoch 29/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0452 - accuracy: 0.9865 - val_loss: 0.0480 - val_accuracy: 0.9853\n",
            "Epoch 30/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0434 - accuracy: 0.9869 - val_loss: 0.0470 - val_accuracy: 0.9844\n",
            "Epoch 31/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0419 - accuracy: 0.9874 - val_loss: 0.0460 - val_accuracy: 0.9859\n",
            "Epoch 32/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0406 - accuracy: 0.9881 - val_loss: 0.0461 - val_accuracy: 0.9849\n",
            "Epoch 33/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0396 - accuracy: 0.9888 - val_loss: 0.0444 - val_accuracy: 0.9875\n",
            "Epoch 34/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0381 - accuracy: 0.9887 - val_loss: 0.0469 - val_accuracy: 0.9852\n",
            "Epoch 35/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0371 - accuracy: 0.9889 - val_loss: 0.0425 - val_accuracy: 0.9871\n",
            "Epoch 36/40\n",
            "938/938 [==============================] - 5s 5ms/step - loss: 0.0358 - accuracy: 0.9894 - val_loss: 0.0487 - val_accuracy: 0.9837\n",
            "Epoch 37/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0348 - accuracy: 0.9902 - val_loss: 0.0427 - val_accuracy: 0.9862\n",
            "Epoch 38/40\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0338 - accuracy: 0.9901 - val_loss: 0.0402 - val_accuracy: 0.9877\n",
            "Epoch 39/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0327 - accuracy: 0.9906 - val_loss: 0.0396 - val_accuracy: 0.9880\n",
            "Epoch 40/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0312 - accuracy: 0.9911 - val_loss: 0.0473 - val_accuracy: 0.9848\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0473 - accuracy: 0.9848\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nRSdJK_x7mJO",
        "outputId": "fccc862c-bec6-43e8-d1ea-1e20e68de78d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_8\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_19 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_19 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_20 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_20 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_8 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_21 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_22 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_23 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0473 - accuracy: 0.9848\n",
            "[4.367655992507935, 3.720479726791382, 3.338361978530884, 3.6639647483825684, 3.739867687225342, 3.9349794387817383, 3.557138204574585, 3.3655545711517334, 3.6426117420196533, 3.6802406311035156, 3.6767983436584473, 3.5736868381500244, 3.6751344203948975, 3.640183687210083, 3.734370708465576, 3.7161905765533447, 4.379518747329712, 3.3885579109191895, 3.7565677165985107, 3.66469407081604, 3.3001341819763184, 3.515160322189331, 3.6699423789978027, 4.0349321365356445, 4.885350704193115, 5.046709060668945, 3.7519640922546387, 4.367229223251343, 3.5672640800476074, 3.3498902320861816, 3.328296661376953, 3.40008544921875, 3.343167304992676, 3.4531607627868652, 4.099081993103027, 4.829409837722778, 3.874272584915161, 4.423378944396973, 3.916652202606201, 3.6958553791046143]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , activation function will be tanh\n",
        "model1.add(Conv2D(32, (3, 3), activation='tanh', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='tanh'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='tanh'))\n",
        "model1.add(Dense(32, activation='tanh'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0G-98cQy7mMc",
        "outputId": "49420981-a714-4bf4-daa7-be9599723b7f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.2344 - accuracy: 0.9365 - val_loss: 0.0736 - val_accuracy: 0.9785\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0673 - accuracy: 0.9805 - val_loss: 0.0517 - val_accuracy: 0.9852\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0467 - accuracy: 0.9862 - val_loss: 0.0413 - val_accuracy: 0.9868\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0348 - accuracy: 0.9902 - val_loss: 0.0374 - val_accuracy: 0.9877\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0278 - accuracy: 0.9920 - val_loss: 0.0365 - val_accuracy: 0.9891\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0224 - accuracy: 0.9938 - val_loss: 0.0330 - val_accuracy: 0.9892\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0174 - accuracy: 0.9958 - val_loss: 0.0310 - val_accuracy: 0.9895\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0138 - accuracy: 0.9964 - val_loss: 0.0340 - val_accuracy: 0.9899\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0111 - accuracy: 0.9975 - val_loss: 0.0307 - val_accuracy: 0.9898\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0087 - accuracy: 0.9983 - val_loss: 0.0321 - val_accuracy: 0.9900\n",
            "Epoch 11/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0072 - accuracy: 0.9988 - val_loss: 0.0471 - val_accuracy: 0.9841\n",
            "Epoch 12/40\n",
            "938/938 [==============================] - 5s 5ms/step - loss: 0.0057 - accuracy: 0.9991 - val_loss: 0.0311 - val_accuracy: 0.9902\n",
            "Epoch 13/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0047 - accuracy: 0.9994 - val_loss: 0.0292 - val_accuracy: 0.9906\n",
            "Epoch 14/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0038 - accuracy: 0.9996 - val_loss: 0.0306 - val_accuracy: 0.9903\n",
            "Epoch 15/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0032 - accuracy: 0.9997 - val_loss: 0.0310 - val_accuracy: 0.9905\n",
            "Epoch 16/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0029 - accuracy: 0.9998 - val_loss: 0.0312 - val_accuracy: 0.9904\n",
            "Epoch 17/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0024 - accuracy: 0.9998 - val_loss: 0.0306 - val_accuracy: 0.9904\n",
            "Epoch 18/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0021 - accuracy: 0.9999 - val_loss: 0.0304 - val_accuracy: 0.9905\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0292 - accuracy: 0.9906\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BCoR9JJb7mPZ",
        "outputId": "6d25733f-3ae2-4a23-c380-9b9067242eba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_9\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_21 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_21 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_22 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_22 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_9 (Flatten)         (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_24 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_25 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_26 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0292 - accuracy: 0.9906\n",
            "[4.3000712394714355, 3.316728115081787, 3.315232038497925, 3.4011199474334717, 3.6583423614501953, 3.651688814163208, 3.6783761978149414, 3.6783487796783447, 3.642878293991089, 3.6715195178985596, 3.7345173358917236, 4.851062297821045, 3.7517940998077393, 4.050658464431763, 3.4767770767211914, 3.6623072624206543, 3.714305877685547, 3.3614184856414795]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , activation function will be softplus\n",
        "model1.add(Conv2D(32, (3, 3), activation='softplus', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='softplus'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='softplus'))\n",
        "model1.add(Dense(32, activation='softplus'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= sgd,\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zpo20E4l7mYD",
        "outputId": "c46305b1-3704-48d7-aaaf-e55295225cbf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 2.0021 - accuracy: 0.2448 - val_loss: 0.5331 - val_accuracy: 0.8260\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1802 - accuracy: 0.9446 - val_loss: 0.1132 - val_accuracy: 0.9635\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.1028 - accuracy: 0.9694 - val_loss: 0.0788 - val_accuracy: 0.9753\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0729 - accuracy: 0.9773 - val_loss: 0.0977 - val_accuracy: 0.9674\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0582 - accuracy: 0.9818 - val_loss: 0.0748 - val_accuracy: 0.9755\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0493 - accuracy: 0.9848 - val_loss: 0.0623 - val_accuracy: 0.9799\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0437 - accuracy: 0.9861 - val_loss: 0.0698 - val_accuracy: 0.9794\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0377 - accuracy: 0.9881 - val_loss: 0.0553 - val_accuracy: 0.9819\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0335 - accuracy: 0.9893 - val_loss: 0.0572 - val_accuracy: 0.9827\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0296 - accuracy: 0.9905 - val_loss: 0.0464 - val_accuracy: 0.9863\n",
            "Epoch 11/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0270 - accuracy: 0.9911 - val_loss: 0.0447 - val_accuracy: 0.9868\n",
            "Epoch 12/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0239 - accuracy: 0.9918 - val_loss: 0.0684 - val_accuracy: 0.9815\n",
            "Epoch 13/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0219 - accuracy: 0.9930 - val_loss: 0.0496 - val_accuracy: 0.9852\n",
            "Epoch 14/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0190 - accuracy: 0.9940 - val_loss: 0.0519 - val_accuracy: 0.9849\n",
            "Epoch 15/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0182 - accuracy: 0.9942 - val_loss: 0.0710 - val_accuracy: 0.9794\n",
            "Epoch 16/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0170 - accuracy: 0.9943 - val_loss: 0.0452 - val_accuracy: 0.9877\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0447 - accuracy: 0.9868\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rwpKgwQP7mZh",
        "outputId": "f281ff91-313d-4134-a420-8e0702e49dfc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_10\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_23 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_23 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_24 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_24 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_10 (Flatten)        (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_27 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_28 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_29 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0447 - accuracy: 0.9868\n",
            "[4.391051292419434, 3.716292142868042, 3.4012131690979004, 3.3829143047332764, 3.4249937534332275, 3.7440598011016846, 3.762845993041992, 3.670835494995117, 3.705263137817383, 3.3708441257476807, 3.508181095123291, 3.6963117122650146, 3.4666929244995117, 3.392502546310425, 3.381305456161499, 3.38926362991333]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "we will keep working with relu"
      ],
      "metadata": {
        "id": "hjz9QDOCArGP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "WE WILL TRY 2 OPTIMIZERS "
      ],
      "metadata": {
        "id": "ZMelvmbbAcz_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , # activation function = relu , adam optimizer \n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "\n",
        "\n",
        "model1.compile(optimizer= 'adam',\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yNyUW-_o7mdm",
        "outputId": "c5240c1e-0189-47fa-86dc-4ffe59896865"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 6s 6ms/step - loss: 0.2007 - accuracy: 0.9396 - val_loss: 0.0568 - val_accuracy: 0.9834\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0585 - accuracy: 0.9821 - val_loss: 0.0425 - val_accuracy: 0.9859\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0410 - accuracy: 0.9870 - val_loss: 0.0349 - val_accuracy: 0.9887\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0307 - accuracy: 0.9901 - val_loss: 0.0353 - val_accuracy: 0.9869\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0250 - accuracy: 0.9920 - val_loss: 0.0359 - val_accuracy: 0.9891\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0208 - accuracy: 0.9934 - val_loss: 0.0357 - val_accuracy: 0.9887\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0167 - accuracy: 0.9947 - val_loss: 0.0326 - val_accuracy: 0.9903\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0135 - accuracy: 0.9960 - val_loss: 0.0411 - val_accuracy: 0.9894\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0117 - accuracy: 0.9962 - val_loss: 0.0343 - val_accuracy: 0.9898\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0104 - accuracy: 0.9966 - val_loss: 0.0370 - val_accuracy: 0.9889\n",
            "Epoch 11/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0085 - accuracy: 0.9971 - val_loss: 0.0393 - val_accuracy: 0.9906\n",
            "Epoch 12/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0082 - accuracy: 0.9972 - val_loss: 0.0392 - val_accuracy: 0.9904\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0326 - accuracy: 0.9903\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HibL-K7A_OI-",
        "outputId": "a2cd816e-8bcc-4a4a-e262-ef1addf46f8f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_11\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_25 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_25 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_26 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_26 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_11 (Flatten)        (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_30 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_31 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_32 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 3ms/step - loss: 0.0326 - accuracy: 0.9903\n",
            "[5.872049570083618, 3.538182497024536, 3.7079007625579834, 3.6989691257476807, 3.4229421615600586, 3.741835832595825, 3.7338030338287354, 3.412816286087036, 3.443467378616333, 3.4417223930358887, 3.822333574295044, 3.717013120651245]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , # activation function = relu ,  Adadelta optimizer \n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= 'adadelta',\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XMkpzEsN_OaS",
        "outputId": "d1fa688a-3bc4-4aa9-9d0f-5cfae76788a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 5s 4ms/step - loss: 2.2913 - accuracy: 0.1049 - val_loss: 2.2710 - val_accuracy: 0.1155\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 2.2543 - accuracy: 0.1389 - val_loss: 2.2356 - val_accuracy: 0.1570\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 2.2156 - accuracy: 0.1815 - val_loss: 2.1923 - val_accuracy: 0.2108\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 2.1684 - accuracy: 0.2344 - val_loss: 2.1422 - val_accuracy: 0.2428\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 2.1142 - accuracy: 0.2675 - val_loss: 2.0842 - val_accuracy: 0.2810\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 2.0521 - accuracy: 0.3085 - val_loss: 2.0179 - val_accuracy: 0.3258\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 1.9814 - accuracy: 0.3532 - val_loss: 1.9427 - val_accuracy: 0.3788\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 1.9019 - accuracy: 0.4060 - val_loss: 1.8582 - val_accuracy: 0.4307\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 1.8126 - accuracy: 0.4627 - val_loss: 1.7636 - val_accuracy: 0.4936\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 1.7143 - accuracy: 0.5205 - val_loss: 1.6605 - val_accuracy: 0.5502\n",
            "Epoch 11/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 1.6080 - accuracy: 0.5724 - val_loss: 1.5494 - val_accuracy: 0.6057\n",
            "Epoch 12/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 1.4949 - accuracy: 0.6191 - val_loss: 1.4316 - val_accuracy: 0.6512\n",
            "Epoch 13/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 1.3760 - accuracy: 0.6645 - val_loss: 1.3099 - val_accuracy: 0.6916\n",
            "Epoch 14/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 1.2571 - accuracy: 0.7056 - val_loss: 1.1910 - val_accuracy: 0.7294\n",
            "Epoch 15/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 1.1428 - accuracy: 0.7408 - val_loss: 1.0781 - val_accuracy: 0.7600\n",
            "Epoch 16/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 1.0375 - accuracy: 0.7681 - val_loss: 0.9765 - val_accuracy: 0.7821\n",
            "Epoch 17/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.9436 - accuracy: 0.7894 - val_loss: 0.8865 - val_accuracy: 0.8030\n",
            "Epoch 18/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.8610 - accuracy: 0.8067 - val_loss: 0.8087 - val_accuracy: 0.8164\n",
            "Epoch 19/40\n",
            "938/938 [==============================] - 5s 5ms/step - loss: 0.7901 - accuracy: 0.8191 - val_loss: 0.7415 - val_accuracy: 0.8292\n",
            "Epoch 20/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.7294 - accuracy: 0.8290 - val_loss: 0.6843 - val_accuracy: 0.8386\n",
            "Epoch 21/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.6778 - accuracy: 0.8372 - val_loss: 0.6358 - val_accuracy: 0.8463\n",
            "Epoch 22/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.6337 - accuracy: 0.8450 - val_loss: 0.5942 - val_accuracy: 0.8533\n",
            "Epoch 23/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.5957 - accuracy: 0.8501 - val_loss: 0.5584 - val_accuracy: 0.8594\n",
            "Epoch 24/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.5632 - accuracy: 0.8556 - val_loss: 0.5279 - val_accuracy: 0.8639\n",
            "Epoch 25/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.5353 - accuracy: 0.8605 - val_loss: 0.5015 - val_accuracy: 0.8685\n",
            "Epoch 26/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.5109 - accuracy: 0.8646 - val_loss: 0.4788 - val_accuracy: 0.8727\n",
            "Epoch 27/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.4895 - accuracy: 0.8693 - val_loss: 0.4591 - val_accuracy: 0.8773\n",
            "Epoch 28/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.4705 - accuracy: 0.8725 - val_loss: 0.4408 - val_accuracy: 0.8804\n",
            "Epoch 29/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.4535 - accuracy: 0.8766 - val_loss: 0.4250 - val_accuracy: 0.8836\n",
            "Epoch 30/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.4383 - accuracy: 0.8795 - val_loss: 0.4101 - val_accuracy: 0.8865\n",
            "Epoch 31/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.4245 - accuracy: 0.8826 - val_loss: 0.3973 - val_accuracy: 0.8887\n",
            "Epoch 32/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.4122 - accuracy: 0.8849 - val_loss: 0.3860 - val_accuracy: 0.8910\n",
            "Epoch 33/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.4009 - accuracy: 0.8871 - val_loss: 0.3750 - val_accuracy: 0.8942\n",
            "Epoch 34/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3906 - accuracy: 0.8894 - val_loss: 0.3653 - val_accuracy: 0.8966\n",
            "Epoch 35/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.3810 - accuracy: 0.8913 - val_loss: 0.3563 - val_accuracy: 0.8984\n",
            "Epoch 36/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3722 - accuracy: 0.8935 - val_loss: 0.3480 - val_accuracy: 0.9006\n",
            "Epoch 37/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3640 - accuracy: 0.8953 - val_loss: 0.3408 - val_accuracy: 0.9024\n",
            "Epoch 38/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3565 - accuracy: 0.8971 - val_loss: 0.3332 - val_accuracy: 0.9037\n",
            "Epoch 39/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3494 - accuracy: 0.8991 - val_loss: 0.3267 - val_accuracy: 0.9051\n",
            "Epoch 40/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3428 - accuracy: 0.9007 - val_loss: 0.3205 - val_accuracy: 0.9065\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.3205 - accuracy: 0.9065\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LexCapFKB8Ma",
        "outputId": "dc52614a-78bc-4f2a-92ad-3f1d3882ac5e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_12\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_27 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_27 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_28 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_28 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_12 (Flatten)        (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_33 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_34 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_35 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 3ms/step - loss: 0.3205 - accuracy: 0.9065\n",
            "[4.59885573387146, 3.413922071456909, 3.708045244216919, 3.7292604446411133, 3.745530128479004, 3.728706121444702, 3.719951629638672, 3.466616630554199, 3.3938167095184326, 3.401606321334839, 3.4632344245910645, 3.415076732635498, 3.4050443172454834, 3.716417074203491, 4.22072696685791, 3.7347335815429688, 3.4106054306030273, 3.8049051761627197, 4.604208946228027, 3.479071855545044, 3.462289571762085, 3.7497849464416504, 3.4755136966705322, 3.748434543609619, 3.5324854850769043, 3.742704391479492, 3.4244515895843506, 3.5063698291778564, 3.4637749195098877, 3.888019561767578, 3.7634711265563965, 3.7837979793548584, 3.455066204071045, 3.711538553237915, 3.458226203918457, 3.8278114795684814, 3.5034546852111816, 3.773885488510132, 3.7787816524505615, 3.497685670852661]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "DROPOUT AT 2 DIFFERENT PLACES AND 2 DIFFERENT DROPOUTS\n"
      ],
      "metadata": {
        "id": "CXKdLQErDp8R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , # activation function = relu , adam optimizer \n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Dropout(rate=0.50))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= 'adam',\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9SNUpGveB8Ob",
        "outputId": "0a06a911-1147-4974-d3c2-1c88b3d0f60e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 5s 4ms/step - loss: 0.2405 - accuracy: 0.9226 - val_loss: 0.0554 - val_accuracy: 0.9815\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0785 - accuracy: 0.9753 - val_loss: 0.0514 - val_accuracy: 0.9833\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0591 - accuracy: 0.9816 - val_loss: 0.0320 - val_accuracy: 0.9906\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0490 - accuracy: 0.9848 - val_loss: 0.0400 - val_accuracy: 0.9870\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0434 - accuracy: 0.9869 - val_loss: 0.0266 - val_accuracy: 0.9919\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0391 - accuracy: 0.9880 - val_loss: 0.0263 - val_accuracy: 0.9901\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0346 - accuracy: 0.9887 - val_loss: 0.0312 - val_accuracy: 0.9897\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0320 - accuracy: 0.9898 - val_loss: 0.0225 - val_accuracy: 0.9930\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0276 - accuracy: 0.9913 - val_loss: 0.0250 - val_accuracy: 0.9924\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0271 - accuracy: 0.9908 - val_loss: 0.0265 - val_accuracy: 0.9922\n",
            "Epoch 11/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0237 - accuracy: 0.9922 - val_loss: 0.0261 - val_accuracy: 0.9910\n",
            "Epoch 12/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0236 - accuracy: 0.9923 - val_loss: 0.0244 - val_accuracy: 0.9929\n",
            "Epoch 13/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0208 - accuracy: 0.9933 - val_loss: 0.0249 - val_accuracy: 0.9915\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0225 - accuracy: 0.9930\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S8IPjqggB8Sh",
        "outputId": "fdcff16f-77cd-4774-92e3-c0c9d0abf5bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_13\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_29 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_29 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_30 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_30 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 5, 5, 64)          0         \n",
            "                                                                 \n",
            " flatten_13 (Flatten)        (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_36 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_37 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_38 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0225 - accuracy: 0.9930\n",
            "[4.511565685272217, 3.5029022693634033, 3.7597556114196777, 3.7877771854400635, 4.244461297988892, 3.3989720344543457, 3.4580373764038086, 3.4615936279296875, 3.4721519947052, 3.749397039413452, 3.4612536430358887, 3.7832045555114746, 3.8770532608032227]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , # activation function = relu , adam optimizer \n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Dropout(rate=0.75))\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= 'adam',\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FnPB4xSeB8Ve",
        "outputId": "79718fbd-d4a5-4224-c567-02a0a81e3f68"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3317 - accuracy: 0.8931 - val_loss: 0.0656 - val_accuracy: 0.9787\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1210 - accuracy: 0.9621 - val_loss: 0.0435 - val_accuracy: 0.9851\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0978 - accuracy: 0.9685 - val_loss: 0.0365 - val_accuracy: 0.9878\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0860 - accuracy: 0.9727 - val_loss: 0.0362 - val_accuracy: 0.9883\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0767 - accuracy: 0.9761 - val_loss: 0.0287 - val_accuracy: 0.9898\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0706 - accuracy: 0.9776 - val_loss: 0.0273 - val_accuracy: 0.9909\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0656 - accuracy: 0.9792 - val_loss: 0.0265 - val_accuracy: 0.9914\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0623 - accuracy: 0.9809 - val_loss: 0.0239 - val_accuracy: 0.9917\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0570 - accuracy: 0.9818 - val_loss: 0.0214 - val_accuracy: 0.9932\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0544 - accuracy: 0.9827 - val_loss: 0.0233 - val_accuracy: 0.9923\n",
            "Epoch 11/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0528 - accuracy: 0.9837 - val_loss: 0.0223 - val_accuracy: 0.9926\n",
            "Epoch 12/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0506 - accuracy: 0.9841 - val_loss: 0.0204 - val_accuracy: 0.9931\n",
            "Epoch 13/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0459 - accuracy: 0.9855 - val_loss: 0.0219 - val_accuracy: 0.9922\n",
            "Epoch 14/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0459 - accuracy: 0.9847 - val_loss: 0.0214 - val_accuracy: 0.9926\n",
            "Epoch 15/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0451 - accuracy: 0.9847 - val_loss: 0.0222 - val_accuracy: 0.9932\n",
            "Epoch 16/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0452 - accuracy: 0.9856 - val_loss: 0.0202 - val_accuracy: 0.9932\n",
            "Epoch 17/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0405 - accuracy: 0.9870 - val_loss: 0.0227 - val_accuracy: 0.9928\n",
            "Epoch 18/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0411 - accuracy: 0.9869 - val_loss: 0.0202 - val_accuracy: 0.9937\n",
            "Epoch 19/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0409 - accuracy: 0.9866 - val_loss: 0.0200 - val_accuracy: 0.9937\n",
            "Epoch 20/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0401 - accuracy: 0.9874 - val_loss: 0.0199 - val_accuracy: 0.9939\n",
            "Epoch 21/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0383 - accuracy: 0.9879 - val_loss: 0.0181 - val_accuracy: 0.9938\n",
            "Epoch 22/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0396 - accuracy: 0.9869 - val_loss: 0.0171 - val_accuracy: 0.9942\n",
            "Epoch 23/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0362 - accuracy: 0.9880 - val_loss: 0.0179 - val_accuracy: 0.9936\n",
            "Epoch 24/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0361 - accuracy: 0.9879 - val_loss: 0.0180 - val_accuracy: 0.9942\n",
            "Epoch 25/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0349 - accuracy: 0.9887 - val_loss: 0.0205 - val_accuracy: 0.9935\n",
            "Epoch 26/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0345 - accuracy: 0.9891 - val_loss: 0.0183 - val_accuracy: 0.9947\n",
            "Epoch 27/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0344 - accuracy: 0.9889 - val_loss: 0.0185 - val_accuracy: 0.9945\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0171 - accuracy: 0.9942\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9k5mif7MB8bX",
        "outputId": "b0c5a3d6-d793-4475-8dec-d24afc5a487f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_14\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_31 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_31 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_32 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_32 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " dropout_1 (Dropout)         (None, 5, 5, 64)          0         \n",
            "                                                                 \n",
            " flatten_14 (Flatten)        (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_39 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_40 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dense_41 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 3ms/step - loss: 0.0171 - accuracy: 0.9942\n",
            "[4.448543310165405, 3.9368348121643066, 3.7539398670196533, 4.479882717132568, 3.551356792449951, 3.850050687789917, 3.8396291732788086, 3.925869941711426, 3.6213254928588867, 3.846511125564575, 3.7533376216888428, 3.782928228378296, 3.5106425285339355, 3.460592269897461, 3.487035036087036, 3.757089376449585, 3.5034842491149902, 3.478555679321289, 3.8498125076293945, 3.831295967102051, 3.419222354888916, 3.4552507400512695, 3.4970269203186035, 3.7841603755950928, 3.5141780376434326, 3.7944211959838867, 3.412412166595459]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , # activation function = relu , adam optimizer \n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dropout(rate=0.50))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= 'adam',\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SmrVE6bKB8cu",
        "outputId": "4143fe0a-bcdb-4d51-d3d0-38d71d92d5b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.4237 - accuracy: 0.8655 - val_loss: 0.0852 - val_accuracy: 0.9748\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1511 - accuracy: 0.9539 - val_loss: 0.0497 - val_accuracy: 0.9855\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.1133 - accuracy: 0.9643 - val_loss: 0.0456 - val_accuracy: 0.9882\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 5s 6ms/step - loss: 0.0936 - accuracy: 0.9713 - val_loss: 0.0416 - val_accuracy: 0.9882\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 5s 5ms/step - loss: 0.0825 - accuracy: 0.9749 - val_loss: 0.0365 - val_accuracy: 0.9902\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.0688 - accuracy: 0.9792 - val_loss: 0.0444 - val_accuracy: 0.9901\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0626 - accuracy: 0.9805 - val_loss: 0.0443 - val_accuracy: 0.9900\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0554 - accuracy: 0.9829 - val_loss: 0.0406 - val_accuracy: 0.9908\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 4s 5ms/step - loss: 0.0486 - accuracy: 0.9852 - val_loss: 0.0385 - val_accuracy: 0.9920\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.0444 - accuracy: 0.9862 - val_loss: 0.0474 - val_accuracy: 0.9907\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0365 - accuracy: 0.9902\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zYJZocRCB8hK",
        "outputId": "62128bac-c4ef-42c5-d497-c0cf7c055209"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_16\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_35 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_35 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_36 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_36 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_16 (Flatten)        (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_45 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_46 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dropout_4 (Dropout)         (None, 32)                0         \n",
            "                                                                 \n",
            " dense_47 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0365 - accuracy: 0.9902\n",
            "[4.440632581710815, 3.7848691940307617, 4.108656883239746, 5.175126075744629, 4.756011247634888, 3.4701156616210938, 3.901496410369873, 4.1055006980896, 4.243941783905029, 3.5041167736053467]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 =Sequential()\n",
        "# testing learning rate = 0.1 \n",
        "# 2 CNN and 3 FC\n",
        "# batch size = 64 , # activation function = relu , adam optimizer \n",
        "model1.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "model1.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model1.add(MaxPooling2D((2, 2),strides=(2,2)))\n",
        "\n",
        "\n",
        "model1.add(Flatten())\n",
        "model1.add(Dense(64, activation='relu'))\n",
        "model1.add(Dense(32, activation='relu'))\n",
        "model1.add(Dropout(rate=0.75))\n",
        "model1.add(Dense(10, activation='softmax'))\n",
        "\n",
        "sgd = SGD(learning_rate=0.1, momentum=0.0)\n",
        "\n",
        "model1.compile(optimizer= 'adam',\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "earlystopping = callbacks.EarlyStopping(monitor =\"val_loss\", \n",
        "                                        mode =\"min\", patience = 5, \n",
        "                                        restore_best_weights = True)\n",
        "\n",
        "time_callback = TimeHistory() \n",
        "history = model1.fit(train_X, train_labels, epochs=40, \n",
        "                    validation_data=(test_X, test_labels),shuffle = True,batch_size = 64 , callbacks =[time_callback,earlystopping])\n",
        "times = time_callback.times\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rWeqXy8UB8kD",
        "outputId": "06917748-edea-4daa-8aa0-bdfb8bd005e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.9102 - accuracy: 0.6470 - val_loss: 0.0890 - val_accuracy: 0.9738\n",
            "Epoch 2/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.5334 - accuracy: 0.7872 - val_loss: 0.0596 - val_accuracy: 0.9835\n",
            "Epoch 3/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.4645 - accuracy: 0.8153 - val_loss: 0.0599 - val_accuracy: 0.9843\n",
            "Epoch 4/40\n",
            "938/938 [==============================] - 3s 4ms/step - loss: 0.4356 - accuracy: 0.8252 - val_loss: 0.0541 - val_accuracy: 0.9873\n",
            "Epoch 5/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.4167 - accuracy: 0.8298 - val_loss: 0.0526 - val_accuracy: 0.9873\n",
            "Epoch 6/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.4077 - accuracy: 0.8355 - val_loss: 0.0635 - val_accuracy: 0.9865\n",
            "Epoch 7/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3887 - accuracy: 0.8403 - val_loss: 0.0577 - val_accuracy: 0.9889\n",
            "Epoch 8/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3802 - accuracy: 0.8468 - val_loss: 0.0632 - val_accuracy: 0.9883\n",
            "Epoch 9/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3633 - accuracy: 0.8528 - val_loss: 0.0557 - val_accuracy: 0.9898\n",
            "Epoch 10/40\n",
            "938/938 [==============================] - 4s 4ms/step - loss: 0.3569 - accuracy: 0.8541 - val_loss: 0.0633 - val_accuracy: 0.9895\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0526 - accuracy: 0.9873\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.summary()\n",
        "test_loss, test_acc = model1.evaluate(test_X,  test_labels, verbose=1)\n",
        "\n",
        "print(times)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dcnnUlEj_Olo",
        "outputId": "a35b4a99-3fc0-4057-a265-d62629b24359"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_17\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv2d_37 (Conv2D)          (None, 26, 26, 32)        320       \n",
            "                                                                 \n",
            " max_pooling2d_37 (MaxPoolin  (None, 13, 13, 32)       0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " conv2d_38 (Conv2D)          (None, 11, 11, 64)        18496     \n",
            "                                                                 \n",
            " max_pooling2d_38 (MaxPoolin  (None, 5, 5, 64)         0         \n",
            " g2D)                                                            \n",
            "                                                                 \n",
            " flatten_17 (Flatten)        (None, 1600)              0         \n",
            "                                                                 \n",
            " dense_48 (Dense)            (None, 64)                102464    \n",
            "                                                                 \n",
            " dense_49 (Dense)            (None, 32)                2080      \n",
            "                                                                 \n",
            " dropout_5 (Dropout)         (None, 32)                0         \n",
            "                                                                 \n",
            " dense_50 (Dense)            (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 123,690\n",
            "Trainable params: 123,690\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "313/313 [==============================] - 1s 3ms/step - loss: 0.0526 - accuracy: 0.9873\n",
            "[4.445716142654419, 3.8321003913879395, 3.781447410583496, 3.4155285358428955, 3.7567224502563477, 3.5473272800445557, 3.7641756534576416, 3.7541558742523193, 3.7932472229003906, 3.8781838417053223]\n"
          ]
        }
      ]
    }
  ]
}