metric_evaluation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "asl_metric_evaluation.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "rqWsIJFoijrX",
        "lCw2TMMni93N",
        "X6w2vO2NjLH_"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aniketsharma00411/sign-language-to-text-translator/blob/main/metric_evaluation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3k99F1O1Ioos"
      },
      "source": [
        "# Initialization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MuOOk2IaHhl7",
        "outputId": "9cf79dbd-2b5f-428c-e73e-445c7b23883e"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PVTNkFh3B5T5"
      },
      "source": [
        "from google.colab import files\n",
        "import os\n",
        "\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras import models\n",
        "from keras.applications import efficientnet\n",
        "from keras.applications import mobilenet\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "from scipy.stats import mode"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3viZCpvmH-M4"
      },
      "source": [
        "os.chdir('./drive/My Drive/Sign Language to Text Translator')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nqHO4sYFIsoD"
      },
      "source": [
        "# Loading Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rUg67V-uIatj"
      },
      "source": [
        "baseline_model = models.load_model('asl_basic.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V2AEdwm_JGB3"
      },
      "source": [
        "baseline_model_augmented = models.load_model('asl_basic_data_augmentation.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d8gp3hil6jDD"
      },
      "source": [
        "efficient_net = models.load_model('asl_efficient_net_b0.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PU264JKuqAG1"
      },
      "source": [
        "mobilenet_augmented = models.load_model('asl_mobilenet_data_augmentation.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V6KkRsrEso7G"
      },
      "source": [
        "ensemble = [models.load_model('asl_basic_ensemble_0.h5'),\n",
        "            models.load_model('asl_basic_ensemble_1.h5'),\n",
        "            models.load_model('asl_basic_ensemble_2.h5'),\n",
        "            models.load_model('asl_basic_ensemble_3.h5'),\n",
        "            models.load_model('asl_basic_ensemble_4.h5')]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3WRC5tg1J6ZD"
      },
      "source": [
        "# ASL Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": "OK"
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "id": "nBJz7AdlJcSK",
        "outputId": "aca0c588-85cc-4519-8110-66ff4119aa87"
      },
      "source": [
        "if not os.path.exists(os.path.expanduser('~')+'/.kaggle'):\n",
        "    ! mkdir ~/.kaggle\n",
        "if not os.path.exists(os.path.expanduser('~')+'/.kaggle/kaggle.json'):\n",
        "    kaggle_api_file = files.upload()\n",
        "    ! mv kaggle.json ~/.kaggle\n",
        "    ! kaggle datasets download -d grassknoted/asl-alphabet\n",
        "    ! mv asl-alphabet.zip ~/.kaggle\n",
        "    ! unzip -q ~/.kaggle/asl-alphabet.zip -d ~/.kaggle\n",
        "    ! rm -rf ~/.kaggle/asl_alphabet_train/asl_alphabet_train/del"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-16367daa-f129-469b-b0cb-3199ad3474ca\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-16367daa-f129-469b-b0cb-3199ad3474ca\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving kaggle.json to kaggle.json\n",
            "Downloading asl-alphabet.zip to /content/drive/My Drive/Sign Language to Text Translator\n",
            " 99% 1.01G/1.03G [00:06<00:00, 165MB/s]\n",
            "100% 1.03G/1.03G [00:07<00:00, 156MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q28qwFnfxQVq"
      },
      "source": [
        "true_labels_data = [num//3000 for num in range(84000)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rqWsIJFoijrX"
      },
      "source": [
        "## Baseline Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jn3Tlh0QuKBi",
        "outputId": "8ac20630-71d0-48ce-c649-c727c618de8c"
      },
      "source": [
        "image_gen = ImageDataGenerator(rescale=1/255)\n",
        "\n",
        "data_gen = image_gen.flow_from_directory(\n",
        "    os.path.expanduser('~')+'/.kaggle/asl_alphabet_train/asl_alphabet_train',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_baseline_model_data = baseline_model.predict(data_gen, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 84000 images belonging to 28 classes.\n",
            "2625/2625 [==============================] - 197s 61ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WP-vXA4MiDsO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cd19b9da-a444-4a13-e36e-a2676a8c30c0"
      },
      "source": [
        "classification_report(true_labels_data, np.argmax(predictions_baseline_model_data, axis=1), output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.9770889487870619,\n",
              "  'precision': 0.9877384196185286,\n",
              "  'recall': 0.9666666666666667,\n",
              "  'support': 3000},\n",
              " '1': {'f1-score': 0.9656425503799142,\n",
              "  'precision': 0.9571054354944335,\n",
              "  'recall': 0.9743333333333334,\n",
              "  'support': 3000},\n",
              " '10': {'f1-score': 0.9918019073113602,\n",
              "  'precision': 0.9956331877729258,\n",
              "  'recall': 0.988,\n",
              "  'support': 3000},\n",
              " '11': {'f1-score': 0.9909213180901143,\n",
              "  'precision': 0.9996607869742198,\n",
              "  'recall': 0.9823333333333333,\n",
              "  'support': 3000},\n",
              " '12': {'f1-score': 0.9487267721954576,\n",
              "  'precision': 0.9804409672830725,\n",
              "  'recall': 0.919,\n",
              "  'support': 3000},\n",
              " '13': {'f1-score': 0.8948929159802306,\n",
              "  'precision': 0.8846905537459283,\n",
              "  'recall': 0.9053333333333333,\n",
              "  'support': 3000},\n",
              " '14': {'f1-score': 0.9450363605614747,\n",
              "  'precision': 0.9591486440096121,\n",
              "  'recall': 0.9313333333333333,\n",
              "  'support': 3000},\n",
              " '15': {'f1-score': 0.9780146568954031,\n",
              "  'precision': 0.9773635153129161,\n",
              "  'recall': 0.9786666666666667,\n",
              "  'support': 3000},\n",
              " '16': {'f1-score': 0.983288770053476,\n",
              "  'precision': 0.9859249329758714,\n",
              "  'recall': 0.9806666666666667,\n",
              "  'support': 3000},\n",
              " '17': {'f1-score': 0.9595723354493818,\n",
              "  'precision': 0.9618218352310783,\n",
              "  'recall': 0.9573333333333334,\n",
              "  'support': 3000},\n",
              " '18': {'f1-score': 0.9003205128205127,\n",
              "  'precision': 0.8669753086419754,\n",
              "  'recall': 0.9363333333333334,\n",
              "  'support': 3000},\n",
              " '19': {'f1-score': 0.961082910321489,\n",
              "  'precision': 0.9759450171821306,\n",
              "  'recall': 0.9466666666666667,\n",
              "  'support': 3000},\n",
              " '2': {'f1-score': 0.9996665555185061,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.9993333333333333,\n",
              "  'support': 3000},\n",
              " '20': {'f1-score': 0.9277966101694914,\n",
              "  'precision': 0.9437931034482758,\n",
              "  'recall': 0.9123333333333333,\n",
              "  'support': 3000},\n",
              " '21': {'f1-score': 0.9484702093397746,\n",
              "  'precision': 0.9174454828660437,\n",
              "  'recall': 0.9816666666666667,\n",
              "  'support': 3000},\n",
              " '22': {'f1-score': 0.9378475629702324,\n",
              "  'precision': 0.9206807964033398,\n",
              "  'recall': 0.9556666666666667,\n",
              "  'support': 3000},\n",
              " '23': {'f1-score': 0.8898845424780286,\n",
              "  'precision': 0.9211559043881555,\n",
              "  'recall': 0.8606666666666667,\n",
              "  'support': 3000},\n",
              " '24': {'f1-score': 0.9374365911396686,\n",
              "  'precision': 0.9512697323266986,\n",
              "  'recall': 0.924,\n",
              "  'support': 3000},\n",
              " '25': {'f1-score': 0.9589926833418412,\n",
              "  'precision': 0.9794925269377824,\n",
              "  'recall': 0.9393333333333334,\n",
              "  'support': 3000},\n",
              " '26': {'f1-score': 0.992717643164515,\n",
              "  'precision': 0.9858645627876397,\n",
              "  'recall': 0.9996666666666667,\n",
              "  'support': 3000},\n",
              " '27': {'f1-score': 0.9490333919156415,\n",
              "  'precision': 0.9113224915618288,\n",
              "  'recall': 0.99,\n",
              "  'support': 3000},\n",
              " '3': {'f1-score': 0.9803855825649623,\n",
              "  'precision': 0.9861720067453625,\n",
              "  'recall': 0.9746666666666667,\n",
              "  'support': 3000},\n",
              " '4': {'f1-score': 0.943504628151931,\n",
              "  'precision': 0.9050826699326393,\n",
              "  'recall': 0.9853333333333333,\n",
              "  'support': 3000},\n",
              " '5': {'f1-score': 0.9912693082605776,\n",
              "  'precision': 0.9986468200270636,\n",
              "  'recall': 0.984,\n",
              "  'support': 3000},\n",
              " '6': {'f1-score': 0.9648274712452075,\n",
              "  'precision': 0.9649883294431477,\n",
              "  'recall': 0.9646666666666667,\n",
              "  'support': 3000},\n",
              " '7': {'f1-score': 0.9595713589868485,\n",
              "  'precision': 0.9354226020892688,\n",
              "  'recall': 0.985,\n",
              "  'support': 3000},\n",
              " '8': {'f1-score': 0.9546241773467267,\n",
              "  'precision': 0.9935111751982696,\n",
              "  'recall': 0.9186666666666666,\n",
              "  'support': 3000},\n",
              " '9': {'f1-score': 0.9686340640809444,\n",
              "  'precision': 0.9802047781569966,\n",
              "  'recall': 0.9573333333333334,\n",
              "  'support': 3000},\n",
              " 'accuracy': 0.9571071428571428,\n",
              " 'macro avg': {'f1-score': 0.9571804406971705,\n",
              "  'precision': 0.9581250566626859,\n",
              "  'recall': 0.9571071428571428,\n",
              "  'support': 84000},\n",
              " 'weighted avg': {'f1-score': 0.9571804406971706,\n",
              "  'precision': 0.9581250566626858,\n",
              "  'recall': 0.9571071428571428,\n",
              "  'support': 84000}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lCw2TMMni93N"
      },
      "source": [
        "## Baseline Model data augmented"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AdBX025GoPSx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc8ecc27-53d8-48dc-b446-d66fcc5fb97d"
      },
      "source": [
        "image_gen_aug = ImageDataGenerator(rescale=1/255)\n",
        "\n",
        "data_gen_aug = image_gen_aug.flow_from_directory(\n",
        "    os.path.expanduser('~')+'/.kaggle/asl_alphabet_train/asl_alphabet_train',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_augmented_model_data = baseline_model_augmented.predict(data_gen_aug, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 84000 images belonging to 28 classes.\n",
            "2625/2625 [==============================] - 158s 60ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FTOVKf1OjEjg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "188fccf2-3fac-4634-e4d9-8249f243667b"
      },
      "source": [
        "classification_report(true_labels_data, np.argmax(predictions_augmented_model_data, axis=1), output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.9833666001330672,\n",
              "  'precision': 0.9814077025232404,\n",
              "  'recall': 0.9853333333333333,\n",
              "  'support': 3000},\n",
              " '1': {'f1-score': 0.9905268406182481,\n",
              "  'precision': 0.9877361617500828,\n",
              "  'recall': 0.9933333333333333,\n",
              "  'support': 3000},\n",
              " '10': {'f1-score': 0.9853449695372962,\n",
              "  'precision': 0.9736413927757891,\n",
              "  'recall': 0.9973333333333333,\n",
              "  'support': 3000},\n",
              " '11': {'f1-score': 0.9973395410708347,\n",
              "  'precision': 0.9950232249502322,\n",
              "  'recall': 0.9996666666666667,\n",
              "  'support': 3000},\n",
              " '12': {'f1-score': 0.8769889840881273,\n",
              "  'precision': 0.8105203619909502,\n",
              "  'recall': 0.9553333333333334,\n",
              "  'support': 3000},\n",
              " '13': {'f1-score': 0.8658294778585591,\n",
              "  'precision': 0.8584534731323722,\n",
              "  'recall': 0.8733333333333333,\n",
              "  'support': 3000},\n",
              " '14': {'f1-score': 0.9885521885521886,\n",
              "  'precision': 0.998639455782313,\n",
              "  'recall': 0.9786666666666667,\n",
              "  'support': 3000},\n",
              " '15': {'f1-score': 0.9415354963058143,\n",
              "  'precision': 0.9085554866707998,\n",
              "  'recall': 0.977,\n",
              "  'support': 3000},\n",
              " '16': {'f1-score': 0.9413607099356186,\n",
              "  'precision': 0.9847105933745904,\n",
              "  'recall': 0.9016666666666666,\n",
              "  'support': 3000},\n",
              " '17': {'f1-score': 0.8704033214709371,\n",
              "  'precision': 0.7839209401709402,\n",
              "  'recall': 0.9783333333333334,\n",
              "  'support': 3000},\n",
              " '18': {'f1-score': 0.9498708010335918,\n",
              "  'precision': 0.9828877005347594,\n",
              "  'recall': 0.919,\n",
              "  'support': 3000},\n",
              " '19': {'f1-score': 0.9660705331773359,\n",
              "  'precision': 0.96882333221589,\n",
              "  'recall': 0.9633333333333334,\n",
              "  'support': 3000},\n",
              " '2': {'f1-score': 0.9979959919839679,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.996,\n",
              "  'support': 3000},\n",
              " '20': {'f1-score': 0.8504289444237224,\n",
              "  'precision': 0.9652836579170194,\n",
              "  'recall': 0.76,\n",
              "  'support': 3000},\n",
              " '21': {'f1-score': 0.9304333388676738,\n",
              "  'precision': 0.9268938140919616,\n",
              "  'recall': 0.934,\n",
              "  'support': 3000},\n",
              " '22': {'f1-score': 0.9414610546686019,\n",
              "  'precision': 0.9119025304592315,\n",
              "  'recall': 0.973,\n",
              "  'support': 3000},\n",
              " '23': {'f1-score': 0.9304888888888888,\n",
              "  'precision': 0.9969523809523809,\n",
              "  'recall': 0.8723333333333333,\n",
              "  'support': 3000},\n",
              " '24': {'f1-score': 0.9752673796791443,\n",
              "  'precision': 0.9778820375335121,\n",
              "  'recall': 0.9726666666666667,\n",
              "  'support': 3000},\n",
              " '25': {'f1-score': 0.970748299319728,\n",
              "  'precision': 0.9909722222222223,\n",
              "  'recall': 0.9513333333333334,\n",
              "  'support': 3000},\n",
              " '26': {'f1-score': 0.9766331229745864,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.9543333333333334,\n",
              "  'support': 3000},\n",
              " '27': {'f1-score': 0.9923256589923256,\n",
              "  'precision': 0.9933199732798931,\n",
              "  'recall': 0.9913333333333333,\n",
              "  'support': 3000},\n",
              " '3': {'f1-score': 0.9994999166527755,\n",
              "  'precision': 0.9996665555185061,\n",
              "  'recall': 0.9993333333333333,\n",
              "  'support': 3000},\n",
              " '4': {'f1-score': 0.9830452127659575,\n",
              "  'precision': 0.9804376657824934,\n",
              "  'recall': 0.9856666666666667,\n",
              "  'support': 3000},\n",
              " '5': {'f1-score': 0.99849774661993,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.997,\n",
              "  'support': 3000},\n",
              " '6': {'f1-score': 0.9667849080941631,\n",
              "  'precision': 0.9362898188632105,\n",
              "  'recall': 0.9993333333333333,\n",
              "  'support': 3000},\n",
              " '7': {'f1-score': 0.9691252144082333,\n",
              "  'precision': 0.9982332155477032,\n",
              "  'recall': 0.9416666666666667,\n",
              "  'support': 3000},\n",
              " '8': {'f1-score': 0.9836010709504686,\n",
              "  'precision': 0.9875672043010753,\n",
              "  'recall': 0.9796666666666667,\n",
              "  'support': 3000},\n",
              " '9': {'f1-score': 0.9859726212607741,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.9723333333333334,\n",
              "  'support': 3000},\n",
              " 'accuracy': 0.9572261904761905,\n",
              " 'macro avg': {'f1-score': 0.9574821012261628,\n",
              "  'precision': 0.960704317940756,\n",
              "  'recall': 0.9572261904761905,\n",
              "  'support': 84000},\n",
              " 'weighted avg': {'f1-score': 0.957482101226163,\n",
              "  'precision': 0.9607043179407561,\n",
              "  'recall': 0.9572261904761905,\n",
              "  'support': 84000}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X6w2vO2NjLH_"
      },
      "source": [
        "## Efficient Net"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DWdFlf3k6sxg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "43f1b5d7-8779-4275-fbb1-100bc2c0c68a"
      },
      "source": [
        "image_gen_efficient_net = ImageDataGenerator(preprocessing_function=efficientnet.preprocess_input)\n",
        "\n",
        "data_gen_efficient_net = image_gen_efficient_net.flow_from_directory(\n",
        "    os.path.expanduser('~')+'/.kaggle/asl_alphabet_train/asl_alphabet_train',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_efficient_net_data = efficient_net.predict(data_gen_efficient_net, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 84000 images belonging to 28 classes.\n",
            "2625/2625 [==============================] - 183s 66ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ikpB8pcYpHuk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e6d384d2-e380-4156-ff7e-0b854a501723"
      },
      "source": [
        "classification_report(true_labels_data, np.argmax(predictions_efficient_net_data, axis=1), output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.9752925877763329,\n",
              "  'precision': 0.9517766497461929,\n",
              "  'recall': 1.0,\n",
              "  'support': 3000},\n",
              " '1': {'f1-score': 0.999333777481679,\n",
              "  'precision': 0.9986684420772304,\n",
              "  'recall': 1.0,\n",
              "  'support': 3000},\n",
              " '10': {'f1-score': 0.9840106595602931,\n",
              "  'precision': 0.9833555259653795,\n",
              "  'recall': 0.9846666666666667,\n",
              "  'support': 3000},\n",
              " '11': {'f1-score': 0.9931219594027849,\n",
              "  'precision': 0.999662276258021,\n",
              "  'recall': 0.9866666666666667,\n",
              "  'support': 3000},\n",
              " '12': {'f1-score': 0.9602385685884691,\n",
              "  'precision': 0.9545454545454546,\n",
              "  'recall': 0.966,\n",
              "  'support': 3000},\n",
              " '13': {'f1-score': 0.9534612176814011,\n",
              "  'precision': 0.9542570951585977,\n",
              "  'recall': 0.9526666666666667,\n",
              "  'support': 3000},\n",
              " '14': {'f1-score': 0.9809001826939047,\n",
              "  'precision': 0.9774908970539556,\n",
              "  'recall': 0.9843333333333333,\n",
              "  'support': 3000},\n",
              " '15': {'f1-score': 0.986795921778372,\n",
              "  'precision': 0.9896077774052967,\n",
              "  'recall': 0.984,\n",
              "  'support': 3000},\n",
              " '16': {'f1-score': 0.9853111074434725,\n",
              "  'precision': 0.9758090879372344,\n",
              "  'recall': 0.995,\n",
              "  'support': 3000},\n",
              " '17': {'f1-score': 0.9823519709714662,\n",
              "  'precision': 0.9722494286647078,\n",
              "  'recall': 0.9926666666666667,\n",
              "  'support': 3000},\n",
              " '18': {'f1-score': 0.9624035462157281,\n",
              "  'precision': 0.9482368165642187,\n",
              "  'recall': 0.977,\n",
              "  'support': 3000},\n",
              " '19': {'f1-score': 0.9809153859145414,\n",
              "  'precision': 0.9941800753166724,\n",
              "  'recall': 0.968,\n",
              "  'support': 3000},\n",
              " '2': {'f1-score': 0.987971659251936,\n",
              "  'precision': 0.9768654284783317,\n",
              "  'recall': 0.9993333333333333,\n",
              "  'support': 3000},\n",
              " '20': {'f1-score': 0.9804516346477924,\n",
              "  'precision': 0.9914792092706203,\n",
              "  'recall': 0.9696666666666667,\n",
              "  'support': 3000},\n",
              " '21': {'f1-score': 0.989010989010989,\n",
              "  'precision': 0.9880239520958084,\n",
              "  'recall': 0.99,\n",
              "  'support': 3000},\n",
              " '22': {'f1-score': 0.9890619821014252,\n",
              "  'precision': 0.983520105471325,\n",
              "  'recall': 0.9946666666666667,\n",
              "  'support': 3000},\n",
              " '23': {'f1-score': 0.9596843369360096,\n",
              "  'precision': 0.9886885825379993,\n",
              "  'recall': 0.9323333333333333,\n",
              "  'support': 3000},\n",
              " '24': {'f1-score': 0.9812882927637027,\n",
              "  'precision': 0.9749917736097401,\n",
              "  'recall': 0.9876666666666667,\n",
              "  'support': 3000},\n",
              " '25': {'f1-score': 0.9750768180266303,\n",
              "  'precision': 0.9993002099370188,\n",
              "  'recall': 0.952,\n",
              "  'support': 3000},\n",
              " '26': {'f1-score': 0.990748528174937,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.9816666666666667,\n",
              "  'support': 3000},\n",
              " '27': {'f1-score': 0.9983355525965381,\n",
              "  'precision': 0.9970079787234043,\n",
              "  'recall': 0.9996666666666667,\n",
              "  'support': 3000},\n",
              " '3': {'f1-score': 0.98989898989899,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.98,\n",
              "  'support': 3000},\n",
              " '4': {'f1-score': 0.9829508196721312,\n",
              "  'precision': 0.9670967741935483,\n",
              "  'recall': 0.9993333333333333,\n",
              "  'support': 3000},\n",
              " '5': {'f1-score': 0.9998333055509252,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.9996666666666667,\n",
              "  'support': 3000},\n",
              " '6': {'f1-score': 0.9782312925170069,\n",
              "  'precision': 0.9986111111111111,\n",
              "  'recall': 0.9586666666666667,\n",
              "  'support': 3000},\n",
              " '7': {'f1-score': 0.9781545484186501,\n",
              "  'precision': 0.9572431397574984,\n",
              "  'recall': 1.0,\n",
              "  'support': 3000},\n",
              " '8': {'f1-score': 0.977957260642773,\n",
              "  'precision': 0.9874277947672443,\n",
              "  'recall': 0.9686666666666667,\n",
              "  'support': 3000},\n",
              " '9': {'f1-score': 0.9706275033377837,\n",
              "  'precision': 0.9719251336898396,\n",
              "  'recall': 0.9693333333333334,\n",
              "  'support': 3000},\n",
              " 'accuracy': 0.981202380952381,\n",
              " 'macro avg': {'f1-score': 0.9811935856805951,\n",
              "  'precision': 0.9815007400120163,\n",
              "  'recall': 0.981202380952381,\n",
              "  'support': 84000},\n",
              " 'weighted avg': {'f1-score': 0.981193585680595,\n",
              "  'precision': 0.9815007400120163,\n",
              "  'recall': 0.981202380952381,\n",
              "  'support': 84000}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SQjfuDkfpVwL"
      },
      "source": [
        "## Mobilenet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wGi4Yj_JqMG5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c28facd8-8a7d-4eb1-ad50-8ed42b9a4523"
      },
      "source": [
        "image_gen_mobilenet = ImageDataGenerator(preprocessing_function=mobilenet.preprocess_input)\n",
        "\n",
        "data_gen_mobilenet = image_gen_mobilenet.flow_from_directory(\n",
        "    os.path.expanduser('~')+'/.kaggle/asl_alphabet_train/asl_alphabet_train',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_mobilenet_data = mobilenet_augmented.predict(data_gen_mobilenet, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 84000 images belonging to 28 classes.\n",
            "2625/2625 [==============================] - 162s 61ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "99HPf80CpZh9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a32848f6-c695-429f-bd27-f9440b2621ea"
      },
      "source": [
        "classification_report(true_labels_data, np.argmax(predictions_mobilenet_data, axis=1), output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.948456888323258,\n",
              "  'precision': 0.9071819841752891,\n",
              "  'recall': 0.9936666666666667,\n",
              "  'support': 3000},\n",
              " '1': {'f1-score': 0.9898922949461474,\n",
              "  'precision': 0.9841845140032949,\n",
              "  'recall': 0.9956666666666667,\n",
              "  'support': 3000},\n",
              " '10': {'f1-score': 0.9480731548007838,\n",
              "  'precision': 0.9292573623559539,\n",
              "  'recall': 0.9676666666666667,\n",
              "  'support': 3000},\n",
              " '11': {'f1-score': 0.9851876234364715,\n",
              "  'precision': 0.9730169050715215,\n",
              "  'recall': 0.9976666666666667,\n",
              "  'support': 3000},\n",
              " '12': {'f1-score': 0.9296270232230823,\n",
              "  'precision': 0.9843517138599106,\n",
              "  'recall': 0.8806666666666667,\n",
              "  'support': 3000},\n",
              " '13': {'f1-score': 0.9387107018100984,\n",
              "  'precision': 0.8963007883565798,\n",
              "  'recall': 0.9853333333333333,\n",
              "  'support': 3000},\n",
              " '14': {'f1-score': 0.9525629887054735,\n",
              "  'precision': 0.9949183303085299,\n",
              "  'recall': 0.9136666666666666,\n",
              "  'support': 3000},\n",
              " '15': {'f1-score': 0.938217261436881,\n",
              "  'precision': 0.8878310026777745,\n",
              "  'recall': 0.9946666666666667,\n",
              "  'support': 3000},\n",
              " '16': {'f1-score': 0.9594337994130848,\n",
              "  'precision': 0.9949874686716792,\n",
              "  'recall': 0.9263333333333333,\n",
              "  'support': 3000},\n",
              " '17': {'f1-score': 0.9158547387068203,\n",
              "  'precision': 0.9773156899810964,\n",
              "  'recall': 0.8616666666666667,\n",
              "  'support': 3000},\n",
              " '18': {'f1-score': 0.8887748117727584,\n",
              "  'precision': 0.9131504922644164,\n",
              "  'recall': 0.8656666666666667,\n",
              "  'support': 3000},\n",
              " '19': {'f1-score': 0.9547450185748058,\n",
              "  'precision': 0.9674880219028063,\n",
              "  'recall': 0.9423333333333334,\n",
              "  'support': 3000},\n",
              " '2': {'f1-score': 0.9840486762045716,\n",
              "  'precision': 0.9711132749107433,\n",
              "  'recall': 0.9973333333333333,\n",
              "  'support': 3000},\n",
              " '20': {'f1-score': 0.9008,\n",
              "  'precision': 0.8661538461538462,\n",
              "  'recall': 0.9383333333333334,\n",
              "  'support': 3000},\n",
              " '21': {'f1-score': 0.9458319818358741,\n",
              "  'precision': 0.9210360075805433,\n",
              "  'recall': 0.972,\n",
              "  'support': 3000},\n",
              " '22': {'f1-score': 0.9626403082593399,\n",
              "  'precision': 0.9676658807679354,\n",
              "  'recall': 0.9576666666666667,\n",
              "  'support': 3000},\n",
              " '23': {'f1-score': 0.8706990220934444,\n",
              "  'precision': 0.9532117367168914,\n",
              "  'recall': 0.8013333333333333,\n",
              "  'support': 3000},\n",
              " '24': {'f1-score': 0.9601747605444464,\n",
              "  'precision': 0.96814639105388,\n",
              "  'recall': 0.9523333333333334,\n",
              "  'support': 3000},\n",
              " '25': {'f1-score': 0.9375499121546079,\n",
              "  'precision': 0.9000306654400491,\n",
              "  'recall': 0.9783333333333334,\n",
              "  'support': 3000},\n",
              " '26': {'f1-score': 0.9581524570237888,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.9196666666666666,\n",
              "  'support': 3000},\n",
              " '27': {'f1-score': 0.9844436929320257,\n",
              "  'precision': 0.9989704873026767,\n",
              "  'recall': 0.9703333333333334,\n",
              "  'support': 3000},\n",
              " '3': {'f1-score': 0.9873083896489204,\n",
              "  'precision': 0.9765242908379524,\n",
              "  'recall': 0.9983333333333333,\n",
              "  'support': 3000},\n",
              " '4': {'f1-score': 0.9375413086582949,\n",
              "  'precision': 0.9295543905635649,\n",
              "  'recall': 0.9456666666666667,\n",
              "  'support': 3000},\n",
              " '5': {'f1-score': 0.9941812136325853,\n",
              "  'precision': 0.9917081260364843,\n",
              "  'recall': 0.9966666666666667,\n",
              "  'support': 3000},\n",
              " '6': {'f1-score': 0.9495646459668147,\n",
              "  'precision': 0.9361839974084872,\n",
              "  'recall': 0.9633333333333334,\n",
              "  'support': 3000},\n",
              " '7': {'f1-score': 0.9609210312956056,\n",
              "  'precision': 0.9355857278181244,\n",
              "  'recall': 0.9876666666666667,\n",
              "  'support': 3000},\n",
              " '8': {'f1-score': 0.9300522501264116,\n",
              "  'precision': 0.9406750767132629,\n",
              "  'recall': 0.9196666666666666,\n",
              "  'support': 3000},\n",
              " '9': {'f1-score': 0.9647334113321078,\n",
              "  'precision': 0.9674824002681864,\n",
              "  'recall': 0.962,\n",
              "  'support': 3000},\n",
              " 'accuracy': 0.9494880952380952,\n",
              " 'macro avg': {'f1-score': 0.9492206916735181,\n",
              "  'precision': 0.9512152347571957,\n",
              "  'recall': 0.9494880952380952,\n",
              "  'support': 84000},\n",
              " 'weighted avg': {'f1-score': 0.9492206916735181,\n",
              "  'precision': 0.9512152347571957,\n",
              "  'recall': 0.9494880952380952,\n",
              "  'support': 84000}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OTw-C19gsiwu"
      },
      "source": [
        "## Ensemble"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zfTD05Sasl72",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1a0d4ea9-f4fa-4f36-9f6b-bb646546ddb6"
      },
      "source": [
        "image_gen_ensemble = ImageDataGenerator(rescale=1/255)\n",
        "\n",
        "data_gen_ensemble = image_gen_ensemble.flow_from_directory(\n",
        "    os.path.expanduser('~')+'/.kaggle/asl_alphabet_train/asl_alphabet_train',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_ensemble_data = mode([np.argmax(model.predict(data_gen_ensemble, verbose=1), axis=1) for model in ensemble])[0][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 84000 images belonging to 28 classes.\n",
            "2625/2625 [==============================] - 158s 60ms/step\n",
            "2625/2625 [==============================] - 156s 59ms/step\n",
            "2625/2625 [==============================] - 156s 59ms/step\n",
            "2625/2625 [==============================] - 155s 59ms/step\n",
            "2625/2625 [==============================] - 157s 60ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bYQat4GLx6CU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c922a766-f51f-43de-8f05-058c631c134d"
      },
      "source": [
        "classification_report(true_labels_data, predictions_ensemble_data, output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '1': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '10': {'f1-score': 0.9998333055509252,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.9996666666666667,\n",
              "  'support': 3000},\n",
              " '11': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '12': {'f1-score': 0.9998333055509252,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.9996666666666667,\n",
              "  'support': 3000},\n",
              " '13': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '14': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '15': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '16': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '17': {'f1-score': 0.9995002498750626,\n",
              "  'precision': 0.999000999000999,\n",
              "  'recall': 1.0,\n",
              "  'support': 3000},\n",
              " '18': {'f1-score': 0.9998333611064822,\n",
              "  'precision': 0.9996667777407531,\n",
              "  'recall': 1.0,\n",
              "  'support': 3000},\n",
              " '19': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '2': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '20': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '21': {'f1-score': 0.9993331110370124,\n",
              "  'precision': 0.9996664442961974,\n",
              "  'recall': 0.999,\n",
              "  'support': 3000},\n",
              " '22': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '23': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '24': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '25': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '26': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '27': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '3': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '4': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '5': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '6': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '7': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '8': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " '9': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 3000},\n",
              " 'accuracy': 0.9999404761904762,\n",
              " 'macro avg': {'f1-score': 0.9999404761828716,\n",
              "  'precision': 0.9999405078942125,\n",
              "  'recall': 0.9999404761904762,\n",
              "  'support': 84000},\n",
              " 'weighted avg': {'f1-score': 0.9999404761828717,\n",
              "  'precision': 0.9999405078942124,\n",
              "  'recall': 0.9999404761904762,\n",
              "  'support': 84000}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QRwX34iZ8Xmf"
      },
      "source": [
        "# ASL Alphabets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ch1ghDw58Xmi"
      },
      "source": [
        "true_labels_alpha = [num//32 for num in range(832)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fI0SlMa98Xmi"
      },
      "source": [
        "## Baseline Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8VOeTJs8Xmi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b276e4fc-7cf7-4ac9-d885-c826af5385e1"
      },
      "source": [
        "image_gen2 = ImageDataGenerator(rescale=1/255)\n",
        "\n",
        "data_gen2 = image_gen2.flow_from_directory(\n",
        "    'asl_alphabets',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_baseline_model_alpha = baseline_model.predict(data_gen2, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 832 images belonging to 26 classes.\n",
            "26/26 [==============================] - 220s 9s/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5nHbOiE18Xmj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "252a6f32-3308-4daf-be0f-14d68f465f03"
      },
      "source": [
        "classification_report(true_labels_alpha, np.argmax(predictions_baseline_model_alpha, axis=1), output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '1': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '10': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '11': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '12': {'f1-score': 0.03125,\n",
              "  'precision': 0.03125,\n",
              "  'recall': 0.03125,\n",
              "  'support': 32},\n",
              " '13': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '14': {'f1-score': 0.9142857142857143,\n",
              "  'precision': 0.8421052631578947,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '15': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '16': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '17': {'f1-score': 0.27027027027027023,\n",
              "  'precision': 0.23809523809523808,\n",
              "  'recall': 0.3125,\n",
              "  'support': 32},\n",
              " '18': {'f1-score': 0.3316062176165803,\n",
              "  'precision': 0.19875776397515527,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '19': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '2': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '20': {'f1-score': 0.5925925925925926,\n",
              "  'precision': 0.42105263157894735,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '21': {'f1-score': 0.6666666666666666,\n",
              "  'precision': 0.5,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '22': {'f1-score': 0.7441860465116279,\n",
              "  'precision': 0.5925925925925926,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '23': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '24': {'f1-score': 0.7692307692307693,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.625,\n",
              "  'support': 32},\n",
              " '25': {'f1-score': 0.2,\n",
              "  'precision': 0.11797752808988764,\n",
              "  'recall': 0.65625,\n",
              "  'support': 32},\n",
              " '26': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 0},\n",
              " '27': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 0},\n",
              " '3': {'f1-score': 0.6666666666666666,\n",
              "  'precision': 0.5,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '4': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '5': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '6': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '7': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '8': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '9': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " 'accuracy': 0.3701923076923077,\n",
              " 'macro avg': {'f1-score': 0.2566698194228888,\n",
              "  'precision': 0.23006539348177557,\n",
              "  'recall': 0.34375,\n",
              "  'support': 832},\n",
              " 'weighted avg': {'f1-score': 0.276413651686188,\n",
              "  'precision': 0.24776273144191216,\n",
              "  'recall': 0.3701923076923077,\n",
              "  'support': 832}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "btjNkmwV8Xmk"
      },
      "source": [
        "## Baseline Model data augmented"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6vS3mGow8Xmk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "80289b3a-5ebb-40dc-d8d7-a0a4fe8920f6"
      },
      "source": [
        "image_gen_aug2 = ImageDataGenerator(rescale=1/255)\n",
        "\n",
        "data_gen_aug2 = image_gen_aug2.flow_from_directory(\n",
        "    'asl_alphabets',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_augmented_model_alpha = baseline_model_augmented.predict(data_gen_aug2, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 832 images belonging to 26 classes.\n",
            "26/26 [==============================] - 2s 86ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRrxBMnp8Xmk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c807917-b8e3-493f-d575-a2f4f682208a"
      },
      "source": [
        "classification_report(true_labels_alpha, np.argmax(predictions_augmented_model_alpha, axis=1), output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.2711864406779661,\n",
              "  'precision': 0.1568627450980392,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '1': {'f1-score': 0.5517241379310345,\n",
              "  'precision': 0.38095238095238093,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '10': {'f1-score': 0.3157894736842105,\n",
              "  'precision': 0.2727272727272727,\n",
              "  'recall': 0.375,\n",
              "  'support': 32},\n",
              " '11': {'f1-score': 0.3157894736842105,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.1875,\n",
              "  'support': 32},\n",
              " '12': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '13': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '14': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '15': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '16': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '17': {'f1-score': 0.18518518518518515,\n",
              "  'precision': 0.22727272727272727,\n",
              "  'recall': 0.15625,\n",
              "  'support': 32},\n",
              " '18': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '19': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '2': {'f1-score': 0.48120300751879697,\n",
              "  'precision': 0.31683168316831684,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '20': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '21': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '22': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '23': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '24': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '25': {'f1-score': 0.4,\n",
              "  'precision': 0.3333333333333333,\n",
              "  'recall': 0.5,\n",
              "  'support': 32},\n",
              " '3': {'f1-score': 0.3142857142857143,\n",
              "  'precision': 0.2894736842105263,\n",
              "  'recall': 0.34375,\n",
              "  'support': 32},\n",
              " '4': {'f1-score': 0.5663716814159292,\n",
              "  'precision': 0.3950617283950617,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '5': {'f1-score': 0.6666666666666666,\n",
              "  'precision': 0.5,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '6': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '7': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '8': {'f1-score': 0.927536231884058,\n",
              "  'precision': 0.8648648648648649,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '9': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " 'accuracy': 0.36778846153846156,\n",
              " 'macro avg': {'f1-score': 0.26906684665129893,\n",
              "  'precision': 0.25913001615471243,\n",
              "  'recall': 0.36778846153846156,\n",
              "  'support': 832},\n",
              " 'weighted avg': {'f1-score': 0.26906684665129893,\n",
              "  'precision': 0.25913001615471243,\n",
              "  'recall': 0.36778846153846156,\n",
              "  'support': 832}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eHeYfCnx8Xml"
      },
      "source": [
        "## Efficient Net"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YAVCbXnn8Xmm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "43679fc2-bf06-49b1-ee5b-86209cc7ee23"
      },
      "source": [
        "image_gen_efficient_net2 = ImageDataGenerator(preprocessing_function=efficientnet.preprocess_input)\n",
        "\n",
        "data_gen_efficient_net2 = image_gen_efficient_net2.flow_from_directory(\n",
        "    'asl_alphabets',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_efficient_net_alpha = efficient_net.predict(data_gen_efficient_net2, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 832 images belonging to 26 classes.\n",
            "26/26 [==============================] - 2s 90ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bzMdepG-8Xmm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5b9e55b0-b81f-4799-d759-66768c8b5f97"
      },
      "source": [
        "classification_report(true_labels_alpha, np.argmax(predictions_efficient_net_alpha, axis=1), output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.45714285714285713,\n",
              "  'precision': 0.2962962962962963,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '1': {'f1-score': 0.8205128205128205,\n",
              "  'precision': 0.6956521739130435,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '10': {'f1-score': 0.35593220338983045,\n",
              "  'precision': 0.2441860465116279,\n",
              "  'recall': 0.65625,\n",
              "  'support': 32},\n",
              " '11': {'f1-score': 0.9333333333333333,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.875,\n",
              "  'support': 32},\n",
              " '12': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '13': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '14': {'f1-score': 0.9846153846153847,\n",
              "  'precision': 0.9696969696969697,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '15': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '16': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '17': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '18': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '19': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '2': {'f1-score': 0.5818181818181819,\n",
              "  'precision': 0.41025641025641024,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '20': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '21': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '22': {'f1-score': 0.7733333333333334,\n",
              "  'precision': 0.6744186046511628,\n",
              "  'recall': 0.90625,\n",
              "  'support': 32},\n",
              " '23': {'f1-score': 0.9846153846153847,\n",
              "  'precision': 0.9696969696969697,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '24': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '25': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '27': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 0},\n",
              " '3': {'f1-score': 0.5777777777777777,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.40625,\n",
              "  'support': 32},\n",
              " '4': {'f1-score': 0.3902439024390244,\n",
              "  'precision': 0.24242424242424243,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '5': {'f1-score': 0.2702702702702703,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.15625,\n",
              "  'support': 32},\n",
              " '6': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '7': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '8': {'f1-score': 0.36994219653179194,\n",
              "  'precision': 0.22695035460992907,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '9': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " 'accuracy': 0.4230769230769231,\n",
              " 'macro avg': {'f1-score': 0.3147976905844441,\n",
              "  'precision': 0.32331770622432043,\n",
              "  'recall': 0.4074074074074074,\n",
              "  'support': 832},\n",
              " 'weighted avg': {'f1-score': 0.3269052940684612,\n",
              "  'precision': 0.3357530026175635,\n",
              "  'recall': 0.4230769230769231,\n",
              "  'support': 832}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WGXy3oUL8Xmn"
      },
      "source": [
        "## Mobilenet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OzIXfBU68Xmn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0283a5e2-c59a-4ddf-dafc-51b2951dc933"
      },
      "source": [
        "image_gen_mobilenet2 = ImageDataGenerator(preprocessing_function=mobilenet.preprocess_input)\n",
        "\n",
        "data_gen_mobilenet2 = image_gen_mobilenet2.flow_from_directory(\n",
        "    'asl_alphabets',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_mobilenet_alpha = mobilenet_augmented.predict(data_gen_mobilenet2, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 832 images belonging to 26 classes.\n",
            "26/26 [==============================] - 2s 85ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Vayv7KB8Xmo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fc3c4056-ed73-47d6-8ca1-209cb4412e10"
      },
      "source": [
        "classification_report(true_labels_alpha, np.argmax(predictions_mobilenet_alpha, axis=1), output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.8999999999999999,\n",
              "  'precision': 0.9642857142857143,\n",
              "  'recall': 0.84375,\n",
              "  'support': 32},\n",
              " '1': {'f1-score': 0.9846153846153847,\n",
              "  'precision': 0.9696969696969697,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '10': {'f1-score': 0.5057471264367817,\n",
              "  'precision': 0.4,\n",
              "  'recall': 0.6875,\n",
              "  'support': 32},\n",
              " '11': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '12': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '13': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '14': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '15': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '16': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '17': {'f1-score': 0.06060606060606061,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.03125,\n",
              "  'support': 32},\n",
              " '18': {'f1-score': 0.3375,\n",
              "  'precision': 0.2109375,\n",
              "  'recall': 0.84375,\n",
              "  'support': 32},\n",
              " '19': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '2': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '20': {'f1-score': 0.761904761904762,\n",
              "  'precision': 0.6153846153846154,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '21': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '22': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '23': {'f1-score': 0.4318181818181819,\n",
              "  'precision': 0.3392857142857143,\n",
              "  'recall': 0.59375,\n",
              "  'support': 32},\n",
              " '24': {'f1-score': 0.5,\n",
              "  'precision': 0.9166666666666666,\n",
              "  'recall': 0.34375,\n",
              "  'support': 32},\n",
              " '25': {'f1-score': 0.20512820512820512,\n",
              "  'precision': 0.5714285714285714,\n",
              "  'recall': 0.125,\n",
              "  'support': 32},\n",
              " '3': {'f1-score': 0.8307692307692308,\n",
              "  'precision': 0.8181818181818182,\n",
              "  'recall': 0.84375,\n",
              "  'support': 32},\n",
              " '4': {'f1-score': 0.2677165354330709,\n",
              "  'precision': 0.17894736842105263,\n",
              "  'recall': 0.53125,\n",
              "  'support': 32},\n",
              " '5': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '6': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '7': {'f1-score': 0.5,\n",
              "  'precision': 0.5833333333333334,\n",
              "  'recall': 0.4375,\n",
              "  'support': 32},\n",
              " '8': {'f1-score': 0.5818181818181819,\n",
              "  'precision': 0.41025641025641024,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '9': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " 'accuracy': 0.4338942307692308,\n",
              " 'macro avg': {'f1-score': 0.37952398725114844,\n",
              "  'precision': 0.4222463339208025,\n",
              "  'recall': 0.4338942307692308,\n",
              "  'support': 832},\n",
              " 'weighted avg': {'f1-score': 0.37952398725114844,\n",
              "  'precision': 0.4222463339208025,\n",
              "  'recall': 0.4338942307692308,\n",
              "  'support': 832}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vavClv_S8Xmo"
      },
      "source": [
        "## Ensemble"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GkGZwnnU8Xmo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4b59ae9b-7a7b-481e-fc3e-cf02a4eeeac3"
      },
      "source": [
        "image_gen_ensemble2 = ImageDataGenerator(rescale=1/255)\n",
        "\n",
        "data_gen_ensemble2 = image_gen_ensemble2.flow_from_directory(\n",
        "    'asl_alphabets',\n",
        "    target_size=(224, 224),\n",
        "    class_mode=None,\n",
        "    color_mode='rgb',\n",
        "    shuffle=False\n",
        "    )\n",
        "\n",
        "predictions_ensemble_alpha = mode([np.argmax(model.predict(data_gen_ensemble2, verbose=1), axis=1) for model in ensemble])[0][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 832 images belonging to 26 classes.\n",
            "26/26 [==============================] - 2s 86ms/step\n",
            "26/26 [==============================] - 2s 84ms/step\n",
            "26/26 [==============================] - 2s 82ms/step\n",
            "26/26 [==============================] - 2s 82ms/step\n",
            "26/26 [==============================] - 2s 80ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bBI5AW1-8Xmo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a4266d3-bac1-4b38-fabc-5808495ac0fb"
      },
      "source": [
        "classification_report(true_labels_alpha, predictions_ensemble_alpha, output_dict=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': {'f1-score': 0.9523809523809523,\n",
              "  'precision': 0.967741935483871,\n",
              "  'recall': 0.9375,\n",
              "  'support': 32},\n",
              " '1': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '10': {'f1-score': 0.358974358974359,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.21875,\n",
              "  'support': 32},\n",
              " '11': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '12': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '13': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '14': {'f1-score': 0.6666666666666666,\n",
              "  'precision': 0.5,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '15': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '16': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '17': {'f1-score': 0.15789473684210525,\n",
              "  'precision': 0.5,\n",
              "  'recall': 0.09375,\n",
              "  'support': 32},\n",
              " '18': {'f1-score': 0.3516483516483517,\n",
              "  'precision': 0.21333333333333335,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '19': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '2': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '20': {'f1-score': 0.6213592233009708,\n",
              "  'precision': 0.4507042253521127,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '21': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '22': {'f1-score': 0.4740740740740741,\n",
              "  'precision': 0.3106796116504854,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '23': {'f1-score': 0.7719298245614036,\n",
              "  'precision': 0.88,\n",
              "  'recall': 0.6875,\n",
              "  'support': 32},\n",
              " '24': {'f1-score': 0.6666666666666666,\n",
              "  'precision': 0.5,\n",
              "  'recall': 1.0,\n",
              "  'support': 32},\n",
              " '25': {'f1-score': 0.2890173410404624,\n",
              "  'precision': 0.1773049645390071,\n",
              "  'recall': 0.78125,\n",
              "  'support': 32},\n",
              " '3': {'f1-score': 0.8571428571428571,\n",
              "  'precision': 1.0,\n",
              "  'recall': 0.75,\n",
              "  'support': 32},\n",
              " '4': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '5': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '6': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '7': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " '8': {'f1-score': 1.0, 'precision': 1.0, 'recall': 1.0, 'support': 32},\n",
              " '9': {'f1-score': 0.0, 'precision': 0.0, 'recall': 0.0, 'support': 32},\n",
              " 'accuracy': 0.4411057692307692,\n",
              " 'macro avg': {'f1-score': 0.3526059635884181,\n",
              "  'precision': 0.36537554116764653,\n",
              "  'recall': 0.4411057692307692,\n",
              "  'support': 832},\n",
              " 'weighted avg': {'f1-score': 0.3526059635884181,\n",
              "  'precision': 0.36537554116764653,\n",
              "  'recall': 0.4411057692307692,\n",
              "  'support': 832}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 131
        }
      ]
    }
  ]
}