6_Final_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2HXaprVAQEzI"
   },
   "source": [
    "**PART 6 - DSIM project**\n",
    "\n",
    "**Authors:** Francesca De Cola, Valentina Moretto, Valentina Zangirolami\n",
    "\n",
    "In this notebook, we test different structure of ensemble model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Yx_b78_hQEzS"
   },
   "source": [
    "**Load packages**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nDhDa2zeQEzV",
    "outputId": "403d043e-63e3-4988-89c5-0e65de40ef82"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import glob\n",
    "from PIL import Image\n",
    "import cv2\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2 as cv\n",
    "import keras\n",
    "\n",
    "\n",
    "from keras_vggface.vggface import VGGFace\n",
    "from keras_vggface.utils import preprocess_input\n",
    "from keras_vggface import utils\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "from keras.models import Model, load_model\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing import image\n",
    "\n",
    "import tensorflow as tf\n",
    "import pickle\n",
    "import random "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d-clzSIyQEzc"
   },
   "source": [
    "**Seed for reproducibility**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZcwCtT2sQEze"
   },
   "outputs": [],
   "source": [
    "SEED = 42\n",
    "np.random.seed(SEED)\n",
    "random.seed(SEED)\n",
    "tf.compat.v1.set_random_seed(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5nUWBQnxQEzg"
   },
   "source": [
    "**Load dataset of images**\n",
    "\n",
    "(We create this npz file in 1_creation_dataset.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "44WN3odUQEzi"
   },
   "outputs": [],
   "source": [
    "data=np.load(\"C:/Users/valen/Desktop/magistrale/DSIM/dataset_imgs.npz\")\n",
    "label=data['Y']\n",
    "img=data['X']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7LEqnIDMQEzk"
   },
   "source": [
    "**Preprocessing labels and images**\n",
    "\n",
    "(Same at previous notebook, Fine_tuning_4 and CNN_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xBakDceKQEzm"
   },
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "label=le.fit_transform(label)\n",
    "img = img.astype('float32')\n",
    "img = preprocess_input(img, version = 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d5FPmUHsQEzp"
   },
   "source": [
    "**Train-Validation-Test partition**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VVHoVQ89QEzr"
   },
   "source": [
    "We use the same partion of previous notebook, we set the same seed to obtain the same test set for compare results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZhJCgi77QEzt",
    "outputId": "fec5eaf5-96f6-477b-9cf7-9dfb06c271bd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1452, 224, 224, 3), (363, 224, 224, 3), (454, 224, 224, 3))"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(img, label, test_size = .2, random_state = SEED, stratify = label)\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = .2, random_state = SEED, stratify = y_train)\n",
    "x_train.shape, x_val.shape, x_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jrCesP42QEzv"
   },
   "source": [
    "We define *y_eff* to save test labels for classification_report (and confusion matrix). After, we complete the preprocessing phase for labels with to_categorical function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9BvLbcXBQEzw"
   },
   "outputs": [],
   "source": [
    "y_eff=y_test\n",
    "y_train= to_categorical(y_train, num_classes=7)\n",
    "y_val= to_categorical(y_val, num_classes=7)\n",
    "y_test= to_categorical(y_test, num_classes=7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IlSWj9E1QEzz"
   },
   "source": [
    "**Ensemble Model**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jvrsS8mdQEzz"
   },
   "source": [
    "Initially, we load four models:\n",
    "* model_ft1 and model_ft2: refers to a model in which we apply fine tuning techniques. They represents the two best model of the scripts Fine_tuning_4.ipynb\n",
    "* model_cnn1: refers to our CNN with the best cut of vggface (layer: add_12). It is the last model of the scripts CNN_5\n",
    "* base_model: it is the pretrained neural network VGGFace. It is necessary for model_cnn1, because we use this to extract features of images before model_cnn1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uw93LzY5QEz1",
    "outputId": "54754bbb-ff5a-4dce-8c3d-12e2f254d928"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:131: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:1834: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\valen\\AppData\\Roaming\\Python\\Python37\\site-packages\\keras\\backend\\tensorflow_backend.py:3980: The name tf.nn.avg_pool is deprecated. Please use tf.nn.avg_pool2d instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model1=tf.keras.models.load_model('C:/Users/valen/Desktop/magistrale/DSIM/aml/weights-tmp4.best.hdf5')\n",
    "model2=tf.keras.models.load_model('C:/Users/valen/Desktop/magistrale/DSIM/aml/weights-tmp3.best.hdf5')\n",
    "model_cnn1=load_model('C:/Users/valen/Desktop/magistrale/DSIM/aml_cnn/cnn_best.h5')\n",
    "base_model = VGGFace(include_top = False, input_shape = (224, 224, 3), model='senet50')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uDO_IqEaQEz3"
   },
   "outputs": [],
   "source": [
    "#feature extraction\n",
    "model = Model(inputs=base_model.input, outputs=base_model.get_layer('add_12').output)\n",
    "extract_features = model.predict(x_test)\n",
    "#prediction for each model\n",
    "pred_cnn = model_cnn1.predict(extract_features)\n",
    "pred1= model1.predict(x_test)\n",
    "pred2= model2.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-1CLvH0cQEz4"
   },
   "source": [
    "We define two initial structure of ensamble:\n",
    "1. First structure, save in final_pred_1, consider a mean with same weight for each prediction\n",
    "2. In final_pred_2, We consider a weighted mean. We define weigths that take into account performance of each models. *model1* and *model_cnn* are the best models that have registred 91% of accuracy in the test phase, for this reasion we assign an 'high' value in ensemble. In different way, *model2* have 89% of accuracy on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "K3khscw_QEz5"
   },
   "outputs": [],
   "source": [
    "final_pred_1= 0.33*(pred1+pred2 + pred_cnn)\n",
    "final_pred_2= 0.2*pred2 + 0.3*pred1 + 0.5*pred_cnn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sfcQSVIKQEz6"
   },
   "source": [
    "**Classification report/Confusion Matrix Ensemble 1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hO1Q3UXyQEz6",
    "outputId": "ee2dd464-304b-4a5c-854f-c31742d552f2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.94      0.87        64\n",
      "           1       0.91      0.83      0.87        60\n",
      "           2       0.89      0.90      0.90        63\n",
      "           3       1.00      0.98      0.99        64\n",
      "           4       0.90      0.93      0.91        69\n",
      "           5       0.97      0.81      0.88        73\n",
      "           6       0.89      0.97      0.93        61\n",
      "\n",
      "    accuracy                           0.91       454\n",
      "   macro avg       0.91      0.91      0.91       454\n",
      "weighted avg       0.91      0.91      0.91       454\n",
      "\n",
      "Confusion matrix:\n",
      "[[60  3  0  0  0  1  0]\n",
      " [10 50  0  0  0  0  0]\n",
      " [ 0  0 57  0  1  0  5]\n",
      " [ 0  1  0 63  0  0  0]\n",
      " [ 0  0  3  0 64  1  1]\n",
      " [ 3  1  4  0  5 59  1]\n",
      " [ 1  0  0  0  1  0 59]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1f72cfe01d0>"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAC0lJREFUeJzt3X+onQUdx/HPZ/fOzc3phnfY2B1OYQhi5WytYiC1TGaKGvSHgkIRjEBjkmXaP+GfEZgVEsi2MvwxxLkwtamgy4SmbnOmc1prTLxMuffqTJeg3e3bH/dZnOal82zn+XH87v2Cyz3n3mfP97t77+c8P845z9cRIQA5TWu7AQD1IeBAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIbrGOlnj47PHNuHavu6jNLPtVKXUkasFurfSJr87WYbf3GX399n8bHx7uWryfgM+dqxue+W8equ3rykVtaqStJs2bU8uNEFxOHDrdWe3CgnZ3gFV9YVmo5dtGBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHECDiQWKmA215l+zXbe2zfXHdTAKrRNeC2ByTdIekSSedKutr2uXU3BqB3ZbbgyyXtiYi9EfGRpA2Srqi3LQBVKBPwhZLe6Lg/UnwNQJ8r8/7Gqd5z+rG34NpeLWm1JGnGab11BaASZbbgI5IWddwflrT/6IUi4s6IWBYRyzx9dlX9AehBmYA/L2mJ7bNsnyTpKkkP1dsWgCp03UWPiAnb10t6TNKApPURsav2zgD0rNQ1hiLiUUmP1twLgIrxSjYgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kVss4zHPPPkMbN9xYx6q7+vQP/tBKXUn6x6++0VrtE1lbEz4/CfjJAIkRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRWZrroetujtl9uoiEA1SmzBf+tpFU19wGgBl0DHhFPS3qngV4AVIxjcCCxygJue7Xtbba3HXh7vKrVAuhBZQHvHB887/ShqlYLoAfsogOJlXma7D5Jf5F0ju0R29+pvy0AVSgzH/zqJhoBUD120YHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJBYLeODZwxO05lDs+pYdVdtjvCdd+UdrdU+8PvrWqstSROHDrdWu83xwf/84N+t1J04HKWWYwsOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kFiZ66Ivsv2U7d22d9le00RjAHpX5s0mE5JujIgdtudI2m77iYh4pebeAPSozPjgNyNiR3H7fUm7JS2suzEAvTumY3DbiyUtlfRsHc0AqFbpgNs+RdJGSTdExHtTfP+/44PHxseq7BHAcSoVcNvTNRnueyLiwamW6RwfPH9ofpU9AjhOZc6iW9I6Sbsj4rb6WwJQlTJb8BWSrpW00vbO4uPrNfcFoAJlxgc/I8kN9AKgYrySDUiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4nVMj64TW2OsW1zhO+8L97QWm1JOrD19lbrt+W0WdNbqTs4rdyrx9mCA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRWZvDBTNvP2X6xGB98axONAehdmTebfChpZUQcLEYYPWP7jxGxtebeAPSozOCDkHSwuDu9+Ig6mwJQjbLDBwds75Q0KumJiGB8MPAJUCrgEXEoIs6XNCxpue3zjl6G8cFA/zmms+gR8a6kLZJWTfE9xgcDfabMWfT5tucWt0+WdJGkV+tuDEDvypxFXyDpLtsDmnxAuD8iHq63LQBVKHMW/a+SljbQC4CK8Uo2IDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcTSzQcfHGjvMeuDDydaq932fO55n7++tdpjW3/ZWu22lL2kEltwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJFY64MV8shdsc0104BPiWLbgayTtrqsRANUrO110WNKlktbW2w6AKpXdgt8u6SZJh2vsBUDFygwfvEzSaERs77Ic44OBPlNmC75C0uW290naIGml7buPXojxwUD/6RrwiLglIoYjYrGkqyQ9GRHX1N4ZgJ7xPDiQ2DFdsikitkjaUksnACrHFhxIjIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgsVrGBx+KaG2U7kmD7T1mHTpcdqhrPvv+9PPWas+/9Get1R575Iet1S6DLTiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYqVei16MLXpf0iFJExGxrM6mAFTjWN5s8pWIGK+tEwCVYxcdSKxswEPS47a321491QKd44PfHmdDD/SDsgFfEREXSLpE0nW2Lzx6gc7xwacPDVXaJIDjUyrgEbG/+DwqaZOk5XU2BaAaXQNue7btOUduS7pY0st1Nwagd2XOop8haZPtI8vfGxGba+0KQCW6Bjwi9kr6bAO9AKgYT5MBiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgsVrGB0+zWxvjOzjQ3mPWnJNP3MfL2TMGWqt9YPOPWqs9b9VPW6n74d/fKrXcifsXCZwACDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSKxVw23NtP2D7Vdu7bX+p7sYA9K7sm01+IWlzRHzT9kmSZtXYE4CKdA247VMlXSjpW5IUER9J+qjetgBUocwu+tmSxiT9xvYLttcWM8r+R+f44PGxscobBXDsygR8UNIFkn4dEUsl/UvSzUcv1Dk+eGj+/IrbBHA8ygR8RNJIRDxb3H9Ak4EH0Oe6Bjwi3pL0hu1zii99VdIrtXYFoBJlz6J/T9I9xRn0vZK+XV9LAKpSKuARsVPSspp7AVAxXskGJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxBwR1a/UHpP0+nH+8yFJ4xW2Q21qZ6x9ZkR0fV92LQHvhe1tEdHK696pTe1stdlFBxIj4EBi/RjwO6lNbWpXo++OwQFUpx+34AAq0lcBt73K9mu299j+2JVba6y73vao7ZebqtlRe5Htp4qJMbtsr2mw9kzbz9l+sah9a1O1O3oYKC7H/XDDdffZfsn2TtvbGq7d2KSgvtlFtz0g6W+SvqbJK7k+L+nqiKj9Ao+2L5R0UNLvIuK8uusdVXuBpAURscP2HEnbJV3Z0P/bkmZHxEHb0yU9I2lNRGytu3ZHD9/X5OXATo2Iyxqsu0/Ssoho/Hlw23dJ+nNErD0yKSgi3q2jVj9twZdL2hMRe4vpKRskXdFE4Yh4WtI7TdSaovabEbGjuP2+pN2SFjZUOyLiYHF3evHR2CO+7WFJl0pa21TNtnVMClonTU4KqivcUn8FfKGkNzruj6ihP/R+YXuxpKWSnv3/S1Zac8D2Tkmjkp7ouP59E26XdJOkww3WPCIkPW57u+3VDdYtNSmoKv0UcE/xtf44fmiA7VMkbZR0Q0S811TdiDgUEedLGpa03HYjhyi2L5M0GhHbm6g3hRURcYGkSyRdVxymNaHUpKCq9FPARyQt6rg/LGl/S700qjj+3Sjpnoh4sI0eit3ELZJWNVRyhaTLi2PhDZJW2r67odqKiP3F51FJmzR5iNiERicF9VPAn5e0xPZZxYmHqyQ91HJPtStOdK2TtDsibmu49nzbc4vbJ0u6SNKrTdSOiFsiYjgiFmvyd/1kRFzTRG3bs4sTmip2jy+W1MgzKE1PCio72aR2ETFh+3pJj0kakLQ+InY1Udv2fZK+LGnI9oikn0TEuiZqa3JLdq2kl4pjYUn6cUQ82kDtBZLuKp7BmCbp/oho9OmqlpwhadPkY6sGJd0bEZsbrN/YpKC+eZoMQPX6aRcdQMUIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4k9h+EQuscTUoPzgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred_1=np.argmax(final_pred_1, axis=1)\n",
    "\n",
    "print('Classification report:')\n",
    "print(classification_report(y_eff, y_pred_1))\n",
    "\n",
    "print('Confusion matrix:')\n",
    "cm=confusion_matrix(y_eff, y_pred_1)\n",
    "print(cm)\n",
    "plt.imshow(cm, cmap=plt.cm.Blues)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "58nunYRkQEz8"
   },
   "source": [
    "Performance are very confortable, Accuracy and other indicator are very high. We note, only, that classes 0,2 and 5 registered less performance than other classes. In fact, if we observe a confusion matrix we note that in corrispondence of these classes we have more 'false negative'."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ULApuI0PQEz9"
   },
   "source": [
    "**Classification report/Confusion Matrix Ensemble 2**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Hhu1cQoVQEz9",
    "outputId": "3c2f9e42-76ed-4942-a026-c6b6041f3e40"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.91      0.87        64\n",
      "           1       0.90      0.87      0.88        60\n",
      "           2       0.92      0.90      0.91        63\n",
      "           3       1.00      0.98      0.99        64\n",
      "           4       0.90      0.94      0.92        69\n",
      "           5       0.95      0.84      0.89        73\n",
      "           6       0.89      0.97      0.93        61\n",
      "\n",
      "    accuracy                           0.91       454\n",
      "   macro avg       0.92      0.92      0.91       454\n",
      "weighted avg       0.92      0.91      0.91       454\n",
      "\n",
      "Confusion matrix:\n",
      "[[58  4  0  0  0  2  0]\n",
      " [ 8 52  0  0  0  0  0]\n",
      " [ 0  0 57  0  1  0  5]\n",
      " [ 0  1  0 63  0  0  0]\n",
      " [ 0  0  2  0 65  1  1]\n",
      " [ 2  1  3  0  5 61  1]\n",
      " [ 1  0  0  0  1  0 59]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1f72d062898>"
      ]
     },
     "execution_count": 11,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAACzlJREFUeJzt3X+onnUZx/HPx7Odtjlr0w0ZO6MpiGBSTtYiJlLLZObQ/jBQUCii9YfGrMA0ihD6qz/MiBBsWxr+GKYuRMxf6LJBTrc50zmNMSYeZp0df6QrcG27+uPci9M89Hznc/94vHy/4LDnOefefV07O59z/3ie+74cEQKQ0wldNwCgOQQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBi05pYqYdnh2ed3MSqe/r06fM7qStJQ3ZntT/KjnRYu6st5Kuv7tX4+HjPH7hmAj7rZH3svB80seqeHt/w7U7qStLsGY18O9HDwUPdRXx4WjcRX/65pUXLsYsOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxIoCbnul7Vds77Z9fdNNAahHz4DbHpL0K0kXSTpL0hW2z2q6MQD9K9mCL5O0OyL2RMRBSRskXdpsWwDqUBLwhZJem/R8tPocgAFXcn3jVNecvm8ciu3VklZLkmbO7a8rALUo2YKPSlo06fmIpH3HLhQRt0bE0ohY6uHZdfUHoA8lAX9W0hm2T7M9LOlySQ802xaAOvTcRY+IQ7avkfSIpCFJ6yNiZ+OdAehb0T2GIuIhSQ813AuAmvFONiAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiTWyDjMTy2erwdu+2YTq+7p7O/+vpO6krT3lss6q/1R1tWEzw8DvjNAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiZVMF11ve8z2i200BKA+JVvw2yStbLgPAA3oGfCIeErSmy30AqBmHIMDidUWcNurbW+1vfXNN/bXtVoAfagt4JPHB598yvy6VgugD+yiA4mVvEx2t6Q/SzrT9qjtbu7FBOC4lcwHv6KNRgDUj110IDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJNbI+ODhIWvBnBlNrLqnLkf4zv3arzur/dbvvtVZbUk6fCQ6qz10gjur/Y9//buTuocKv99swYHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSK7kv+iLbT9reZXun7TVtNAagfyUXmxyS9P2I2G77JEnbbD8WES813BuAPpWMD349IrZXj9+VtEvSwqYbA9C/4zoGt71Y0hJJW5poBkC9igNue7ak+yRdGxHvTPH1/44P3j/O+GBgEBQF3PZ0TYT7zoi4f6plJo8Pnj+P8cHAICg5i25J6yTtioibmm8JQF1KtuDLJV0laYXtHdXHVxruC0ANSsYHb5bU3U2vAHxgvJMNSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiTUyPrhLXY6x7XKE79zzruustiS9tflnndbvyidmTe+k7rTCkclswYHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSKxl8MMP2M7afr8YH39hGYwD6V3KxyXuSVkTEgWqE0Wbbf4iIpxvuDUCfSgYfhKQD1dPp1Ud3l2wBKFY6fHDI9g5JY5IeiwjGBwMfAkUBj4jDEXGOpBFJy2yffewyjA8GBs9xnUWPiLclbZK0coqvMT4YGDAlZ9Hn255TPZ4p6QJJLzfdGID+lZxFXyDpdttDmviFcE9EPNhsWwDqUHIW/S+SlrTQC4Ca8U42IDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcTSzQcfKpyb3ISDh450Vrvr+dxzP3tNZ7XHt/yys9pdKb2lEltwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJFYc8Go+2XO2uSc68CFxPFvwNZJ2NdUIgPqVThcdkXSxpLXNtgOgTqVb8JslXSepu8ulABy3kuGDqySNRcS2HssxPhgYMCVb8OWSLrG9V9IGSSts33HsQowPBgZPz4BHxA0RMRIRiyVdLumJiLiy8c4A9I3XwYHEjuuWTRGxSdKmRjoBUDu24EBiBBxIjIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJNTI++Ii6G6Xb5fjgw0dKh7rms/ePP++s9rwLf9pZ7fFHf9RZ7RJswYHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSK3ovejW26F1JhyUdioilTTYFoB7Hc7HJFyNivLFOANSOXXQgsdKAh6RHbW+zvXqqBSaPD35jP+ODgUFQGvDlEXGupIskXW37/GMXmDw++JT5jA8GBkFRwCNiX/XnmKSNkpY12RSAevQMuO0TbZ909LGkCyW92HRjAPpXchb9VEkbbR9d/q6IeLjRrgDUomfAI2KPpM+00AuAmvEyGZAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxJrZHyw1d0Y3y7HB88cHuqsdtdmz2jkR6nIW4//uLPac1d1Mzb5vd1/L1qOLTiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYkUBtz3H9r22X7a9y/bnm24MQP9KrxD4haSHI+Iy28OSZjXYE4Ca9Ay47Y9LOl/S1yUpIg5KOthsWwDqULKLfrqk/ZJ+Y/s522urGWX/Y/L44PFxxgcDg6Ak4NMknSvplohYIumfkq4/dqHJ44PnzWN8MDAISgI+Kmk0IrZUz+/VROABDLieAY+Iv0l6zfaZ1ae+JOmlRrsCUIvSs+jfkXRndQZ9j6RvNNcSgLoUBTwidkha2nAvAGrGO9mAxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHECDiQmCOi/pXa+yW9+gH/+jxJ4zW2Q21qZ6z9yYjoeV12IwHvh+2tEdHJ+96pTe1stdlFBxIj4EBigxjwW6lNbWrXY+COwQHUZxC34ABqMlABt73S9iu2d9t+351bG6y73vaY7Rfbqjmp9iLbT1YTY3baXtNi7Rm2n7H9fFX7xrZqT+phqLod94Mt191r+wXbO2xvbbl2a5OCBmYX3faQpL9K+rIm7uT6rKQrIqLxGzzaPl/SAUm/jYizm653TO0FkhZExHbbJ0naJumrLf27LenEiDhge7qkzZLWRMTTTdee1MP3NHE7sI9HxKoW6+6VtDQiWn8d3Pbtkv4UEWuPTgqKiLebqDVIW/BlknZHxJ5qesoGSZe2UTginpL0Zhu1pqj9ekRsrx6/K2mXpIUt1Y6IOFA9nV59tPYb3/aIpIslrW2rZtcmTQpaJ01MCmoq3NJgBXyhpNcmPR9VSz/og8L2YklLJG35/0vWWnPI9g5JY5Iem3T/+zbcLOk6SUdarHlUSHrU9jbbq1usWzQpqC6DFHBP8bnBOH5oge3Zku6TdG1EvNNW3Yg4HBHnSBqRtMx2K4cotldJGouIbW3Um8LyiDhX0kWSrq4O09pQNCmoLoMU8FFJiyY9H5G0r6NeWlUd/94n6c6IuL+LHqrdxE2SVrZUcrmkS6pj4Q2SVti+o6Xaioh91Z9jkjZq4hCxDa1OChqkgD8r6Qzbp1UnHi6X9EDHPTWuOtG1TtKuiLip5drzbc+pHs+UdIGkl9uoHRE3RMRIRCzWxP/1ExFxZRu1bZ9YndBUtXt8oaRWXkFpe1JQ6WSTxkXEIdvXSHpE0pCk9RGxs43atu+W9AVJ82yPSvpJRKxro7YmtmRXSXqhOhaWpB9GxEMt1F4g6fbqFYwTJN0TEa2+XNWRUyVtnPjdqmmS7oqIh1us39qkoIF5mQxA/QZpFx1AzQg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiT2H+Na6Q99i0mzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred_2=np.argmax(final_pred_2, axis=1)\n",
    "\n",
    "print('Classification report:')\n",
    "print(classification_report(y_eff, y_pred_2))\n",
    "\n",
    "print('Confusion matrix:')\n",
    "cm=confusion_matrix(y_eff, y_pred_2)\n",
    "print(cm)\n",
    "plt.imshow(cm, cmap=plt.cm.Blues)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eyYdvj-zQEz_"
   },
   "source": [
    "The difference between the first ensemble model is really minimal. We obtain the same value of accuracy, but it is possible to notice that other metrics (recall, precision and f1-score) are more balanced. We think that this model is better than the first, it is possible to note that the problems highlighted above remain but seem to be less relevant."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "e_s0eBU5QE0A"
   },
   "source": [
    "**Classification report/Confusion Matrix Ensemble 3**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LKO-rmm7QE0B"
   },
   "source": [
    "We define another structure of ensemble in which we give much more importance to *model1* and *model_cnn*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7tXoojhRQE0C",
    "outputId": "2d19b8da-f932-4d71-ac44-a6f70698bf7e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.92      0.89        64\n",
      "           1       0.91      0.87      0.89        60\n",
      "           2       0.95      0.90      0.93        63\n",
      "           3       1.00      0.98      0.99        64\n",
      "           4       0.90      0.94      0.92        69\n",
      "           5       0.94      0.86      0.90        73\n",
      "           6       0.89      0.97      0.93        61\n",
      "\n",
      "    accuracy                           0.92       454\n",
      "   macro avg       0.92      0.92      0.92       454\n",
      "weighted avg       0.92      0.92      0.92       454\n",
      "\n",
      "Confusion matrix:\n",
      "[[59  4  0  0  0  1  0]\n",
      " [ 7 52  0  0  0  1  0]\n",
      " [ 0  0 57  0  1  0  5]\n",
      " [ 0  1  0 63  0  0  0]\n",
      " [ 0  0  1  0 65  2  1]\n",
      " [ 2  0  2  0  5 63  1]\n",
      " [ 1  0  0  0  1  0 59]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1f72d0b8fd0>"
      ]
     },
     "execution_count": 12,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAACzhJREFUeJzt3VuIXfUVx/HfL5PExERrNFFCJjS2iCDSGklTJEVsaiXWoH2woKDQUpqHqsRaEO1L8bE+WFsogk3SWrwEq6aIWDVFUxvwlsRYjdESQsQh1sl4qaYF0ySrD7NTpnHo2fHs/97H5fcDw5wzs3PWysz5zb6cs/dyRAhATlO6bgBAOQQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiU0s8qKfPCs88ucRD9/SlL57aSV1JGrI7q/1Z1uV7Mbv6jb/xxh6NjY31LF8m4DNP1nHn3VDioXv60wPXdVJXkmbPKPLjRA+HDncX8aEp3UR82VeX1FqOTXQgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kVivgtlfYft32Lts3lW4KQDN6Btz2kKRfSbpY0lmSrrR9VunGAPSvzhp8qaRdEbE7Ig5IWi/psrJtAWhCnYAvkPTmhPsj1dcADLg65zdOdj7cx87Ps71K0ipJ0ow5/XUFoBF11uAjkhZOuD8sae/RC0XEnRGxJCKWePqspvoD0Ic6AX9B0hm2T7c9XdIVkh4u2xaAJvTcRI+Ig7avlfS4pCFJ6yJiR/HOAPSt1jWGIuJRSY8W7gVAw3gnG5AYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIrMg7z7NNP1SP3/rDEQ/eu/aM/dFJXkvbccXlntYHJsAYHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSKzOdNF1tkdtv9JGQwCaU2cN/ltJKwr3AaCAngGPiKclvdtCLwAaxj44kFhjAbe9yvYW21vefWdfUw8LoA+NBXzi+OCTT5nX1MMC6AOb6EBidV4mu0/SM5LOtD1i+/vl2wLQhDrzwa9soxEAzWMTHUiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kVGR88bcg69cTjSjx0T12O8J3znV93Vvu93/+gs9qSdOhwdFZ7aIo7q/2Pf/27k7oHa/68WYMDiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJFbnuugLbT9le6ftHbZXt9EYgP7VOdnkoKQfR8Q22ydI2mp7Y0S8Wrg3AH2qMz74rYjYVt3+UNJOSQtKNwagf8e0D257kaTFkp4r0QyAZtUOuO3Zkh6UdH1EfDDJ9/87PnjfGOODgUFQK+C2p2k83PdExEOTLTNxfPC8uYwPBgZBnaPolrRW0s6IuK18SwCaUmcNvkzS1ZKW295efXyrcF8AGlBnfPBmSd1d9ArAJ8Y72YDECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJBYkfHBXepyjG2XI3znfO3GzmpL0nubb+20flc+d/y0TupOrTkymTU4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGJ1Bh/MsP287Zeq8cG3tNEYgP7VOdnkI0nLI2J/NcJos+0/RsSzhXsD0Kc6gw9C0v7q7rTqo7tTtgDUVnf44JDt7ZJGJW2MCMYHA58CtQIeEYci4hxJw5KW2j776GUYHwwMnmM6ih4R70vaJGnFJN9jfDAwYOocRZ9n+6Tq9kxJF0p6rXRjAPpX5yj6fEl32R7S+B+E+yPikbJtAWhCnaPof5W0uIVeADSMd7IBiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJJZuPvhQzbnJJXQ6m7zj+dxzvnJtZ7XffuaXndXu6vlW95nGGhxIjIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJ1Q54NZ/sRdtcEx34lDiWNfhqSTtLNQKgeXWniw5LukTSmrLtAGhS3TX47ZJulHS4YC8AGlZn+OBKSaMRsbXHcowPBgZMnTX4MkmX2t4jab2k5bbvPnohxgcDg6dnwCPi5ogYjohFkq6Q9GREXFW8MwB943VwILFjumRTRGyStKlIJwAaxxocSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwILEi44MPSzpwsJtrQ0yf2t3frC7HB3c5NlmS9vz5553VPu2CmzqrPfb0zzqrXQdrcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHEar0XvRpb9KGkQ5IORsSSkk0BaMaxnGzy9YgYK9YJgMaxiQ4kVjfgIekJ21ttr5psgYnjg9/Zx/hgYBDUDfiyiDhX0sWSrrF9/tELTBwffMo8xgcDg6BWwCNib/V5VNIGSUtLNgWgGT0DbnuW7ROO3JZ0kaRXSjcGoH91jqKfJmmD7SPL3xsRjxXtCkAjegY8InZL+nILvQBoGC+TAYkRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwILEi44Ot7sfZdqHL0cVdmz2jyFOplvc239pZ7Tkruxmb/NGut2st99l9RgKfAQQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADidUKuO2TbD9g+zXbO22fV7oxAP2re4bALyQ9FhGX254u6fiCPQFoSM+A2z5R0vmSvitJEXFA0oGybQFoQp1N9C9I2ifpN7ZftL2mmlH2PyaODx4bY3wwMAjqBHyqpHMl3RERiyX9U9JNRy80cXzw3LmMDwYGQZ2Aj0gaiYjnqvsPaDzwAAZcz4BHxN8lvWn7zOpL35D0atGuADSi7lH06yTdUx1B3y3pe+VaAtCUWgGPiO2SlhTuBUDDeCcbkBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHECDiQGAEHEnNENP+g9j5Jb3zCfz5X0liD7VCb2hlrfz4iep6XXSTg/bC9JSI6ed87tamdrTab6EBiBBxIbBADfie1qU3tZgzcPjiA5gziGhxAQwYq4LZX2H7d9i7bH7tya8G662yP2n6lrZoTai+0/VQ1MWaH7dUt1p5h+3nbL1W1b2mr9oQehqrLcT/Sct09tl+2vd32lpZrtzYpaGA20W0PSfqbpG9q/EquL0i6MiKKX+DR9vmS9kv6XUScXbreUbXnS5ofEdtsnyBpq6Rvt/T/tqRZEbHf9jRJmyWtjohnS9ee0MMNGr8c2IkRsbLFunskLYmI1l8Ht32XpL9ExJojk4Ii4v0StQZpDb5U0q6I2F1NT1kv6bI2CkfE05LebaPWJLXfioht1e0PJe2UtKCl2hER+6u706qP1v7i2x6WdImkNW3V7NqESUFrpfFJQaXCLQ1WwBdIenPC/RG19EQfFLYXSVos6bn/v2SjNYdsb5c0KmnjhOvft+F2STdKOtxizSNC0hO2t9pe1WLdWpOCmjJIAfckXxuM/YcW2J4t6UFJ10fEB23VjYhDEXGOpGFJS223sotie6Wk0YjY2ka9SSyLiHMlXSzpmmo3rQ21JgU1ZZACPiJp4YT7w5L2dtRLq6r93wcl3RMRD3XRQ7WZuEnSipZKLpN0abUvvF7Sctt3t1RbEbG3+jwqaYPGdxHb0OqkoEEK+AuSzrB9enXg4QpJD3fcU3HVga61knZGxG0t155n+6Tq9kxJF0p6rY3aEXFzRAxHxCKN/66fjIir2qhte1Z1QFPV5vFFklp5BaXtSUF1J5sUFxEHbV8r6XFJQ5LWRcSONmrbvk/SBZLm2h6R9NOIWNtGbY2vya6W9HK1LyxJP4mIR1uoPV/SXdUrGFMk3R8Rrb5c1ZHTJG0Y/9uqqZLujYjHWqzf2qSggXmZDEDzBmkTHUDDCDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJPYfHqnpLhnaPHkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "final_pred_3 = 0.1*pred2 + 0.4*pred1 + 0.5*pred_cnn\n",
    "y_pred_3=np.argmax(final_pred_3, axis=1)\n",
    "\n",
    "print('Classification report:')\n",
    "print(classification_report(y_eff, y_pred_3))\n",
    "\n",
    "print('Confusion matrix:')\n",
    "cm=confusion_matrix(y_eff, y_pred_3)\n",
    "print(cm)\n",
    "plt.imshow(cm, cmap=plt.cm.Blues)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1ZNiqoi0QE0E"
   },
   "source": [
    "Accuracy value is more high. It is possible to observe that the problem is less relevant. We consider this model, our final model. In the following notebook, we consider this model for the demo."
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Final_model_6.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}