Model Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array\n",
    "from keras.models import Sequential, load_model\n",
    "\n",
    "img_width, img_height = 224, 224\n",
    "model_path = './models/MelanomaModel.h5'\n",
    "model_weights_path = './models/MelanomaWeights.h5'\n",
    "model = load_model(model_path)\n",
    "model.load_weights(model_weights_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 90 images belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "batch_size=16\n",
    "test_dir = './dataset/test'\n",
    "check_datagen = ImageDataGenerator(\n",
    "    rescale=1. / 255)\n",
    "\n",
    "check_generator = check_datagen.flow_from_directory(\n",
    "    test_dir,\n",
    "    target_size=(img_height, img_width),\n",
    "    batch_size=batch_size,\n",
    "    shuffle=False,\n",
    "    class_mode='categorical')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.625\n"
     ]
    }
   ],
   "source": [
    "y_img_batch, y_class_batch = check_generator[5] \n",
    "y_pred = np.argmax(model.predict(y_img_batch),-1)\n",
    "y_true = np.argmax(y_class_batch,-1)\n",
    "print(sum(y_pred==y_true)/batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "      benign       0.92      0.92      0.92        13\n",
      "   malignant       0.67      0.67      0.67         3\n",
      "\n",
      "    accuracy                           0.88        16\n",
      "   macro avg       0.79      0.79      0.79        16\n",
      "weighted avg       0.88      0.88      0.88        16\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_true, y_pred,target_names=check_generator.class_indices.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "tn, fp, fn, tp = cm.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[12,  1],\n",
       "       [ 1,  2]], dtype=int64)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8777777777777778"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_generator.reset()\n",
    "Y_pred = model.predict_generator(check_generator)\n",
    "classes = check_generator.classes[check_generator.index_array]\n",
    "y_pred = np.argmax(Y_pred, axis=-1)\n",
    "sum(y_pred==classes)/90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[43,  7],\n",
       "       [ 4, 36]], dtype=int64)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(check_generator.classes[check_generator.index_array],y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8777777777777778"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true = check_generator.classes[check_generator.index_array]\n",
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(y_true,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "y_true = check_generator.classes[check_generator.index_array]\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "false_positive_rate = fp / (fp + tn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "false_positive_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp,fp,fn,tn = confusion_matrix(y_true,y_pred).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "false_negative_rate = fn / (tp + fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0851063829787234"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "false_negative_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8372093023255814"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "true_negative_rate = tn / (tn + fp)\n",
    "true_negative_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negative_predictive_value = tn/ (tn + fn)\n",
    "negative_predictive_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9\n",
      "0.9\n"
     ]
    }
   ],
   "source": [
    "#TP RATE || Recall || Sensitivity\n",
    "from sklearn.metrics import confusion_matrix, recall_score\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "true_positive_rate = tp / (tp + fn)\n",
    "print(true_positive_rate)\n",
    "# or simply\n",
    "print(recall_score(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8372093023255814\n",
      "0.8372093023255814\n"
     ]
    }
   ],
   "source": [
    "##PPV || Precision\n",
    "from sklearn.metrics import confusion_matrix, precision_score\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "positive_predictive_value = tp/ (tp + fp)\n",
    "# or simply\n",
    "print(positive_predictive_value)\n",
    "print(precision_score(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8777777777777778"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "accuracy = (tp + tn) / (tp + fp + fn + tn)\n",
    "# or simply\n",
    "accuracy_score(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7543424317617866"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import cohen_kappa_score\n",
    "cohen_kappa_score(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    " \n",
    "roc_auc = roc_auc_score(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.88"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}