7_azure_read_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ready to use Azure ML 1.31.0 to work with ws_Machine_Learning\n"
     ]
    }
   ],
   "source": [
    "import azureml.core\r\n",
    "from azureml.core import Workspace\r\n",
    "\r\n",
    "# Load the workspace from the saved config file\r\n",
    "ws = Workspace.from_config()\r\n",
    "print('Ready to use Azure ML {} to work with {}'.format(azureml.core.VERSION, ws.name))\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core import Experiment, Run\r\n",
    "import pandas as pd\r\n",
    "\r\n",
    "\"\"\"\r\n",
    "Available metrics:\r\n",
    "- Patients used for training\r\n",
    "- Patients used for testing\r\n",
    "- Batch dimensions\r\n",
    "- Number of epochs\r\n",
    "- Batch size\r\n",
    "- Patch size\r\n",
    "- Number of K folds\r\n",
    "- Learning rates\r\n",
    "- OACC\r\n",
    "- ACC\r\n",
    "- SEN\r\n",
    "- SPE\r\n",
    "- PRECISSION\r\n",
    "- CONFUSION MATRIX\r\n",
    "- Patients used to classify\r\n",
    "- Predicted GT classification map\r\n",
    "- Predicted and true GT classification maps\r\n",
    "- Model loss and accuracy by epoch\r\n",
    "- Time loading arguments\r\n",
    "- Time preparing train data\r\n",
    "- Time training CNN\r\n",
    "- Time preparing test data\r\n",
    "- Time predicting GT test image\r\n",
    "- Time generating classification maps\r\n",
    "\"\"\"\r\n",
    "\r\n",
    "\"\"\"\r\n",
    "Available experiments:\r\n",
    "- exp-2-PyTorch-2D-CNN-train_noCV\r\n",
    "- exp-2-PyTorch-2D-CNN-train_CV\r\n",
    "- exp-3-PyTorch-2D-CNN-train_CV\r\n",
    "- exp-3-PyTorch-2D-CNN-train_noCV\r\n",
    "- exp-4-PyTorch-2D-CNN-train_CV\r\n",
    "- exp-4-PyTorch-2D-CNN-train_noCV\r\n",
    "\"\"\"\r\n",
    "\r\n",
    "# Create Python list to iterate over all run experiments \r\n",
    "experiments = [#'exp-2-PyTorch-2D-CNN-train_noCV', \r\n",
    "                    'exp-2-PyTorch-2D-CNN-train_CV',\r\n",
    "                    'exp-3-PyTorch-2D-CNN-train_CV',\r\n",
    "                    'exp-3-PyTorch-2D-CNN-train_noCV',\r\n",
    "                    'exp-4-PyTorch-2D-CNN-train_CV',\r\n",
    "                    'exp-4-PyTorch-2D-CNN-train_noCV']\r\n",
    "\r\n",
    "\r\n",
    "column_names = ['Experiment Name', 'Patients used for testing','OACC','ACC_tumor','SEN_tumor','SPE_tumor','Time loading arguments (s)',\r\n",
    "                'Time preparing train data (s)', 'Time training CNN (s)', 'Time preparing test data (s)', 'Time predicting GT test image (s)',\r\n",
    "                    'Time generating classification maps (s)']\r\n",
    "\r\n",
    "\r\n",
    "for experiment_name in experiments:\r\n",
    "\r\n",
    "    # Load experiment\r\n",
    "    experiment = Experiment(ws, name=experiment_name)\r\n",
    "\r\n",
    "    output_csv = './Results/Training_metrics/' + experiment_name + '.csv'\r\n",
    "\r\n",
    "    # Initialize empty dataframe with column names\r\n",
    "    df = pd.DataFrame(columns=column_names)\r\n",
    "                                \r\n",
    "    # Get all runs in the experiment and get their metrics (only if they are completed)\r\n",
    "    for r in list(experiment.get_runs()):\r\n",
    "\r\n",
    "        # Get the run by using the Workspace and the run id (only if the run is completed)\r\n",
    "        if(r.status == 'Completed'):\r\n",
    "            run = Run.get(ws, run_id=r.id)\r\n",
    "\r\n",
    "            # Get all metrics from the run\r\n",
    "            metrics = run.get_metrics()\r\n",
    "\r\n",
    "            #print(metrics)\r\n",
    "\r\n",
    "            df_temp = pd.DataFrame(data=[ [experiment_name, metrics['Patients used for testing'], metrics['OACC'], metrics['ACC'][1], metrics['SEN'][1],\r\n",
    "                                        metrics['SPE'][1], metrics['Time loading arguments (s)'], metrics['Time preparing train data (s)'],\r\n",
    "                                        metrics['Time training CNN (s)'], metrics['Time preparing test data (s)'],\r\n",
    "                                        metrics['Time predicting GT test image (s)'], metrics['Time generating classification maps (s)']] ],\r\n",
    "                                        columns=column_names)\r\n",
    "\r\n",
    "            df = df.append(df_temp, ignore_index=True)\r\n",
    "\r\n",
    "    df.to_csv(output_csv, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "2f8e7f234165e2952df9dbfc489e836223a0dce84478ffe92e6b1e6e49bd3a71"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}