iccm2019.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import run_info_utils\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_partial_df(df, n_rows):\n",
    "    if df.shape[0] < n_rows:\n",
    "        return df\n",
    "    else:\n",
    "        return df[:n_rows]\n",
    "    \n",
    "def preprocess(df):\n",
    "    h_dim_list = list()\n",
    "    for i in range(df.shape[0]):\n",
    "        h_dim = df['hidden_dimensions'].iloc[i][0]\n",
    "        h_dim_list.append(h_dim)\n",
    "    df['hidden_dimension'] = h_dim_list\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = run_info_utils.get_df_run_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = preprocess(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>all_correct_stop</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>carry_list</th>\n",
       "      <th>confidence_prob</th>\n",
       "      <th>dev/carry-0/early_stopping/accuracy</th>\n",
       "      <th>dev/carry-1/early_stopping/accuracy</th>\n",
       "      <th>dev/carry-2/early_stopping/accuracy</th>\n",
       "      <th>dev/carry-3/early_stopping/accuracy</th>\n",
       "      <th>dev/carry-4/early_stopping/accuracy</th>\n",
       "      <th>dev/early_stopping/max_correct_answer_step</th>\n",
       "      <th>...</th>\n",
       "      <th>time/running_time</th>\n",
       "      <th>time/start_time</th>\n",
       "      <th>train_dev_test_ratio</th>\n",
       "      <th>train_set_size</th>\n",
       "      <th>train_set_size/carry-0</th>\n",
       "      <th>train_set_size/carry-1</th>\n",
       "      <th>train_set_size/carry-2</th>\n",
       "      <th>train_set_size/carry-3</th>\n",
       "      <th>train_set_size/carry-4</th>\n",
       "      <th>hidden_dimension</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>32</td>\n",
       "      <td>[0, 1, 2, 3, 4]</td>\n",
       "      <td>0.9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>00:29:54.329538</td>\n",
       "      <td>2019-03-29 22:22:05.621229</td>\n",
       "      <td>(1.0, 0, 0)</td>\n",
       "      <td>256</td>\n",
       "      <td>81.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 212 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   all_correct_stop  batch_size       carry_list  confidence_prob  \\\n",
       "0              True          32  [0, 1, 2, 3, 4]              0.9   \n",
       "\n",
       "   dev/carry-0/early_stopping/accuracy  dev/carry-1/early_stopping/accuracy  \\\n",
       "0                                  1.0                                  1.0   \n",
       "\n",
       "   dev/carry-2/early_stopping/accuracy  dev/carry-3/early_stopping/accuracy  \\\n",
       "0                                  1.0                                  1.0   \n",
       "\n",
       "   dev/carry-4/early_stopping/accuracy  \\\n",
       "0                                  1.0   \n",
       "\n",
       "   dev/early_stopping/max_correct_answer_step        ...         \\\n",
       "0                                         9.0        ...          \n",
       "\n",
       "   time/running_time            time/start_time  train_dev_test_ratio  \\\n",
       "0    00:29:54.329538 2019-03-29 22:22:05.621229           (1.0, 0, 0)   \n",
       "\n",
       "   train_set_size  train_set_size/carry-0  train_set_size/carry-1  \\\n",
       "0             256                    81.0                    54.0   \n",
       "\n",
       "   train_set_size/carry-2  train_set_size/carry-3  train_set_size/carry-4  \\\n",
       "0                    52.0                    42.0                    27.0   \n",
       "\n",
       "   hidden_dimension  \n",
       "0                24  \n",
       "\n",
       "[1 rows x 212 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(n=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['all_correct_stop', 'batch_size', 'carry_list', 'confidence_prob', 'dev/carry-0/early_stopping/accuracy', 'dev/carry-1/early_stopping/accuracy', 'dev/carry-2/early_stopping/accuracy', 'dev/carry-3/early_stopping/accuracy', 'dev/carry-4/early_stopping/accuracy', 'dev/early_stopping/max_correct_answer_step', 'dev/early_stopping/mean_correct_answer_step', 'dev/early_stopping/min_correct_answer_step', 'dev/init_all_correct_carry-0_epoch', 'dev/init_all_correct_carry-12_epoch', 'dev/init_all_correct_carry-1_epoch', 'dev/init_all_correct_carry-2_epoch', 'dev/init_all_correct_carry-3_epoch', 'dev/init_all_correct_carry-4_epoch', 'dev/init_all_correct_carry-5_epoch', 'dev/init_all_correct_carry-6_epoch', 'dev/init_all_correct_carry-8_epoch', 'dev/init_all_correct_digit-1_epoch', 'dev/init_all_correct_digit-2_epoch', 'dev/init_all_correct_digit-3_epoch', 'dev/init_all_correct_digit-4_epoch', 'dev/init_all_correct_digit-5_epoch', 'dev/init_all_correct_digit-6_epoch', 'dev/init_all_correct_digit-7_epoch', 'dev/init_all_correct_digit-8_epoch', 'dev/init_all_correct_epoch', 'dev/init_complete_all_correct_carry-0_epoch', 'dev/init_complete_all_correct_carry-12_epoch', 'dev/init_complete_all_correct_carry-1_epoch', 'dev/init_complete_all_correct_carry-2_epoch', 'dev/init_complete_all_correct_carry-3_epoch', 'dev/init_complete_all_correct_carry-4_epoch', 'dev/init_complete_all_correct_carry-5_epoch', 'dev/init_complete_all_correct_carry-6_epoch', 'dev/init_complete_all_correct_carry-8_epoch', 'dev/init_complete_all_correct_digit-1_epoch', 'dev/init_complete_all_correct_digit-2_epoch', 'dev/init_complete_all_correct_digit-3_epoch', 'dev/init_complete_all_correct_digit-4_epoch', 'dev/init_complete_all_correct_digit-5_epoch', 'dev/init_complete_all_correct_digit-6_epoch', 'dev/init_complete_all_correct_digit-7_epoch', 'dev/init_complete_all_correct_digit-8_epoch', 'dev/last_accuracy', 'dev/last_carry-0_accuracy', 'dev/last_carry-0_max_correct_answer_step', 'dev/last_carry-0_mean_correct_answer_step', 'dev/last_carry-0_min_correct_answer_step', 'dev/last_carry-0_wrong', 'dev/last_carry-12_accuracy', 'dev/last_carry-12_wrong', 'dev/last_carry-1_accuracy', 'dev/last_carry-1_max_correct_answer_step', 'dev/last_carry-1_mean_correct_answer_step', 'dev/last_carry-1_min_correct_answer_step', 'dev/last_carry-1_wrong', 'dev/last_carry-2_accuracy', 'dev/last_carry-2_max_correct_answer_step', 'dev/last_carry-2_mean_correct_answer_step', 'dev/last_carry-2_min_correct_answer_step', 'dev/last_carry-2_wrong', 'dev/last_carry-3_accuracy', 'dev/last_carry-3_max_correct_answer_step', 'dev/last_carry-3_mean_correct_answer_step', 'dev/last_carry-3_min_correct_answer_step', 'dev/last_carry-3_wrong', 'dev/last_carry-4_accuracy', 'dev/last_carry-4_max_correct_answer_step', 'dev/last_carry-4_mean_correct_answer_step', 'dev/last_carry-4_min_correct_answer_step', 'dev/last_carry-4_wrong', 'dev/last_carry-5_accuracy', 'dev/last_carry-5_wrong', 'dev/last_carry-6_accuracy', 'dev/last_carry-6_wrong', 'dev/last_carry-8_accuracy', 'dev/last_carry-8_wrong', 'dev/last_digit-1_accuracy', 'dev/last_digit-1_wrong', 'dev/last_digit-2_accuracy', 'dev/last_digit-2_wrong', 'dev/last_digit-3_accuracy', 'dev/last_digit-3_wrong', 'dev/last_digit-4_accuracy', 'dev/last_digit-4_wrong', 'dev/last_digit-5_accuracy', 'dev/last_digit-5_wrong', 'dev/last_digit-6_accuracy', 'dev/last_digit-6_wrong', 'dev/last_digit-7_accuracy', 'dev/last_digit-7_wrong', 'dev/last_digit-8_accuracy', 'dev/last_digit-8_wrong', 'dev/last_loss', 'dev/last_max_correct_answer_step', 'dev/last_mean_correct_answer_step', 'dev/last_min_correct_answer_step', 'dev/last_op_wrong', 'dev/last_tlu_accuracy', 'dev/last_tlu_loss', 'dev/last_tlu_op_wrong', 'dev/max_accuracy', 'dev/max_accuracy_epoch', 'dev_set_size', 'dev_set_size/carry-0', 'dev_set_size/carry-1', 'dev_set_size/carry-2', 'dev_set_size/carry-3', 'dev_set_size/carry-4', 'experiment_name', 'hidden_activation', 'hidden_dimensions', 'init_all_correct_digit-7_epoch', 'init_complete_all_correct_digit-7_epoch', 'last_carry-0_max_answer_step', 'last_carry-0_mean_answer_step', 'last_carry-0_min_answer_step', 'last_carry-12_max_answer_step', 'last_carry-12_mean_answer_step', 'last_carry-12_min_answer_step', 'last_carry-1_max_answer_step', 'last_carry-1_mean_answer_step', 'last_carry-1_min_answer_step', 'last_carry-2_max_answer_step', 'last_carry-2_mean_answer_step', 'last_carry-2_min_answer_step', 'last_carry-3_max_answer_step', 'last_carry-3_mean_answer_step', 'last_carry-3_min_answer_step', 'last_carry-4_max_answer_step', 'last_carry-4_mean_answer_step', 'last_carry-4_min_answer_step', 'last_carry-5_max_answer_step', 'last_carry-5_mean_answer_step', 'last_carry-5_min_answer_step', 'last_carry-6_max_answer_step', 'last_carry-6_mean_answer_step', 'last_carry-6_min_answer_step', 'last_carry-8_max_answer_step', 'last_carry-8_mean_answer_step', 'last_carry-8_min_answer_step', 'last_digit-7_accuracy', 'last_digit-7_wrong', 'last_epoch', 'last_test_max_answer_step', 'last_test_mean_answer_step', 'last_test_min_answer_step', 'last_tlu_op_wrong', 'last_tlu_test_accuracy', 'learning_rate', 'max_steps', 'network_input_dimension', 'network_output_dimension', 'nn_model_type', 'on_single_loss', 'operand_bits', 'operator', 'optimizer', 'result_bits', 'rnn_type', 'run_id', 'test/carry-0/early_stopping/accuracy', 'test/carry-0/early_stopping/max_correct_answer_step', 'test/carry-0/early_stopping/mean_correct_answer_step', 'test/carry-0/early_stopping/min_correct_answer_step', 'test/carry-1/early_stopping/accuracy', 'test/carry-1/early_stopping/max_correct_answer_step', 'test/carry-1/early_stopping/mean_correct_answer_step', 'test/carry-1/early_stopping/min_correct_answer_step', 'test/carry-2/early_stopping/accuracy', 'test/carry-2/early_stopping/max_correct_answer_step', 'test/carry-2/early_stopping/mean_correct_answer_step', 'test/carry-2/early_stopping/min_correct_answer_step', 'test/carry-3/early_stopping/accuracy', 'test/carry-3/early_stopping/max_correct_answer_step', 'test/carry-3/early_stopping/mean_correct_answer_step', 'test/carry-3/early_stopping/min_correct_answer_step', 'test/carry-4/early_stopping/accuracy', 'test/carry-4/early_stopping/max_correct_answer_step', 'test/carry-4/early_stopping/mean_correct_answer_step', 'test/carry-4/early_stopping/min_correct_answer_step', 'test/early_stopping/accuracy', 'test/early_stopping/max_correct_answer_step', 'test/early_stopping/mean_correct_answer_step', 'test/early_stopping/min_correct_answer_step', 'test/last_accuracy', 'test/last_loss', 'test/last_max_correct_answer_step', 'test/last_mean_correct_answer_step', 'test/last_min_correct_answer_step', 'test/last_op_wrong', 'test_set_size', 'test_set_size/carry-0', 'test_set_size/carry-1', 'test_set_size/carry-2', 'test_set_size/carry-3', 'test_set_size/carry-4', 'time/last_time', 'time/running_time', 'time/start_time', 'train_dev_test_ratio', 'train_set_size', 'train_set_size/carry-0', 'train_set_size/carry-1', 'train_set_size/carry-2', 'train_set_size/carry-3', 'train_set_size/carry-4', 'hidden_dimension']\n"
     ]
    }
   ],
   "source": [
    "print(list(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = 'iccm2019'\n",
    "df = df.loc[df['experiment_name'] == experiment_name]\n",
    "\n",
    "cols = ['run_id', 'operator', 'rnn_type', 'confidence_prob', \n",
    "        'operand_bits', 'hidden_activation', 'max_steps', 'hidden_dimension',\n",
    "        'time/running_time', 'last_epoch',\n",
    "'dev/last_carry-0_mean_correct_answer_step', \n",
    "'dev/last_carry-1_mean_correct_answer_step', \n",
    "'dev/last_carry-2_mean_correct_answer_step', \n",
    "'dev/last_carry-3_mean_correct_answer_step', \n",
    "'dev/last_carry-4_mean_correct_answer_step', \n",
    "'dev/last_mean_correct_answer_step', \n",
    "'dev/last_max_correct_answer_step',\n",
    "'dev/last_min_correct_answer_step',\n",
    "'dev/last_accuracy']\n",
    "df = df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.loc[df['dev/last_accuracy'] == 1.0]\n",
    "df = df.loc[df['max_steps'] == 30]\n",
    "df = df.loc[df['hidden_activation'] == \"relu\"]\n",
    "df = df.loc[df['rnn_type'] == \"jordan\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total df.shape: (6745, 19)\n"
     ]
    }
   ],
   "source": [
    "print(\"Total df.shape:\", df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add. Confidence probability: 0.9.\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "Add. Confidence probability: 0.8.\n",
      "(301, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "Add. Confidence probability: 0.7.\n",
      "(303, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "Add. Confidence probability: 0.6.\n",
      "(303, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "Subtract. Confidence probability: 0.9.\n",
      "(303, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "Subtract. Confidence probability: 0.8.\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "Subtract. Confidence probability: 0.7.\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "Subtract. Confidence probability: 0.6.\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n"
     ]
    }
   ],
   "source": [
    "print(\"Add. Confidence probability: 0.9.\")\n",
    "df_add_cp9_h24 = df.query('operator == \"add\"').query('confidence_prob == 0.9').query('hidden_dimension == 24'); print(df_add_cp9_h24.shape)\n",
    "df_add_cp9_h48 = df.query('operator == \"add\"').query('confidence_prob == 0.9').query('hidden_dimension == 48'); print(df_add_cp9_h48.shape)\n",
    "df_add_cp9_h72 = df.query('operator == \"add\"').query('confidence_prob == 0.9').query('hidden_dimension == 72'); print(df_add_cp9_h72.shape)\n",
    "\n",
    "print(\"Add. Confidence probability: 0.8.\")\n",
    "df_add_cp8_h24 = df.query('operator == \"add\"').query('confidence_prob == 0.8').query('hidden_dimension == 24'); print(df_add_cp8_h24.shape)\n",
    "df_add_cp8_h48 = df.query('operator == \"add\"').query('confidence_prob == 0.8').query('hidden_dimension == 48'); print(df_add_cp8_h48.shape)\n",
    "df_add_cp8_h72 = df.query('operator == \"add\"').query('confidence_prob == 0.8').query('hidden_dimension == 72'); print(df_add_cp8_h72.shape)\n",
    "\n",
    "print(\"Add. Confidence probability: 0.7.\")\n",
    "df_add_cp7_h24 = df.query('operator == \"add\"').query('confidence_prob == 0.7').query('hidden_dimension == 24'); print(df_add_cp7_h24.shape)\n",
    "df_add_cp7_h48 = df.query('operator == \"add\"').query('confidence_prob == 0.7').query('hidden_dimension == 48'); print(df_add_cp7_h48.shape)\n",
    "df_add_cp7_h72 = df.query('operator == \"add\"').query('confidence_prob == 0.7').query('hidden_dimension == 72'); print(df_add_cp7_h72.shape)\n",
    "\n",
    "print(\"Add. Confidence probability: 0.6.\")\n",
    "df_add_cp6_h24 = df.query('operator == \"add\"').query('confidence_prob == 0.6').query('hidden_dimension == 24'); print(df_add_cp7_h24.shape)\n",
    "df_add_cp6_h48 = df.query('operator == \"add\"').query('confidence_prob == 0.6').query('hidden_dimension == 48'); print(df_add_cp7_h48.shape)\n",
    "df_add_cp6_h72 = df.query('operator == \"add\"').query('confidence_prob == 0.6').query('hidden_dimension == 72'); print(df_add_cp7_h72.shape)\n",
    "\n",
    "print(\"Subtract. Confidence probability: 0.9.\")\n",
    "df_sub_cp9_h24 = df.query('operator == \"subtract\"').query('confidence_prob == 0.9').query('hidden_dimension == 24'); print(df_sub_cp9_h24.shape)\n",
    "df_sub_cp9_h48 = df.query('operator == \"subtract\"').query('confidence_prob == 0.9').query('hidden_dimension == 48'); print(df_sub_cp9_h48.shape)\n",
    "df_sub_cp9_h72 = df.query('operator == \"subtract\"').query('confidence_prob == 0.9').query('hidden_dimension == 72'); print(df_sub_cp9_h72.shape)\n",
    "\n",
    "print(\"Subtract. Confidence probability: 0.8.\")\n",
    "df_sub_cp8_h24 = df.query('operator == \"subtract\"').query('confidence_prob == 0.8').query('hidden_dimension == 24'); print(df_sub_cp8_h24.shape)\n",
    "df_sub_cp8_h48 = df.query('operator == \"subtract\"').query('confidence_prob == 0.8').query('hidden_dimension == 48'); print(df_sub_cp8_h48.shape)\n",
    "df_sub_cp8_h72 = df.query('operator == \"subtract\"').query('confidence_prob == 0.8').query('hidden_dimension == 72'); print(df_sub_cp8_h72.shape)\n",
    "\n",
    "print(\"Subtract. Confidence probability: 0.7.\")\n",
    "df_sub_cp7_h24 = df.query('operator == \"subtract\"').query('confidence_prob == 0.7').query('hidden_dimension == 24'); print(df_sub_cp7_h24.shape)\n",
    "df_sub_cp7_h48 = df.query('operator == \"subtract\"').query('confidence_prob == 0.7').query('hidden_dimension == 48'); print(df_sub_cp7_h48.shape)\n",
    "df_sub_cp7_h72 = df.query('operator == \"subtract\"').query('confidence_prob == 0.7').query('hidden_dimension == 72'); print(df_sub_cp7_h72.shape)\n",
    "\n",
    "print(\"Subtract. Confidence probability: 0.6.\")\n",
    "df_sub_cp6_h24 = df.query('operator == \"subtract\"').query('confidence_prob == 0.6').query('hidden_dimension == 24'); print(df_sub_cp7_h24.shape)\n",
    "df_sub_cp6_h48 = df.query('operator == \"subtract\"').query('confidence_prob == 0.6').query('hidden_dimension == 48'); print(df_sub_cp7_h48.shape)\n",
    "df_sub_cp6_h72 = df.query('operator == \"subtract\"').query('confidence_prob == 0.6').query('hidden_dimension == 72'); print(df_sub_cp7_h72.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n",
      "(300, 19)\n"
     ]
    }
   ],
   "source": [
    "n_rows = 300\n",
    "\n",
    "df_add_cp9_h24 = get_partial_df(df_add_cp9_h24, n_rows); print(df_add_cp9_h24.shape)\n",
    "df_add_cp9_h48 = get_partial_df(df_add_cp9_h48, n_rows); print(df_add_cp9_h48.shape)\n",
    "df_add_cp9_h72 = get_partial_df(df_add_cp9_h72, n_rows); print(df_add_cp9_h72.shape)\n",
    "\n",
    "df_add_cp8_h24 = get_partial_df(df_add_cp8_h24, n_rows); print(df_add_cp8_h24.shape)\n",
    "df_add_cp8_h48 = get_partial_df(df_add_cp8_h48, n_rows); print(df_add_cp8_h48.shape)\n",
    "df_add_cp8_h72 = get_partial_df(df_add_cp8_h72, n_rows); print(df_add_cp8_h72.shape)\n",
    "\n",
    "df_add_cp7_h24 = get_partial_df(df_add_cp7_h24, n_rows); print(df_add_cp7_h24.shape)\n",
    "df_add_cp7_h48 = get_partial_df(df_add_cp7_h48, n_rows); print(df_add_cp7_h48.shape)\n",
    "df_add_cp7_h72 = get_partial_df(df_add_cp7_h72, n_rows); print(df_add_cp7_h72.shape)\n",
    "\n",
    "df_sub_cp9_h24 = get_partial_df(df_sub_cp9_h24, n_rows); print(df_sub_cp9_h24.shape)\n",
    "df_sub_cp9_h48 = get_partial_df(df_sub_cp9_h48, n_rows); print(df_sub_cp9_h48.shape)\n",
    "df_sub_cp9_h72 = get_partial_df(df_sub_cp9_h72, n_rows); print(df_sub_cp9_h72.shape)\n",
    "\n",
    "df_sub_cp8_h24 = get_partial_df(df_sub_cp8_h24, n_rows); print(df_sub_cp8_h24.shape)\n",
    "df_sub_cp8_h48 = get_partial_df(df_sub_cp8_h48, n_rows); print(df_sub_cp8_h48.shape)\n",
    "df_sub_cp8_h72 = get_partial_df(df_sub_cp8_h72, n_rows); print(df_sub_cp8_h72.shape)\n",
    "\n",
    "df_sub_cp7_h24 = get_partial_df(df_sub_cp7_h24, n_rows); print(df_sub_cp7_h24.shape)\n",
    "df_sub_cp7_h48 = get_partial_df(df_sub_cp7_h48, n_rows); print(df_sub_cp7_h48.shape)\n",
    "df_sub_cp7_h72 = get_partial_df(df_sub_cp7_h72, n_rows); print(df_sub_cp7_h72.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute `mean(epochs)` and `mean(training_time)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_add_list = [\n",
    "    df_add_cp9_h24, df_add_cp9_h48, df_add_cp9_h72,\n",
    "    df_add_cp8_h24, df_add_cp8_h48, df_add_cp8_h72,\n",
    "    df_add_cp7_h24, df_add_cp7_h48, df_add_cp7_h72,\n",
    "]\n",
    "df_sub_list = [\n",
    "    df_sub_cp9_h24, df_sub_cp9_h48, df_sub_cp9_h72,\n",
    "    df_sub_cp8_h24, df_sub_cp8_h48, df_sub_cp8_h72,\n",
    "    df_sub_cp7_h24, df_sub_cp7_h48, df_sub_cp7_h72,\n",
    "]\n",
    "df_add = pd.concat(df_add_list, ignore_index=True)\n",
    "df_sub = pd.concat(df_sub_list, ignore_index=True)\n",
    "df_all = pd.concat(df_add_list + df_sub_list, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=====[add]=====\n",
      "time/running_time    0 days 00:58:22.097969\n",
      "last_epoch                          4000.33\n",
      "dtype: object\n",
      "time/running_time    0 days 21:05:41.520583\n",
      "last_epoch                            92530\n",
      "dtype: object\n",
      "time/running_time    0 days 00:03:49.328507\n",
      "last_epoch                              215\n",
      "dtype: object\n",
      "\n",
      "=====[sub]=====\n",
      "time/running_time    0 days 00:12:57.314291\n",
      "last_epoch                          1079.86\n",
      "dtype: object\n",
      "time/running_time    0 days 01:43:52.289184\n",
      "last_epoch                            13362\n",
      "dtype: object\n",
      "time/running_time    0 days 00:02:52.343140\n",
      "last_epoch                              207\n",
      "dtype: object\n",
      "\n",
      "=====[all]=====\n",
      "time/running_time    0 days 00:35:39.706130\n",
      "last_epoch                          2540.09\n",
      "dtype: object\n",
      "time/running_time    0 days 21:05:41.520583\n",
      "last_epoch                            92530\n",
      "dtype: object\n",
      "time/running_time    0 days 00:02:52.343140\n",
      "last_epoch                              207\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "cols = ['time/running_time', 'last_epoch']\n",
    "print('\\n=====[add]=====')\n",
    "print(df_add[cols].mean())\n",
    "print(df_add[cols].max())\n",
    "print(df_add[cols].min())\n",
    "\n",
    "print('\\n=====[sub]=====')\n",
    "print(df_sub[cols].mean())\n",
    "print(df_sub[cols].max())\n",
    "print(df_sub[cols].min())\n",
    "\n",
    "print('\\n=====[all]=====')\n",
    "print(df_all[cols].mean())\n",
    "print(df_all[cols].max())\n",
    "print(df_all[cols].min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "time/running_time    133 days 17:33:33.105931\n",
       "last_epoch                        1.37165e+07\n",
       "dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[cols].mean() * 5400 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "time/running_time    2 days 05:29:33.551765\n",
       "last_epoch                           228608\n",
       "dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[cols].mean() * 5400 / (60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "time/running_time    33 days 10:23:23.276482\n",
       "last_epoch                       3.42913e+06\n",
       "dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[cols].mean() * 5400 / 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export as CSV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functionalize\n",
    "__Save 4 type of CSV files__\n",
    "* mean_answer_step_by_carries\n",
    "* mean_answer_step\n",
    "* max_answer_step\n",
    "* min_answer_step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import data_utils \n",
    "import config\n",
    "from utils import create_dir\n",
    "from os.path import join\n",
    "\n",
    "def get_csv_df(df, filename, experiment_name):\n",
    "    # Part 1 ================================\n",
    "    # Get configurations\n",
    "    n_rows = df.shape[0]\n",
    "    operator = df['operator'].iloc[0]\n",
    "    operand_digits = df['operand_bits'].iloc[0]\n",
    "    carry_list = list(data_utils.import_carry_datasets(operand_digits, operator).keys())\n",
    "    \n",
    "    # Gather for each \n",
    "    csv_df_list = list()\n",
    "    for carries in carry_list:\n",
    "        col = 'dev/last_carry-{}_mean_correct_answer_step'.format(carries)\n",
    "        csv_df = pd.DataFrame(data={'mean_answer_steps':df[col], 'carries':np.full((n_rows), carries)})\n",
    "        csv_df_list.append(csv_df)\n",
    "    csv_df = pd.concat(csv_df_list, ignore_index=True)\n",
    "    \n",
    "    # Change the order of columns\n",
    "    csv_df = csv_df[['mean_answer_steps', 'carries']]\n",
    "    \n",
    "    # Create dir\n",
    "    dir_to_save = join(config.dir_result_statistics(), experiment_name)\n",
    "    create_dir(dir_to_save)\n",
    "    \n",
    "    # Save the dataframe to a CSV file.\n",
    "    csv_df.to_csv(join(dir_to_save, filename), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save CSV files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_csv_df(df_add_cp9_h24, 'mean_answer_step_by_carries-add-cp9-h24.csv', experiment_name)\n",
    "get_csv_df(df_add_cp9_h48, 'mean_answer_step_by_carries-add-cp9-h48.csv', experiment_name)\n",
    "get_csv_df(df_add_cp9_h72, 'mean_answer_step_by_carries-add-cp9-h72.csv', experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_csv_df(df_add_cp8_h24, 'mean_answer_step_by_carries-add-cp8-h24.csv', experiment_name)\n",
    "get_csv_df(df_add_cp8_h48, 'mean_answer_step_by_carries-add-cp8-h48.csv', experiment_name)\n",
    "get_csv_df(df_add_cp8_h72, 'mean_answer_step_by_carries-add-cp8-h72.csv', experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_csv_df(df_add_cp7_h24, 'mean_answer_step_by_carries-add-cp7-h24.csv', experiment_name)\n",
    "get_csv_df(df_add_cp7_h48, 'mean_answer_step_by_carries-add-cp7-h48.csv', experiment_name)\n",
    "get_csv_df(df_add_cp7_h72, 'mean_answer_step_by_carries-add-cp7-h72.csv', experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_csv_df(df_sub_cp9_h24, 'mean_answer_step_by_carries-sub-cp9-h24.csv', experiment_name)\n",
    "get_csv_df(df_sub_cp9_h48, 'mean_answer_step_by_carries-sub-cp9-h48.csv', experiment_name)\n",
    "get_csv_df(df_sub_cp9_h72, 'mean_answer_step_by_carries-sub-cp9-h72.csv', experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_csv_df(df_sub_cp8_h24, 'mean_answer_step_by_carries-sub-cp8-h24.csv', experiment_name)\n",
    "get_csv_df(df_sub_cp8_h48, 'mean_answer_step_by_carries-sub-cp8-h48.csv', experiment_name)\n",
    "get_csv_df(df_sub_cp8_h72, 'mean_answer_step_by_carries-sub-cp8-h72.csv', experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_csv_df(df_sub_cp7_h24, 'mean_answer_step_by_carries-sub-cp7-h24.csv', experiment_name)\n",
    "get_csv_df(df_sub_cp7_h48, 'mean_answer_step_by_carries-sub-cp7-h48.csv', experiment_name)\n",
    "get_csv_df(df_sub_cp7_h72, 'mean_answer_step_by_carries-sub-cp7-h72.csv', experiment_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mean answer step by confidence probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_csv_mas_by_cp(df_list, filename, experiment_name):\n",
    "    selected_cols = ['dev/last_mean_correct_answer_step', 'confidence_prob']\n",
    "    \n",
    "    csv_df_list = list()\n",
    "    for df in df_list:\n",
    "        csv_df_list.append(df[selected_cols])\n",
    "        \n",
    "    csv_df = pd.concat(csv_df_list, ignore_index=True)\n",
    "    csv_df.rename(columns={'dev/last_mean_correct_answer_step':'mean_answer_step'}, inplace=True)\n",
    "    \n",
    "    dir_to_save = join(config.dir_result_statistics(), experiment_name)\n",
    "    create_dir(dir_to_save)\n",
    "    \n",
    "    csv_df.to_csv(join(dir_to_save, filename), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_add_cp9_h24,\n",
    "    df_add_cp8_h24,\n",
    "    df_add_cp7_h24\n",
    "]\n",
    "save_csv_mas_by_cp(df_list, 'mean_answer_step_by_cp-add-h24.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_add_cp9_h48,\n",
    "    df_add_cp8_h48,\n",
    "    df_add_cp7_h48\n",
    "]\n",
    "save_csv_mas_by_cp(df_list, 'mean_answer_step_by_cp-add-h48.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_add_cp9_h72,\n",
    "    df_add_cp8_h72,\n",
    "    df_add_cp7_h72\n",
    "]\n",
    "save_csv_mas_by_cp(df_list, 'mean_answer_step_by_cp-add-h72.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_sub_cp9_h24,\n",
    "    df_sub_cp8_h24,\n",
    "    df_sub_cp7_h24\n",
    "]\n",
    "save_csv_mas_by_cp(df_list, 'mean_answer_step_by_cp-sub-h24.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_sub_cp9_h48,\n",
    "    df_sub_cp8_h48,\n",
    "    df_sub_cp7_h48\n",
    "]\n",
    "save_csv_mas_by_cp(df_list, 'mean_answer_step_by_cp-sub-h48.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "df_list = [\n",
    "    df_sub_cp9_h72,\n",
    "    df_sub_cp8_h72,\n",
    "    df_sub_cp7_h72\n",
    "]\n",
    "save_csv_mas_by_cp(df_list, 'mean_answer_step_by_cp-sub-h72.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mean answer step across the number of the hidden dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_csv_mas_by_hd(df_list, filename, experiment_name):\n",
    "    selected_cols = ['dev/last_mean_correct_answer_step', 'hidden_dimension']\n",
    "    \n",
    "    csv_df_list = list()\n",
    "    for df in df_list:\n",
    "        csv_df_list.append(df[selected_cols])\n",
    "        \n",
    "    csv_df = pd.concat(csv_df_list, ignore_index=True)\n",
    "    csv_df.rename(columns={'dev/last_mean_correct_answer_step':'mean_answer_step'}, inplace=True)\n",
    "    \n",
    "    dir_to_save = join(config.dir_result_statistics(), experiment_name)\n",
    "    create_dir(dir_to_save)\n",
    "    \n",
    "    csv_df.to_csv(join(dir_to_save, filename), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_add_cp9_h24,\n",
    "    df_add_cp9_h48,\n",
    "    df_add_cp9_h72\n",
    "]\n",
    "save_csv_mas_by_hd(df_list, 'mean_answer_step_by_hd-add-cp9.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_add_cp8_h24,\n",
    "    df_add_cp8_h48,\n",
    "    df_add_cp8_h72\n",
    "]\n",
    "save_csv_mas_by_hd(df_list, 'mean_answer_step_by_hd-add-cp8.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_add_cp7_h24,\n",
    "    df_add_cp7_h48,\n",
    "    df_add_cp7_h72\n",
    "]\n",
    "save_csv_mas_by_hd(df_list, 'mean_answer_step_by_hd-add-cp7.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_sub_cp9_h24,\n",
    "    df_sub_cp9_h48,\n",
    "    df_sub_cp9_h72\n",
    "]\n",
    "save_csv_mas_by_hd(df_list, 'mean_answer_step_by_hd-sub-cp9.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_sub_cp8_h24,\n",
    "    df_sub_cp8_h48,\n",
    "    df_sub_cp8_h72\n",
    "]\n",
    "save_csv_mas_by_hd(df_list, 'mean_answer_step_by_hd-sub-cp8.csv', 'iccm2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_sub_cp7_h24,\n",
    "    df_sub_cp7_h48,\n",
    "    df_sub_cp7_h72\n",
    "]\n",
    "save_csv_mas_by_hd(df_list, 'mean_answer_step_by_hd-sub-cp7.csv', 'iccm2019')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}