jordan_cp9_add_sub_maxstep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import run_info_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = run_info_utils.get_df_run_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>all_correct_stop</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>carry_list</th>\n",
       "      <th>confidence_prob</th>\n",
       "      <th>dev/carry-0/early_stopping/accuracy</th>\n",
       "      <th>dev/carry-1/early_stopping/accuracy</th>\n",
       "      <th>dev/carry-2/early_stopping/accuracy</th>\n",
       "      <th>dev/carry-3/early_stopping/accuracy</th>\n",
       "      <th>dev/carry-4/early_stopping/accuracy</th>\n",
       "      <th>dev/early_stopping/max_correct_answer_step</th>\n",
       "      <th>...</th>\n",
       "      <th>time/last_time</th>\n",
       "      <th>time/running_time</th>\n",
       "      <th>time/start_time</th>\n",
       "      <th>train_dev_test_ratio</th>\n",
       "      <th>train_set_size</th>\n",
       "      <th>train_set_size/carry-0</th>\n",
       "      <th>train_set_size/carry-1</th>\n",
       "      <th>train_set_size/carry-2</th>\n",
       "      <th>train_set_size/carry-3</th>\n",
       "      <th>train_set_size/carry-4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>32</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>(1.0, 0, 0)</td>\n",
       "      <td>256</td>\n",
       "      <td>81.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>32</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>(1.0, 0, 0)</td>\n",
       "      <td>256</td>\n",
       "      <td>81.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>32</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>136</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>True</td>\n",
       "      <td>32</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>256</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>True</td>\n",
       "      <td>32</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>240</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 210 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   all_correct_stop  batch_size carry_list  confidence_prob  \\\n",
       "0              True          32        NaN              0.9   \n",
       "1              True          32        NaN              0.9   \n",
       "2              True          32        NaN              NaN   \n",
       "3              True          32        NaN              NaN   \n",
       "4              True          32        NaN              NaN   \n",
       "\n",
       "   dev/carry-0/early_stopping/accuracy  dev/carry-1/early_stopping/accuracy  \\\n",
       "0                                  NaN                                  NaN   \n",
       "1                                  NaN                                  NaN   \n",
       "2                                  NaN                                  NaN   \n",
       "3                                  NaN                                  NaN   \n",
       "4                                  NaN                                  NaN   \n",
       "\n",
       "   dev/carry-2/early_stopping/accuracy  dev/carry-3/early_stopping/accuracy  \\\n",
       "0                                  NaN                                  NaN   \n",
       "1                                  NaN                                  NaN   \n",
       "2                                  NaN                                  NaN   \n",
       "3                                  NaN                                  NaN   \n",
       "4                                  NaN                                  NaN   \n",
       "\n",
       "   dev/carry-4/early_stopping/accuracy  \\\n",
       "0                                  NaN   \n",
       "1                                  NaN   \n",
       "2                                  NaN   \n",
       "3                                  NaN   \n",
       "4                                  NaN   \n",
       "\n",
       "   dev/early_stopping/max_correct_answer_step           ...            \\\n",
       "0                                         NaN           ...             \n",
       "1                                         NaN           ...             \n",
       "2                                         NaN           ...             \n",
       "3                                         NaN           ...             \n",
       "4                                         NaN           ...             \n",
       "\n",
       "   time/last_time  time/running_time  time/start_time  train_dev_test_ratio  \\\n",
       "0             NaT                NaT              NaT           (1.0, 0, 0)   \n",
       "1             NaT                NaT              NaT           (1.0, 0, 0)   \n",
       "2             NaT                NaT              NaT                   NaN   \n",
       "3             NaT                NaT              NaT                   NaN   \n",
       "4             NaT                NaT              NaT                   NaN   \n",
       "\n",
       "   train_set_size  train_set_size/carry-0  train_set_size/carry-1  \\\n",
       "0             256                    81.0                    54.0   \n",
       "1             256                    81.0                    54.0   \n",
       "2             136                     NaN                     NaN   \n",
       "3             256                     NaN                     NaN   \n",
       "4             240                     NaN                     NaN   \n",
       "\n",
       "   train_set_size/carry-2  train_set_size/carry-3  train_set_size/carry-4  \n",
       "0                    52.0                    42.0                    27.0  \n",
       "1                    52.0                    42.0                    27.0  \n",
       "2                     NaN                     NaN                     NaN  \n",
       "3                     NaN                     NaN                     NaN  \n",
       "4                     NaN                     NaN                     NaN  \n",
       "\n",
       "[5 rows x 210 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(list(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = 'jordan_cp9_add_sub_maxstep'\n",
    "df = df.loc[df['experiment_name'] == experiment_name]\n",
    "\n",
    "cols = ['run_id', 'operator', 'rnn_type', 'confidence_prob', 'operand_bits', 'hidden_activation', 'max_steps',\n",
    "'dev/last_carry-0_mean_correct_answer_step', \n",
    "'dev/last_carry-1_mean_correct_answer_step', \n",
    "'dev/last_carry-2_mean_correct_answer_step', \n",
    "'dev/last_carry-3_mean_correct_answer_step', \n",
    "'dev/last_carry-4_mean_correct_answer_step', \n",
    "'dev/last_mean_correct_answer_step', 'dev/last_accuracy']\n",
    "df = df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.loc[df['dev/last_accuracy'] == 1.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1151, 14)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_add_jordan_9_relu_maxstep10 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 10')\n",
    "df_sub_jordan_9_relu_maxstep10 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 10')\n",
    "df_add_jordan_9_relu_maxstep20 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 20')\n",
    "df_sub_jordan_9_relu_maxstep20 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 20')\n",
    "df_add_jordan_9_relu_maxstep30 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 30')\n",
    "df_sub_jordan_9_relu_maxstep30 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 30')\n",
    "df_add_jordan_9_relu_maxstep40 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 40')\n",
    "df_sub_jordan_9_relu_maxstep40 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 40')\n",
    "df_add_jordan_9_relu_maxstep50 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 50')\n",
    "df_sub_jordan_9_relu_maxstep50 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 50')\n",
    "df_add_jordan_9_relu_maxstep60 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 60')\n",
    "df_sub_jordan_9_relu_maxstep60 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 60')\n",
    "df_add_jordan_9_relu_maxstep90 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"relu\"').query('max_steps == 90')\n",
    "\n",
    "\n",
    "df_add_jordan_9_tanh_maxstep10 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 10')\n",
    "df_sub_jordan_9_tanh_maxstep10 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 10')\n",
    "df_add_jordan_9_tanh_maxstep20 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 20')\n",
    "df_sub_jordan_9_tanh_maxstep20 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 20')\n",
    "df_add_jordan_9_tanh_maxstep30 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 30')\n",
    "df_sub_jordan_9_tanh_maxstep30 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 30')\n",
    "df_add_jordan_9_tanh_maxstep40 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 40')\n",
    "df_sub_jordan_9_tanh_maxstep40 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 40')\n",
    "df_add_jordan_9_tanh_maxstep50 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 50')\n",
    "df_sub_jordan_9_tanh_maxstep50 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 50')\n",
    "df_add_jordan_9_tanh_maxstep60 = df.query('operator == \"add\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 60')\n",
    "df_sub_jordan_9_tanh_maxstep60 = df.query('operator == \"subtract\"').query('rnn_type == \"jordan\"').query('confidence_prob == 0.9').query('hidden_activation == \"tanh\"').query('max_steps == 60')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 14)\n",
      "(30, 14)\n",
      "(100, 14)\n",
      "(30, 14)\n",
      "(100, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(102, 14)\n",
      "(30, 14)\n",
      "(102, 14)\n",
      "(30, 14)\n",
      "(60, 14)\n"
     ]
    }
   ],
   "source": [
    "print(df_add_jordan_9_relu_maxstep10.shape)\n",
    "print(df_sub_jordan_9_relu_maxstep10.shape)\n",
    "print(df_add_jordan_9_relu_maxstep20.shape)\n",
    "print(df_sub_jordan_9_relu_maxstep20.shape)\n",
    "print(df_add_jordan_9_relu_maxstep30.shape)\n",
    "print(df_sub_jordan_9_relu_maxstep30.shape)\n",
    "print(df_add_jordan_9_relu_maxstep40.shape)\n",
    "print(df_sub_jordan_9_relu_maxstep40.shape)\n",
    "print(df_add_jordan_9_relu_maxstep50.shape)\n",
    "print(df_sub_jordan_9_relu_maxstep50.shape)\n",
    "print(df_add_jordan_9_relu_maxstep60.shape)\n",
    "print(df_sub_jordan_9_relu_maxstep60.shape)\n",
    "print(df_add_jordan_9_relu_maxstep90.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(37, 14)\n",
      "(40, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n",
      "(30, 14)\n"
     ]
    }
   ],
   "source": [
    "print(df_add_jordan_9_tanh_maxstep10.shape)\n",
    "print(df_sub_jordan_9_tanh_maxstep10.shape)\n",
    "print(df_add_jordan_9_tanh_maxstep20.shape)\n",
    "print(df_sub_jordan_9_tanh_maxstep20.shape)\n",
    "print(df_add_jordan_9_tanh_maxstep30.shape)\n",
    "print(df_sub_jordan_9_tanh_maxstep30.shape)\n",
    "print(df_add_jordan_9_tanh_maxstep40.shape)\n",
    "print(df_sub_jordan_9_tanh_maxstep40.shape)\n",
    "print(df_add_jordan_9_tanh_maxstep50.shape)\n",
    "print(df_sub_jordan_9_tanh_maxstep50.shape)\n",
    "print(df_add_jordan_9_tanh_maxstep60.shape)\n",
    "print(df_sub_jordan_9_tanh_maxstep60.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>confidence_prob</th>\n",
       "      <th>operand_bits</th>\n",
       "      <th>max_steps</th>\n",
       "      <th>dev/last_carry-0_mean_correct_answer_step</th>\n",
       "      <th>dev/last_carry-1_mean_correct_answer_step</th>\n",
       "      <th>dev/last_carry-2_mean_correct_answer_step</th>\n",
       "      <th>dev/last_carry-3_mean_correct_answer_step</th>\n",
       "      <th>dev/last_carry-4_mean_correct_answer_step</th>\n",
       "      <th>dev/last_mean_correct_answer_step</th>\n",
       "      <th>dev/last_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3.000000e+01</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>2.428395</td>\n",
       "      <td>4.304938</td>\n",
       "      <td>8.871154</td>\n",
       "      <td>13.933333</td>\n",
       "      <td>20.608642</td>\n",
       "      <td>7.937891</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.258405e-16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.441839</td>\n",
       "      <td>2.770914</td>\n",
       "      <td>4.956138</td>\n",
       "      <td>7.701149</td>\n",
       "      <td>9.403478</td>\n",
       "      <td>3.598127</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.802469</td>\n",
       "      <td>1.222222</td>\n",
       "      <td>1.692308</td>\n",
       "      <td>2.047619</td>\n",
       "      <td>2.370370</td>\n",
       "      <td>1.691406</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.385802</td>\n",
       "      <td>2.337963</td>\n",
       "      <td>5.562500</td>\n",
       "      <td>8.809524</td>\n",
       "      <td>14.490741</td>\n",
       "      <td>5.565430</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.864198</td>\n",
       "      <td>3.259259</td>\n",
       "      <td>7.653846</td>\n",
       "      <td>12.321428</td>\n",
       "      <td>22.629629</td>\n",
       "      <td>7.587891</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>3.453704</td>\n",
       "      <td>6.032407</td>\n",
       "      <td>11.442308</td>\n",
       "      <td>19.619047</td>\n",
       "      <td>28.305555</td>\n",
       "      <td>10.259766</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>6.370370</td>\n",
       "      <td>11.481482</td>\n",
       "      <td>23.076923</td>\n",
       "      <td>29.857143</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>16.605469</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       confidence_prob  operand_bits  max_steps  \\\n",
       "count     3.000000e+01          30.0       30.0   \n",
       "mean      9.000000e-01           4.0       40.0   \n",
       "std       2.258405e-16           0.0        0.0   \n",
       "min       9.000000e-01           4.0       40.0   \n",
       "25%       9.000000e-01           4.0       40.0   \n",
       "50%       9.000000e-01           4.0       40.0   \n",
       "75%       9.000000e-01           4.0       40.0   \n",
       "max       9.000000e-01           4.0       40.0   \n",
       "\n",
       "       dev/last_carry-0_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                    2.428395   \n",
       "std                                     1.441839   \n",
       "min                                     0.802469   \n",
       "25%                                     1.385802   \n",
       "50%                                     1.864198   \n",
       "75%                                     3.453704   \n",
       "max                                     6.370370   \n",
       "\n",
       "       dev/last_carry-1_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                    4.304938   \n",
       "std                                     2.770914   \n",
       "min                                     1.222222   \n",
       "25%                                     2.337963   \n",
       "50%                                     3.259259   \n",
       "75%                                     6.032407   \n",
       "max                                    11.481482   \n",
       "\n",
       "       dev/last_carry-2_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                    8.871154   \n",
       "std                                     4.956138   \n",
       "min                                     1.692308   \n",
       "25%                                     5.562500   \n",
       "50%                                     7.653846   \n",
       "75%                                    11.442308   \n",
       "max                                    23.076923   \n",
       "\n",
       "       dev/last_carry-3_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                   13.933333   \n",
       "std                                     7.701149   \n",
       "min                                     2.047619   \n",
       "25%                                     8.809524   \n",
       "50%                                    12.321428   \n",
       "75%                                    19.619047   \n",
       "max                                    29.857143   \n",
       "\n",
       "       dev/last_carry-4_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                   20.608642   \n",
       "std                                     9.403478   \n",
       "min                                     2.370370   \n",
       "25%                                    14.490741   \n",
       "50%                                    22.629629   \n",
       "75%                                    28.305555   \n",
       "max                                    39.000000   \n",
       "\n",
       "       dev/last_mean_correct_answer_step  dev/last_accuracy  \n",
       "count                          30.000000               30.0  \n",
       "mean                            7.937891                1.0  \n",
       "std                             3.598127                0.0  \n",
       "min                             1.691406                1.0  \n",
       "25%                             5.565430                1.0  \n",
       "50%                             7.587891                1.0  \n",
       "75%                            10.259766                1.0  \n",
       "max                            16.605469                1.0  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_add_jordan_9_relu_maxstep40.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>confidence_prob</th>\n",
       "      <th>operand_bits</th>\n",
       "      <th>max_steps</th>\n",
       "      <th>dev/last_carry-0_mean_correct_answer_step</th>\n",
       "      <th>dev/last_carry-1_mean_correct_answer_step</th>\n",
       "      <th>dev/last_carry-2_mean_correct_answer_step</th>\n",
       "      <th>dev/last_carry-3_mean_correct_answer_step</th>\n",
       "      <th>dev/last_carry-4_mean_correct_answer_step</th>\n",
       "      <th>dev/last_mean_correct_answer_step</th>\n",
       "      <th>dev/last_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3.000000e+01</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>2.436214</td>\n",
       "      <td>8.538272</td>\n",
       "      <td>18.387719</td>\n",
       "      <td>29.874074</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.691912</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.258405e-16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.918958</td>\n",
       "      <td>5.948510</td>\n",
       "      <td>7.776826</td>\n",
       "      <td>8.827360</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.451691</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.604938</td>\n",
       "      <td>1.518519</td>\n",
       "      <td>2.947368</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.191176</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.120370</td>\n",
       "      <td>3.342593</td>\n",
       "      <td>13.750000</td>\n",
       "      <td>23.916667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.509191</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.765432</td>\n",
       "      <td>7.185185</td>\n",
       "      <td>17.394736</td>\n",
       "      <td>32.333332</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.845588</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>2.876543</td>\n",
       "      <td>10.500000</td>\n",
       "      <td>24.302631</td>\n",
       "      <td>36.666667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.786765</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>9.000000e-01</td>\n",
       "      <td>4.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>8.567902</td>\n",
       "      <td>22.851852</td>\n",
       "      <td>37.263157</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17.426470</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       confidence_prob  operand_bits  max_steps  \\\n",
       "count     3.000000e+01          30.0       30.0   \n",
       "mean      9.000000e-01           4.0       40.0   \n",
       "std       2.258405e-16           0.0        0.0   \n",
       "min       9.000000e-01           4.0       40.0   \n",
       "25%       9.000000e-01           4.0       40.0   \n",
       "50%       9.000000e-01           4.0       40.0   \n",
       "75%       9.000000e-01           4.0       40.0   \n",
       "max       9.000000e-01           4.0       40.0   \n",
       "\n",
       "       dev/last_carry-0_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                    2.436214   \n",
       "std                                     1.918958   \n",
       "min                                     0.604938   \n",
       "25%                                     1.120370   \n",
       "50%                                     1.765432   \n",
       "75%                                     2.876543   \n",
       "max                                     8.567902   \n",
       "\n",
       "       dev/last_carry-1_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                    8.538272   \n",
       "std                                     5.948510   \n",
       "min                                     1.518519   \n",
       "25%                                     3.342593   \n",
       "50%                                     7.185185   \n",
       "75%                                    10.500000   \n",
       "max                                    22.851852   \n",
       "\n",
       "       dev/last_carry-2_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                   18.387719   \n",
       "std                                     7.776826   \n",
       "min                                     2.947368   \n",
       "25%                                    13.750000   \n",
       "50%                                    17.394736   \n",
       "75%                                    24.302631   \n",
       "max                                    37.263157   \n",
       "\n",
       "       dev/last_carry-3_mean_correct_answer_step  \\\n",
       "count                                  30.000000   \n",
       "mean                                   29.874074   \n",
       "std                                     8.827360   \n",
       "min                                     8.000000   \n",
       "25%                                    23.916667   \n",
       "50%                                    32.333332   \n",
       "75%                                    36.666667   \n",
       "max                                    39.000000   \n",
       "\n",
       "       dev/last_carry-4_mean_correct_answer_step  \\\n",
       "count                                        0.0   \n",
       "mean                                         NaN   \n",
       "std                                          NaN   \n",
       "min                                          NaN   \n",
       "25%                                          NaN   \n",
       "50%                                          NaN   \n",
       "75%                                          NaN   \n",
       "max                                          NaN   \n",
       "\n",
       "       dev/last_mean_correct_answer_step  dev/last_accuracy  \n",
       "count                          30.000000               30.0  \n",
       "mean                            7.691912                1.0  \n",
       "std                             3.451691                0.0  \n",
       "min                             2.191176                1.0  \n",
       "25%                             5.509191                1.0  \n",
       "50%                             6.845588                1.0  \n",
       "75%                             9.786765                1.0  \n",
       "max                            17.426470                1.0  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sub_jordan_9_relu_maxstep40.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export as CSV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functionalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import data_utils \n",
    "from utils import create_dir\n",
    "from os.path import join\n",
    "\n",
    "def get_csv_df(df, filename, experiment_name):\n",
    "    # Get configurations\n",
    "    n_rows = df.shape[0]\n",
    "    operator = df['operator'].iloc[0]\n",
    "    operand_digits = df['operand_bits'].iloc[0]\n",
    "    carry_list = list(data_utils.import_carry_datasets(operand_digits, operator).keys())\n",
    "    \n",
    "    # Gather for each \n",
    "    csv_df_list = list()\n",
    "    for carries in carry_list:\n",
    "        col = 'dev/last_carry-{}_mean_correct_answer_step'.format(carries)\n",
    "        csv_df = pd.DataFrame(data={'mean_anwer_steps':df[col], 'carries':np.full((n_rows), carries)})\n",
    "        csv_df_list.append(csv_df)\n",
    "    csv_df = pd.concat(csv_df_list, ignore_index=True)\n",
    "    \n",
    "    # Change the order of columns\n",
    "    csv_df = csv_df[['mean_anwer_steps', 'carries']]\n",
    "    \n",
    "    # Create dir\n",
    "    dir_to_save = join('result_statistics', experiment_name)\n",
    "    create_dir(dir_to_save)\n",
    "    \n",
    "    # Save the dataframe to a CSV file.\n",
    "    csv_df.to_csv(join(dir_to_save, filename), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = 'jordan_cp9_add_sub_maxstep'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_csv_df(df_add_jordan_9_relu_maxstep10, 'df_add_jordan_9_relu_maxstep10.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_relu_maxstep10, 'df_sub_jordan_9_relu_maxstep10.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_relu_maxstep20, 'df_add_jordan_9_relu_maxstep20.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_relu_maxstep20, 'df_sub_jordan_9_relu_maxstep20.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_relu_maxstep30, 'df_add_jordan_9_relu_maxstep30.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_relu_maxstep30, 'df_sub_jordan_9_relu_maxstep30.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_relu_maxstep40, 'df_add_jordan_9_relu_maxstep40.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_relu_maxstep40, 'df_sub_jordan_9_relu_maxstep40.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_relu_maxstep50, 'df_add_jordan_9_relu_maxstep50.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_relu_maxstep50, 'df_sub_jordan_9_relu_maxstep50.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_relu_maxstep60, 'df_add_jordan_9_relu_maxstep60.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_relu_maxstep60, 'df_sub_jordan_9_relu_maxstep60.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_relu_maxstep90, 'df_add_jordan_9_relu_maxstep90.csv', experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_csv_df(df_add_jordan_9_tanh_maxstep10, 'df_add_jordan_9_tanh_maxstep10.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_tanh_maxstep10, 'df_sub_jordan_9_tanh_maxstep10.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_tanh_maxstep20, 'df_add_jordan_9_tanh_maxstep20.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_tanh_maxstep20, 'df_sub_jordan_9_tanh_maxstep20.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_tanh_maxstep30, 'df_add_jordan_9_tanh_maxstep30.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_tanh_maxstep30, 'df_sub_jordan_9_tanh_maxstep30.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_tanh_maxstep40, 'df_add_jordan_9_tanh_maxstep40.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_tanh_maxstep40, 'df_sub_jordan_9_tanh_maxstep40.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_tanh_maxstep50, 'df_add_jordan_9_tanh_maxstep50.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_tanh_maxstep50, 'df_sub_jordan_9_tanh_maxstep50.csv', experiment_name)\n",
    "get_csv_df(df_add_jordan_9_tanh_maxstep60, 'df_add_jordan_9_tanh_maxstep60.csv', experiment_name)\n",
    "get_csv_df(df_sub_jordan_9_tanh_maxstep60, 'df_sub_jordan_9_tanh_maxstep60.csv', experiment_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}