notebooks/03_modelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook builds some models on the Rossmann dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Navigate to the project's root directory:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/rossmann\n"
     ]
    }
   ],
   "source": [
    "%cd .."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import src.custom_transform as custom_transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.tabular import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utilities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to make a submission file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_submission_filename(learner, file_name):\n",
    "    test_preds = learn.get_preds(DatasetType.Test)\n",
    "    test_df[\"Sales\"] = np.exp(test_preds[0].data).numpy().T[0]\n",
    "    test_df[[\"Id\",\"Sales\"]] = test_df[[\"Id\",\"Sales\"]].astype(\"int\")\n",
    "    test_df[[\"Id\",\"Sales\"]].to_csv(file_name,index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the training and test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = Path('./data/rossmann')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/pyarrow/pandas_compat.py:708: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n",
      "  labels = getattr(columns, 'labels', None) or [\n",
      "/opt/conda/lib/python3.6/site-packages/pyarrow/pandas_compat.py:735: FutureWarning: the 'labels' keyword is deprecated, use 'codes' instead\n",
      "  return pd.MultiIndex(levels=new_levels, labels=labels, names=columns.names)\n",
      "/opt/conda/lib/python3.6/site-packages/pyarrow/pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n",
      "  labels, = index.labels\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_parquet(DATA_DIR/'train_clean.parquet')\n",
    "test_df = pd.read_parquet(DATA_DIR/'test_clean.parquet')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the continuous and categorical variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_vars = ['Store', 'DayOfWeek', 'Year', 'Month', 'Day', 'StateHoliday', 'CompetitionMonthsOpen',\n",
    "    'Promo2Weeks', 'StoreType', 'Assortment', 'PromoInterval', 'CompetitionOpenSinceYear', 'Promo2SinceYear',\n",
    "    'State', 'Week', 'Events', 'Promo_fw', 'Promo_bw', 'StateHoliday_fw', 'StateHoliday_bw',\n",
    "    'SchoolHoliday_fw', 'SchoolHoliday_bw', 'Promo', 'SchoolHoliday', 'CloudCover']\n",
    "\n",
    "\n",
    "cont_vars = ['CompetitionDistance', 'Max_TemperatureC', 'Mean_TemperatureC', 'Min_TemperatureC',\n",
    "   'Max_Humidity', 'Mean_Humidity', 'Min_Humidity', 'Max_Wind_SpeedKm_h', \n",
    "   'Mean_Wind_SpeedKm_h', 'trend', 'trend_DE',\n",
    "   'AfterStateHoliday', 'BeforeStateHoliday']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the dependent variable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep_var = 'Sales'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the validation set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cut = train_df['Date'][(train_df['Date'] == train_df['Date'][len(test_df)])].index.max()\n",
    "valid_idx = range(cut)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modelling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the preprocessing steps we want to do:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "procs=[FillMissing, Categorify, Normalize]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the databunch we will use for modelling:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of categorical variables: 25\n",
      "Number of continuous variables: 13\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(844338, 40)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = train_df[cat_vars + cont_vars + [dep_var,'Date']].copy()\n",
    "\n",
    "print(f'Number of categorical variables: {len(cat_vars)}')\n",
    "print(f'Number of continuous variables: {len(cont_vars)}')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (TabularList.from_df(df, path=DATA_DIR, cat_names=cat_vars, cont_names=cont_vars, procs=procs)\n",
    "                .split_by_idx(valid_idx)\n",
    "                .label_from_df(cols=dep_var, label_cls=FloatList, log=True)\n",
    "                .add_test(TabularList.from_df(test_df, path=DATA_DIR, cat_names=cat_vars, cont_names=cont_vars))\n",
    "                .databunch())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_log_y = np.log(np.max(train_df['Sales'])*1.2)\n",
    "y_range = torch.tensor([0, max_log_y], device=defaults.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = tabular_learner(data, layers=[1000,500], ps=[0.001,0.01], emb_drop=0.04, \n",
    "                        y_range=y_range, metrics=exp_rmspe)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "View the model's architecture:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TabularModel(\n",
       "  (embeds): ModuleList(\n",
       "    (0): Embedding(1116, 81)\n",
       "    (1): Embedding(8, 5)\n",
       "    (2): Embedding(4, 3)\n",
       "    (3): Embedding(13, 7)\n",
       "    (4): Embedding(32, 11)\n",
       "    (5): Embedding(3, 3)\n",
       "    (6): Embedding(26, 10)\n",
       "    (7): Embedding(27, 10)\n",
       "    (8): Embedding(5, 4)\n",
       "    (9): Embedding(4, 3)\n",
       "    (10): Embedding(4, 3)\n",
       "    (11): Embedding(24, 9)\n",
       "    (12): Embedding(9, 5)\n",
       "    (13): Embedding(13, 7)\n",
       "    (14): Embedding(53, 15)\n",
       "    (15): Embedding(22, 9)\n",
       "    (16): Embedding(7, 5)\n",
       "    (17): Embedding(7, 5)\n",
       "    (18): Embedding(4, 3)\n",
       "    (19): Embedding(4, 3)\n",
       "    (20): Embedding(9, 5)\n",
       "    (21): Embedding(9, 5)\n",
       "    (22): Embedding(3, 3)\n",
       "    (23): Embedding(3, 3)\n",
       "    (24): Embedding(10, 6)\n",
       "    (25): Embedding(3, 3)\n",
       "  )\n",
       "  (emb_drop): Dropout(p=0.04)\n",
       "  (bn_cont): BatchNorm1d(13, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (layers): Sequential(\n",
       "    (0): Linear(in_features=239, out_features=1000, bias=True)\n",
       "    (1): ReLU(inplace)\n",
       "    (2): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): Dropout(p=0.001)\n",
       "    (4): Linear(in_features=1000, out_features=500, bias=True)\n",
       "    (5): ReLU(inplace)\n",
       "    (6): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (7): Dropout(p=0.01)\n",
       "    (8): Linear(in_features=500, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do the training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXzU9Z3H8ddnMjnICTlJCBAIt9wERKtoPVrrVW211Wqr1a3Wrtpj22277nZ7rL2PrevaSle09aCHtlW0Fq0VUTwg3Pd9BQIJV0ICOee7f8xYIwYIZGZ+c7yfj8c8MvPLb+b3zjC85ze/32++P3POISIiycPndQAREYkuFb+ISJJR8YuIJBkVv4hIklHxi4gkGRW/iEiSiVjxm9ksM6szs1Vdpn3HzFaY2TIze8HMyiK1fBER6Z5F6jh+M5sBNAG/cc6NDU3Ldc41hq7fDYxxzn32ZI9VWFjoKioqIpJTRCRRLV68eJ9zrujY6f5ILdA5N9/MKo6Z1tjlZhbQo3ediooKqqurwxdORCQJmNn27qZHrPhPEORe4FNAA/D+aC9fRCTZRX3nrnPuHufcQOBx4M7jzWdmt5lZtZlV19fXRy+giEiC8/KonieAjx7vl865mc65KudcVVHRezZRiYjIaYpq8ZvZ8C43rwTWRXP5IiISwW38ZjYbOB8oNLMa4D+BS81sJBAAtgMnPaJHRETCK5JH9VzfzeSHIrU8ERHpGX1zV0QkyUT9cM5Ec6C5jcXbDwKQkeoj3Z9Cis9wztEZcDhgTFkuuRmp77pfZ8Axd/UeNu5tok+aj4zUFPqkpjCiJIfRpbmk+fWeLCKRoeI/Dev2NDJ31V7mbahj2c5DnOzLz31SU7hyQhk3Th/MyP45/HnpLn7xyma27mvudv40v49xA/IY1T+HzLQU0v0pwWnlecwYXkSKzyLwV4lIslDxH+NoWyfr9jSysa6Jopx0RvXPoX9uBu2djudX1fLoG9up3n4QM5hQ3pfPXzicc4YVku5PobWjk9aOAO2dAXxmpPiMts4Ac1ft4ellu/ld9U6y0/00tXZwRlkuv7hhMheNKaGtI8DR9k6aWjpYU9vI0h0HWbrjEH9ZWUtrR4CW9k4CoTeXAX37cN3UgXxs6kAKs9MxwELvA2Z6QxCRk4vYWD3hVFVV5SI1ZENLeycLNu3jb2v3Ur3tIJvrm/5Rsm/L65OKz+DgkXYqCjK5cfpgrpo0gMLs9B4vp+FoO39euoslOw5y1aQBnD+i6JSKuqW9k5fW1vHEwu0s2LT/Pb/PzfBTVZHP1Ip8plb0o29mGh2BAB2djrbOAG0dAVo7gj9L8zI4oyz3lN8o2jsDpKZoE5RIvDCzxc65qvdMT8bir204yrz19fx9XR2vbqynpT1AdrqfM4fkc8aAPMaU5jKiJJv6w62s33uYdXsOc6S1g6snl3PusEJ8Hm9q2bqvmRdW76GlPYALDXe0p6GFhdsOsKW++81HxxqUn8ml40q5bFwpY8pyu918VHe4hcXbDvLGlv28uWU/G+uaqCjIYtKgvkwe1I8J5X0ZWpRFVro+OIrEoqQv/vbOAA++splnV9Sybs9hILjZ5MLRxVw8poQzhxQkxA7V/U2tLNlxiCNtHaSm+PD7jFS/j/QUH+mpPlJTfKytbeS5lXtYsGkfnQFHut/H8JJsRpbkkp+Vyro9h1lbe5h9Ta0AZKalUFWRzxlluWyqa2LJ9oPsb277xzJL8zIYWpTFoPwsBub3obxfJoPyMxlalPWendoiEj1JXfx1h1u48/GlLNx2gDOH5HPBqGLeP6qY4cXZSb1d/GBzGy+vr2PN7sZ/fLJpONLO8JJsRpfmMro0l4kD+zK+PO9dm3icc+w4cIQ1uxvZXN/ElvpmNtc3sfPgUQ50eUMAKM5Jp7IomzFluUwbks+0inz6ZaVF+08VSUpJW/zV2w7wuceXcLilg+9/dBwfnjggzOkSi3OuV2+GTa0d7Dp4lO37m9myr5nNdU1sqm9ize5GWjsCAAwrziYrLYX2zuAhr2aQm5FKbp9U8vqkMqBvBpXF2VQWZTO0KIvMNG1KEjkdxyv+hP4f9eib2/nWM6sZ0K8Pv7l1GqP653odKeb19hNQdrqfkf1zGNk/513TWzs6WVnTwFtbD7B0xyE6AgH8vuCRT50BONzSTs3BI6ze3c7expZ/7GA3gzGluZw5pIBpQ/KZMDCP4pwMHdIq0gsJXfw+g/NHFvGTj00kr4+2NXsp3R/cT1BVkX/SeVs7Otm27wib65tYV9vIom0Hefyt7cxasBUAv88oyc2grG8Gw4qzGVOay5iyXEb1z9WOZpEeSOhNPW//bcm8HT9RtHZ0sqKmgXV7DlN76Ci1DS3sOniUDXWHOXSkHYDUFOOCUcVcM2Ug548s0qGnkvSSclOPCj9xpPtTQt9RePcnBucctQ0trK1t5I3N+/nzsl3MXb2Xgqw0rpo0gGumlDO6VJv4RLpK6DV+ST7tnQHmb6jnD9U1vLRuL+2djrEDcrlmcjkXjSmhvF+m1xFFoiZpj+qR5HWguY2nl+3iycU1rN7dCMCQwizeN6yAc4cXMWN4EX3SUjxOKRI5Kn5Jahv3HubVjftYsGkfb27ZT3NbJxmpPs4dXsQHz+jPxaNLyMvUAQCSWJJyG7/I24aX5DC8JIdbzhlCe2eARVsPMHf1Hl5Ys5cX1+wlNcU4b0QxV04s46LRxfrugCQ0rfFLUnPOsaKmgWdX7GbO8lr2NLaQkRocFnt8eV8mDOzL9KH5FOdkeB1V5JRpU4/ISQQCjoXbgp8Elu88xKrdjbR1BEj3+7j9vEruOK9S+wQkrmhTj8hJ+HzG9KEFTB9aAASPEFpXe5iZr27hvpc28mT1Tv7tstFcNq5UhwpLXNMav0gPvLVlP9+cs4a1tY0UZqdzdmVB6FLIoAIdIiqxSZt6RHqpM+CYs3w389bX8frm/dQdDg5bPbggkxnDi5gxooj3DSvQjmGJGSp+kTByzrG5vpnXNtYzf+M+3ti8n6PtnRRkpXHH+ZXcOH0wGanaHyDeUvGLRFBrRycLtx7gwVe28NqmffTPzeDuC4dzbVW5xgwSz6j4RaLk9c37+PHc9SzZcYjKoiy+/qHRXDi6WDuEJeqOV/xaFREJs7MrC3nqjrP51aeqcA7+6TfVfOJXb7F6d4PX0UQAFb9IRJgZF48pYe4XZ/CtK89g3Z5GrvrfBTy9bJfX0URU/CKRlJri46azK3j5y+czeVA/Pv/bZfzylc3EwyZWSVwqfpEo6JuZxm9uncbl40v5/vPr+OYzq+kMqPzFGzrgWCRK0v0p3HfdJErzMvjVq1vZ19TGzz4+kTS/1r8kulT8IlHk8xn3XDaG4pwM7v3LWo62d/LADZN1zL9EVcRWNcxslpnVmdmqLtN+ZGbrzGyFmf3JzPpGavkisewzM4Zy79VjeXl9Hbc8sojm1g6vI0kSieRnzEeAS46Z9iIw1jk3HtgAfD2CyxeJaTecOZiffmwCb27Zzycfeosmlb9EScSK3zk3HzhwzLQXnHNvv7rfBMojtXyReHD1pHIeuGEyy2sauHv2Uu3wlajwcq/SLcDzHi5fJCZcMraUb115Bn9fV8d/PbfG6ziSBDzZuWtm9wAdwOMnmOc24DaAQYMGRSmZiDdunD6YLfXNzFqwlaGFWXzyrAqvI0kCi/oav5ndBFwO3OBO8C0W59xM51yVc66qqKgoegFFPHLPZaO5cFQx35yzhnnr67yOIwksqsVvZpcAXwWudM4dieayRWJdis/4+fWTGFGSw51PLGVtbaPXkSRBRfJwztnAG8BIM6sxs1uB+4Ec4EUzW2Zmv4zU8kXiUXa6n1k3V5Gd7ufTDy+ituGo15EkAWlYZpEYtLa2kWt/+Qbl/frw+8+eRW5GqteRJA5pWGaRODK6NJdf3DiZTXVNfO6xJbR3BryOJAlExS8So84dXsT3PjKO1zbt49tzdJinhI/G6hGJYddWDWRjXRMz52+hqqIfH544wOtIkgC0xi8S477ywZFMrejH1/+4ko17D3sdRxKAil8kxqWm+Lj/E5PJTEvhjseXaEA36TUVv0gcKMnN4OfXTWJLfRNf/+NKncFLekXFLxIn3jeskM9fOIJnlu/mra0HTn4HkeNQ8YvEkdvPG0pBVhq/mLfZ6ygSx1T8InEkIzWFW84Zwisb6lm9u8HrOBKnVPwicebG6YPJTvfzy1e2eB1F4pSKXyTO5PVJ5Ybpg3huxW627Wv2Oo7EIRW/SBy69X1D8Kf4mPmq1vrl1Kn4ReJQcW4G10wp58nqGuoaW7yOI3FGxS8Sp247dygdgQAPLdjqdRSJMyp+kThVUZjFpeNKeeLNHRxuafc6jsQRFb9IHLt9RiWHWzuYvXCH11Ekjqj4ReLYuPI8zq4s4KHXttLWoTH7pWdU/CJx7vbzKtnb2MrTy3Z5HUXihIpfJM7NGF7IqP45/OrVLQQCGrxNTk7FLxLnzIzbzxvKhr1NzNtQ53UciQMqfpEEcPn4MsryMnhQwzhID6j4RRJAaoqPW84ZwltbD7B85yGv40iMU/GLJIjrpg2iT2oKv1200+soEuNU/CIJIjvdz4fG9efZ5btpae/0Oo7EMBW/SAK5Zko5h1s7mLt6j9dRJIap+EUSyPQhBQzo24cnF9d4HUVimIpfJIH4fMZHp5Tz2qZ97D501Os4EqNU/CIJ5qOTB+Ac/Gmpvskr3VPxiySYwQVZTBuSz5OLa3BO3+SV91LxiySga6aUs3VfM0t2HPQ6isQgFb9IArp0XCl9UlO0k1e6FbHiN7NZZlZnZqu6TLvWzFabWcDMqiK1bJFk984x/bU6pl/eI5Jr/I8AlxwzbRXwEWB+BJcrIsBVEwdwuLWDeevrvY4iMSZixe+cmw8cOGbaWufc+kgtU0TecXZlAflZacxZsdvrKBJjtI1fJEH5U3xcOq4/L63dS3Nrh9dxJIbEbPGb2W1mVm1m1fX1+qgqcjquGF9GS3uAv63d63UUiSExW/zOuZnOuSrnXFVRUZHXcUTi0tSKfEpy05mzvNbrKBJDYrb4RaT3fD7j8vFlvLKhjoYj7V7HkRgRycM5ZwNvACPNrMbMbjWzq82sBjgLeM7M5kZq+SISdMWEMto7HXPXaMROCfJH6oGdc9cf51d/itQyReS9JpTnMSg/kznLd/OxqoFex5EYoE09IgnOzLhiQimvb97PvqZWr+NIDFDxiySBKyaU0RlwPL9SO3lFxS+SFEaW5DCsOJs5K1T8ouIXSQpmxhXjy1i07QB7Glq8jiMeU/GLJInLJ5TiHDynzT1JT8UvkiQqi7IZU5rLsxq7J+mp+EWSyOUTSlm64xA7DxzxOop4SMUvkkSuGF8GaHNPslPxiySRgfmZTBjYV5t7kpyKXyTJXDG+lFW7Gtm6r9nrKOIRFb9IkrlsfCkAzy7XWn+yUvGLJJnSvD5MrejHs/oyV9LqUfGbWaWZpYeun29md5tZ38hGE5FIuWJCGev3Hmb9nsNeRxEP9HSN/ymg08yGAQ8BQ4AnIpZKRCLqQ2NL8RnayZukelr8AedcB3A18N/OuS8CpZGLJSKRVJSTztmVhcxZvhvnnNdxJMp6WvztZnY9cBPwbGhaamQiiUg0XDGhlG37j7ByV4PXUSTKelr8nyZ41qx7nXNbzWwI8FjkYolIpF1yRimpKcYcHd2TdHpU/M65Nc65u51zs82sH5DjnPt+hLOJSATlZaZy3oginl1RSyCgzT3JpKdH9cwzs1wzyweWAw+b2U8jG01EIu2KCWXUNrRQvf2g11Ekinq6qSfPOdcIfAR42Dk3BbgocrFEJBouGl1CRqpPm3uSTE+L329mpcDHeGfnrojEuax0PxeOLuEvK2vp6Ax4HUeipKfF/21gLrDZObfIzIYCGyMXS0Si5YrxZexvbuP1zfu9jiJR0tOdu39wzo13zt0Rur3FOffRyEYTkWg4f2QROel+ntHmnqTR05275Wb2JzOrM7O9ZvaUmZVHOpyIRF5GagofHNufv67aw5G2Dq/jSBT0dFPPw8AzQBkwAJgTmiYiCeDjUwfS1NrBcxq4LSn0tPiLnHMPO+c6QpdHgKII5hKRKKoa3I+hRVn8btFOr6NIFPS0+PeZ2Y1mlhK63AhoT5BIgjAzrps6kOrtB9lUpxE7E11Pi/8Wgody7gFqgWsIDuMgIgniI5PL8ftMa/1JoKdH9exwzl3pnCtyzhU7564i+GUuEUkQhdnpXDymhKeW7KKtQ8f0J7LenIHrS2FLISIx4eNTB3KguY2/rd3rdRSJoN4Uv4UthYjEhHOHF1GWl8FvtbknofWm+E84nJ+ZzQod97+qy7R8M3vRzDaGfvbrxfJFJMxSfMa1VQN5dWM9NQePeB1HIuSExW9mh82ssZvLYYLH9J/II8Alx0z7GvCSc2448FLotojEkGurgt/NnL1wh8dJJFJOWPzOuRznXG43lxznnP8k950PHDhm8oeBX4eu/xq46rSTi0hElPfL5KLRJTzx1g5a2ju9jiMR0JtNPaejxDlXCxD6WRzl5YtID9x6zhAOHmnnj0t2eR1FIiDaxd9jZnabmVWbWXV9fb3XcUSSyplD8jmjLJdZC7bqZOwJKNrFvzc0rj+hn3XHm9E5N9M5V+Wcqyoq0ugQItFkZtx6zhA21TXxygateCWaaBf/M8BNoes3AU9Hefki0kOXjy+jOCedh17b6nUUCbOIFb+ZzQbeAEaaWY2Z3Qp8H7jYzDYCF4dui0gMSvP7+NRZg3l14z427NX4PYkkYsXvnLveOVfqnEt1zpU75x5yzu13zl3onBse+nnsUT8iEkM+ceZg0v0+ZmmtP6HE7M5dEfFeflYaH5k8gD8u3UXd4Rav40iYqPhF5IRum1FJZ8Dx4CtbvI4iYaLiF5ETGlKYxVUTB/DYm9upa9RafyJQ8YvISd194TA6Ao4H5m32OoqEgYpfRE5qcEEW10wu54mFO9jToLX+eKfiF5EeufOCYQQCjgfmbfI6ivSSil9EemRgfibXVg3ktwt3svvQUa/jSC+o+EWkx+68YBgOx/0va60/nqn4RaTHBvTtw3VTB/H7RTvZtq/Z6zhymlT8InJK7rpgGP4U46cvbvA6ipwmFb+InJLi3Axued8Qnlm+m9W7G7yOI6dBxS8ip+z2GZXkZvj58dz1XkeR06DiF5FTlpeZyh3nD+Pl9fUs2qaxFuONil9ETsvNZ1dQnJPOD55fp7N0xRkVv4iclj5pKdx14XCqtx/k7+uOezI9iUEqfhE5bddNHciQwiy+9/w6OjoDXseRHlLxi8hpS03x8dVLRrKpronfV9d4HUd6SMUvIr3ywTP6UzW4Hz99cQPNrR1ex5EeUPGLSK+YGV+/dDT7mlqZOV8na4kHKn4R6bUpg/tx2bhSZs7fwl6drCXmqfhFJCz+9ZKRdAQC/ExDOcQ8Fb+IhMXggiw+dVYFv6veqS91xTgVv4iEzZcuHsGAvn34yh+Wc7St0+s4chwqfhEJm6x0Pz+8Zjzb9h/hRxrHJ2ap+EUkrM6uLOST0wfz8OtbtcknRqn4RSTsvvahUdrkE8NU/CISdl03+fzkBW3yiTUqfhGJiLMrC7l+2iAefn0bq3bphC2xRMUvIhHztUtG0S8zlXv+tJLOgIZujhUqfhGJmLzMVP7j8jEsr2ng8be2ex1HQlT8IhJRV04o49zhhfzwr+s1nEOM8KT4zezzZrbKzFab2Re8yCAi0WFmfOfDY2nrDPDtOWu8jiN4UPxmNhb4DDANmABcbmbDo51DRKKnojCLuy8YxnMra3l62S6v4yQ9L9b4RwNvOueOOOc6gFeAqz3IISJRdPt5lUyryOdrT61k3Z5Gr+MkNS+KfxUww8wKzCwTuBQY6EEOEYmi1BQf998wiZwMP599dDENR9u9jpS0ol78zrm1wA+AF4G/AsuB95y2x8xuM7NqM6uur6+PckoRiYTinAz+94bJ1Bw8yr/8fhkBHeLpCU927jrnHnLOTXbOzQAOABu7mWemc67KOVdVVFQU/ZAiEhFTK/K557LR/G1tHQ/M2+R1nKTk1VE9xaGfg4CPALO9yCEi3rj57AqunFDGT17cwPwN+kQfbV4dx/+Uma0B5gD/7Jw76FEOEfGAmfH9j45jRHEOd/92KTsPHPE6UlLxalPPuc65Mc65Cc65l7zIICLeykzz88tPTqGz03HH44tpadcontGib+6KiGeGFGbxs49PZNWuRr7x9Cqc087eaFDxi4inLhpTwl0XDOP31TX8btFOr+MkBRW/iHjuCxeN4NzhhfznM6tZW6svd0Wail9EPJfiM3728Yn0zUzlnx9fQlPre77aI2Gk4heRmFCYnc59101i2/5m/u2PK7W9P4JU/CISM84cWsC/fGAkzyzfzRMLd3gdJ2Gp+EUkptxxXiXnjSjiW3PW6JSNEaLiF5GY4gtt7y/MSuOzjy3m0JE2ryMlHBW/iMSc/Kw0HrhxCnsbW/jS75drMLcwU/GLSEyaOLAv37h8DH9fp8Hcwk3FLyIx68bpg7lqYnAwt1c3ajC3cFHxi0jMMjO++5FxDC/O5u7ZGswtXFT8IhLTMtP8PPjJKjoCjtsfXczRNg3m1lsqfhGJeUMKs7jvukms3dPI1/+4Ql/u6iUVv4jEhfePKuZfLh7Bn5ftZtaCbV7HiWsqfhGJG587fxgfPKOE7/5lLa9v3ud1nLil4heRuOHzGT/52EQqCjK564ml1DYc9TpSXFLxi0hcyU4P7uxtae/kjseW0Nqhnb2nSsUvInFnWHE2P752Ast2HuLbc9Z4HSfuqPhFJC59aFwpt583lMff2sEfqnXmrlOh4heRuPWVD4zk7MoC/v3Pq1izW2fu6ikVv4jELX+Kj/uun0Ren1TufEJn7uopFb+IxLXC7HTuu75nZ+56eX0dTy6uiWK62OT3OoCISG9NH1rAly4ewY9f2MCZQ/O54czB75nnYHMbd89eyuGWDg42t/GZGUM9SBobtMYvIgnhc+cP49zhhcc9c9f9L2+iubWDc4YVcu9f1vLoG9uinjFWqPhFJCH4fMZ/f3wi/TJT+exjiznY/M6Zu3YeOMKjb2znminlzLp5KheNLuY/nl6dtEcDqfhFJGEUZKfzyxunUNfYyl2zl9LRGQDgJy+sxwy+ePEI0vw+7v/EZM4dXshXn1rBM8t3e5w6+lT8IpJQJg3qx39dPZbXNu3jB39dx6pdDfx52W5uOWcIpXl9AMhITeHBT06hqiKfL/5uGc+uSK7y185dEUk4H6sayOpdDfzq1a08v2oPfTNT+ex5le+aJzPNz8M3T+Xmhxfy+d8uw2fGpeNKPUocXVrjF5GE9O+Xj2HakHxqDh7lzvcPI69P6nvmyUr38/CnpzFpYF/umr2U51fWepA0+jwpfjP7opmtNrNVZjbbzDK8yCEiiSs1xceDN07hv64ay6fOqjjufNnpfh65ZRoTQ+X/0tq90QvpkagXv5kNAO4GqpxzY4EU4Lpo5xCRxNcvK40bpw8mzX/iqstO9/PIp6cypiyXOx5fkvBj/Xu1qccP9DEzP5AJJNeeFRGJOTkZqfz609OoKMjkn35dzZIdB72OFDFRL37n3C7gx8AOoBZocM69EO0cIiLH6peVxmO3nklRTjo3z1rI8p2HvI4UEV5s6ukHfBgYApQBWWZ2Yzfz3WZm1WZWXV9fH+2YIpKkinMzeOzWM8lK93PVAwu4a/ZSNtU1nfbjOecIBGLr5PAW7bPVm9m1wCXOuVtDtz8FTHfOfe5496mqqnLV1dXRiigiwsHmNma+uoVfv76NlvZOrphQxofG9mdqRT4F2eknvG8g4Fi07QDPrazlLyv30BEIcMkZ/bl8fBnTh+bjT4nOOreZLXbOVb1nugfFfyYwC5gKHAUeAaqdc/9zvPuo+EXEK/ubWpk5fwuPvrmdI23B0zxWFmUxoiSHgHN0BhztnY72zgCtHQFaOzrZ09DKvqZWMlJ9XDCqmNQUH39bs5fmtk4KstKYNKgvI0pyGNk/h4qCLLLSU0j3p9AnLYX8zDR8PgtL9pgp/lCYbwEfBzqApcA/Oedajze/il9EvNbWEWDlrgYWbj3Awq372XHgCH6fD3+KkeIz0v0+0v0ppPt95PZJ5fyRRVw0uoSs9OD3ZFvaO3l5XR1zV+9hTW0jW+qb6ehmE9Co/jnce/VYpgzO73XmmCr+U6XiF5FE09YRYOu+ZnYeOMLR9k6OtnfScKSdWQu2UtvQwvXTBvLVS0bRNzPttJeh4hcRiQPNrR389982MGvBNvL6pPK/n5jMWZUFp/VYxyt+DdkgIhJDstL93HPZGJ696xzOKMtlSGFW2JehQdpERGLQ6NJcHr31zIg8ttb4RUSSjIpfRCTJqPhFRJKMil9EJMmo+EVEkoyKX0Qkyaj4RUSSjIpfRCTJxMWQDWZWDxwCGo75Vd5Jpp3s+ts/C4HTOddad8s/lXwny9xd1q6/P53cp5P5RLm6u91d1t4819HM3PV6rL8+YiVzd9P1+ji5aLw++jrnit7zyM65uLgAM0912smud/lZHa5M4cx8nKxd5z3l3KeT+US5evL89va5jmbmeHp9xEpmvT5i//Vx7CWeNvXMOY1pJ7ve3f17m+lkvz+VzF1ve5m5u+knut1d1t7kjmbmrtdj/fURK5m7m67Xx8lF8/XxLnGxqSfSzKzadTOCXayLx9zKHB3xmBniM3c8Zo6nNf5Imul1gNMUj7mVOTriMTPEZ+64y6w1fhGRJKM1fhGRJJNwxW9ms8yszsxWncZ9p5jZSjPbZGb3mZl1+d1dZrbezFab2Q9jPbOZfdPMdpnZstDl0nBmjlTuLr//spk5MysMX+KIPdffMbMVoef5BTMri4PMPzKzdaHcfzKzvnGQ+drQ/7+AmYVtm3pvsh7n8W4ys42hy01dpp/wNR9Vp3MYUixfgBnAZGDVadx3IXAWYMDzwIdC098P/A1ID90ujoPM3wS+HG/Pdeh3A4G5wHagMNYzA7ld5rkb+GUcZP4A4A9d/wHwgzjIPBoYCcwDqrzOGspRccy0fGBL6Ge/0PV+J/q7vLgk3Bq/c24+cKDrNA2/PHsAAAYZSURBVDOrNLO/mtliM3vVzEYdez8zKyX4H/gNF/xX+g1wVejXdwDfd861hpZRFweZIy6CuX8G/CsQ9h1QkcjsnGvsMmtWuHNHKPMLzrmO0KxvAuVxkHmtc259OHP2JutxfBB40Tl3wDl3EHgRuMTr/6vHSrjiP46ZwF3OuSnAl4EHuplnAFDT5XZNaBrACOBcM3vLzF4xs6kRTRvU28wAd4Y+ys8ys36Ri/ouvcptZlcCu5xzyyMdtIteP9dmdq+Z7QRuAL4RwaxvC8fr4223EFwDjbRwZo60nmTtzgBgZ5fbb+ePlb8LSIJz7ppZNnA28Icum9TSu5u1m2lvr7n5CX5smw5MBX5vZkND79xhF6bMvwC+E7r9HeAnBP+DR0xvc5tZJnAPwc0QURGm5xrn3D3APWb2deBO4D/DHPWdIGHKHHqse4AO4PFwZnxPkDBmjrQTZTWzTwOfD00bBvzFzNqArc65qzl+fs//rq4SvvgJfqo55Jyb2HWimaUAi0M3nyFYlF0/7pYDu0PXa4A/hop+oZkFCI7PUR+rmZ1ze7vc71fAsxHK2lVvc1cCQ4Dlof9w5cASM5vmnNsTo5mP9QTwHBEsfsKUObTj8XLgwkitxHQR7uc5krrNCuCcexh4GMDM5gE3O+e2dZmlBji/y+1ygvsCavD+73qHVzsXInkBKuiyowZ4Hbg2dN2ACce53yKCa/Vv73y5NDT9s8C3Q9dHEPwoZzGeubTLPF8EfhsPz/Ux82wjzDt3I/RcD+8yz13Ak3GQ+RJgDVAUiddFJF8bhHnn7ulm5fg7d7cS3ELQL3Q9v6ev+WhdPFloRP8gmA3UAu0E32VvJbgW+VdgeejF/o3j3LcKWAVsBu7nnS+4pQGPhX63BLggDjI/CqwEVhBckyoNZ+ZI5T5mnm2E/6ieSDzXT4WmryA4PsqAOMi8ieAKzLLQJdxHIkUi89Whx2oF9gJzvcxKN8Ufmn5L6PndBHz6VF7z0brom7siIkkmWY7qERGREBW/iEiSUfGLiCQZFb+ISJJR8YuIJBkVv8QlM2uK8vL+z8zGhOmxOi04kucqM5tzspExzayvmX0uHMsWAZ2IReKUmTU557LD+Hh+986gZRHVNbuZ/RrY4Jy79wTzVwDPOufGRiOfJD6t8UvCMLMiM3vKzBaFLu8LTZ9mZq+b2dLQz5Gh6Teb2R/MbA7wgpmdb2bzzOxJC45V//jbY6aHpleFrjeFBmVbbmZvmllJaHpl6PYiM/t2Dz+VvME7A9Rlm9lLZrbEguO2fzg0z/eBytCnhB+F5v1KaDkrzOxbYXwaJQmo+CWR/Bz4mXNuKvBR4P9C09cBM5xzkwiOnPndLvc5C7jJOXdB6PYk4AvAGGAo8L5ulpMFvOmcmwDMBz7TZfk/Dy3/pOOwhMapuZDgN6sBWoCrnXOTCZ4D4iehN56vAZudcxOdc18xsw8Aw4FpwERgipnNONnyRN6WDIO0SfK4CBjTZUTFXDPLAfKAX5vZcIIjIqZ2uc+LzrmuY7EvdM7VAJjZMoJjuLx2zHLaeGfQu8XAxaHrZ/HOGOtPAD8+Ts4+XR57McEx2yE4hst3QyUeIPhJoKSb+38gdFkaup1N8I1g/nGWJ/IuKn5JJD7gLOfc0a4Tzex/gJedc1eHtpfP6/Lr5mMeo7XL9U66/z/S7t7ZOXa8eU7kqHNuopnlEXwD+WfgPoJj+RcBU5xz7Wa2Dcjo5v4GfM859+ApLlcE0KYeSSwvEBwLHwAze3tY3TxgV+j6zRFc/psENzEBXHeymZ1zDQRP1fhlM0slmLMuVPrvBwaHZj0M5HS561zgltC48ZjZADMrDtPfIElAxS/xKtPMarpcvkSwRKtCOzzXEBxOG+CHwPfMbAGQEsFMXwC+ZGYLgVKg4WR3cM4tJTgC5HUET4ZSZWbVBNf+14Xm2Q8sCB3++SPn3AsENyW9YWYrgSd59xuDyAnpcE6RMAmdQeyoc86Z2XXA9c65D5/sfiLRpm38IuEzBbg/dCTOISJ8qkuR06U1fhGRJKNt/CIiSUbFLyKSZFT8IiJJRsUvIpJkVPwiIklGxS8ikmT+HxtmdwYYUOpOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47min 55s, sys: 2min 30s, total: 50min 25s\n",
      "Wall time: 44min 29s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for i in range(3):\n",
    "    learn.fit_one_cycle(5, 1e-3, wd=0.2)\n",
    "    create_submission_filename(learn, f'submission_standard_stage_{i+1}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# learn.fit_one_cycle(5, 1e-3, wd=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# With quantile normalization and powers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reset the variables we will use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_vars = ['Store', 'DayOfWeek', 'Year', 'Month', 'Day', 'StateHoliday', 'CompetitionMonthsOpen',\n",
    "    'Promo2Weeks', 'StoreType', 'Assortment', 'PromoInterval', 'CompetitionOpenSinceYear', 'Promo2SinceYear',\n",
    "    'State', 'Week', 'Events', 'Promo_fw', 'Promo_bw', 'StateHoliday_fw', 'StateHoliday_bw',\n",
    "    'SchoolHoliday_fw', 'SchoolHoliday_bw', 'Promo', 'SchoolHoliday', 'CloudCover']\n",
    "\n",
    "\n",
    "cont_vars = ['CompetitionDistance', 'Max_TemperatureC', 'Mean_TemperatureC', 'Min_TemperatureC',\n",
    "   'Max_Humidity', 'Mean_Humidity', 'Min_Humidity', 'Max_Wind_SpeedKm_h', \n",
    "   'Mean_Wind_SpeedKm_h', 'trend', 'trend_DE',\n",
    "   'AfterStateHoliday', 'BeforeStateHoliday']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will do the preprocessing ourselves:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of categorical variables: 26\n",
      "Number of continuous variables: 39\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(844338, 67)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "procs = []\n",
    "\n",
    "categorify_proc = Categorify(cat_names=list(cat_vars), cont_names=list(cont_vars))\n",
    "quantile_proc = custom_transform.QuantileNormalize(cat_names=list(cat_vars), cont_names=list(cont_vars))\n",
    "power_proc = custom_transform.Power(cat_names=list(cat_vars), cont_names=list(cont_vars))\n",
    "fill_missing_proc = FillMissing(cat_names=list(cat_vars), cont_names=list(cont_vars))\n",
    "\n",
    "tmp_valid_df = train_df.iloc[list(valid_idx)].copy()\n",
    "tmp_train_df = train_df.iloc[(valid_idx[-1] + 1):].copy()\n",
    "assert tmp_valid_df.shape[0] + tmp_train_df.shape[0] == train_df.shape[0]\n",
    "\n",
    "fill_missing_proc.apply_train(tmp_train_df)\n",
    "categorify_proc.apply_train(tmp_train_df)\n",
    "quantile_proc.apply_train(tmp_train_df)\n",
    "power_proc.apply_train(tmp_train_df)\n",
    "\n",
    "fill_missing_proc.apply_test(tmp_valid_df)\n",
    "categorify_proc.apply_test(tmp_valid_df)\n",
    "quantile_proc.apply_test(tmp_valid_df)\n",
    "power_proc.apply_test(tmp_valid_df)\n",
    "\n",
    "fill_missing_proc.apply_test(test_df)\n",
    "categorify_proc.apply_test(test_df)\n",
    "quantile_proc.apply_test(test_df)\n",
    "power_proc.apply_test(test_df)\n",
    "\n",
    "# put everything back together\n",
    "new_cont_vars = itertools.chain(*[[var, var + '_square', var + '_sqrt']\n",
    "                                  for var in cont_vars])\n",
    "new_cont_vars = list(new_cont_vars)\n",
    "\n",
    "new_cat_vars = cat_vars + ['CompetitionDistance_na']\n",
    "\n",
    "train_df = pd.concat([tmp_valid_df, tmp_train_df], axis=0)\n",
    "\n",
    "df = train_df[new_cat_vars + new_cont_vars + [dep_var, 'Date']].copy()\n",
    "\n",
    "data = (TabularList.from_df(df, path=DATA_DIR, cat_names=cat_vars, cont_names=cont_vars, procs=procs, )\n",
    "        .split_by_idx(valid_idx)\n",
    "        .label_from_df(cols=dep_var, label_cls=FloatList, log=True)\n",
    "        .add_test(TabularList.from_df(test_df, path=DATA_DIR, cat_names=cat_vars, cont_names=cont_vars))\n",
    "        .databunch())\n",
    "\n",
    "print(f'Number of categorical variables: {len(new_cat_vars)}')\n",
    "print(f'Number of continuous variables: {len(new_cont_vars)}')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_log_y = np.log(np.max(train_df['Sales'])*1.2)\n",
    "y_range = torch.tensor([0, max_log_y], device=defaults.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = tabular_learner(data, layers=[1000,500], ps=[0.001,0.01], emb_drop=0.04, \n",
    "                        y_range=y_range, metrics=exp_rmspe)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "View model's architecture:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TabularModel(\n",
       "  (embeds): ModuleList(\n",
       "    (0): Embedding(1116, 81)\n",
       "    (1): Embedding(8, 5)\n",
       "    (2): Embedding(4, 3)\n",
       "    (3): Embedding(13, 7)\n",
       "    (4): Embedding(32, 11)\n",
       "    (5): Embedding(3, 3)\n",
       "    (6): Embedding(26, 10)\n",
       "    (7): Embedding(27, 10)\n",
       "    (8): Embedding(5, 4)\n",
       "    (9): Embedding(4, 3)\n",
       "    (10): Embedding(4, 3)\n",
       "    (11): Embedding(24, 9)\n",
       "    (12): Embedding(9, 5)\n",
       "    (13): Embedding(13, 7)\n",
       "    (14): Embedding(53, 15)\n",
       "    (15): Embedding(22, 9)\n",
       "    (16): Embedding(7, 5)\n",
       "    (17): Embedding(7, 5)\n",
       "    (18): Embedding(4, 3)\n",
       "    (19): Embedding(4, 3)\n",
       "    (20): Embedding(9, 5)\n",
       "    (21): Embedding(9, 5)\n",
       "    (22): Embedding(3, 3)\n",
       "    (23): Embedding(3, 3)\n",
       "    (24): Embedding(10, 6)\n",
       "  )\n",
       "  (emb_drop): Dropout(p=0.04)\n",
       "  (bn_cont): BatchNorm1d(13, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (layers): Sequential(\n",
       "    (0): Linear(in_features=236, out_features=1000, bias=True)\n",
       "    (1): ReLU(inplace)\n",
       "    (2): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): Dropout(p=0.001)\n",
       "    (4): Linear(in_features=1000, out_features=500, bias=True)\n",
       "    (5): ReLU(inplace)\n",
       "    (6): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (7): Dropout(p=0.01)\n",
       "    (8): Linear(in_features=500, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do the training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU9d328c83+x4CTCALEHZlhwQErGgVfax1w12rdWmli3Wt93O3d5+2drvbalvrUiu2at21dalLW9GqiAsIAdkF2SFAIBCyERKy/J4/ZqwRA4RkJmeW6/16zSszJ2fmXBmGa86cc+Z3zDmHiIjEjjivA4iISPdS8YuIxBgVv4hIjFHxi4jEGBW/iEiMSfA6QEf07t3bFRUVeR1DRCSiLFq0aLdzznfw9Igo/qKiIkpLS72OISISUcxsc3vTtalHRCTGqPhFRGKMil9EJMao+EVEYoyKX0Qkxqj4RURijIpfRCTGRMRx/J315uqdrN1Zx5jCHowqyCIzJdHrSCIinovq4p+zpoJH5/m/v2AGg3qnc8JQH6eP6svEop7Ex5knuQ40t7L/QAv1Tc0kJ8STk5aImTdZRCT2WCSciKWkpMR19pu7e+oaWb6tmmVl1Xy4ZS/vr99DY3MrvTOSOH1UX2aeMJj+vdKCknNrZT0PvruRpWVVZCQnkJWSSEZyArWNTeyobmBndQMVdY00tXz2OU9KiCMvO4W+WSn0SEskIzmRzJQEUhLjaXWO5hZHq3P4MpMZkZ/FqPxsfJnJQcksItHLzBY550o+Nz3ai/9g+xqbeWvNLv61opx/r9pJq3NcNqk/3zl5KL7MZJpbWlm7q46PdtTQ2NyKAXFmZKYkMHlQL3LSkz7zeM45Vm6v4YG5G/jH8h3EGRQPyKGhqZXahiZqG5rJSEkIFHsqvsxkMpLjSU1KIC0pnoamFsqrG9hR3UB5dQM1gfvUNjTR0NRKfJwRH2fEGdQ0NP9nuX2ykjl+cG9OHO7jhKE+eqYnUdPQxObd9ZTtrad4QA65WSlBec5EJDKp+NtRXt3A3W+u5ZmFW0mKj2N430xWl9fQ0NTa7vxxBuP753DSMB9m8OGWKj7cWkXlvgOkJ8XzlckDuPr4IvKyU4OeFaCmoYlV22tYub2GJVureHdtBXvrmzCDHqmJ7K1v+s+8mckJ/PeXjuGySf2J82iTloh4S8V/GBt37+PuN9ayrWo/owuyGVOYzcj8bDKSE2h1Dof/TeLtjyuYs2YXy8qqARjsS2d8/xwm9M/hy2PyyE7t3p3HLa2O5duqmbNmFztrGhnQK42iXun0TE/irjc+5r11e5hYlMMvzxvNkNzMbs0mIt5T8QdR5b4DxJuRnRa+Rwk553h2URk//8dH1DQ0MbxPJscN7MnEgT05bmCvI+4jqKhtZNHmSmoamklNjCclMZ6M5ATG9+9BSmJ8N/0VItIVKv4YtbuukacXbOGDjZUs2ryX+gMtAAzNzWDq4F5MGtiL+Dioqm+ian8TWyrr+WDDHtZX7Gv38dKT4jn52D6cMaov04b5SE+O6gPDRCKail9oamll5fYa5m/Yw/vr97BwYyX7m1o+M09mcgIlRTlMGtiLSQN7kpuZTGNzCw1NrVTUNvLaqp3MXllO5b4DAORlpzCwdzoDe6czIj+Lcf16MLxPJgnxceypa2T+hkrmb9jj3w+SHE9GciJpSfHUNTazt/4Ae+ubaG11DMnNYHjfTIb3zWSwL6PbN5uJRCMVv3zOgeZWVpfXkBAXR4+0RHqkJZKaGH/E7xQ0t7SyYGMlpZv3smn3Pjbu2cf6XXX/OeooNTGevtkpbNzt/9SQnuS/va+xhbrGZvYdaCYjKYGc9CRy0hJxwLpddf/5NALQMz2Jol5pDPZlcOJwH18cnqtPFyJHScUvIeWcY0tlPUu2VrFkaxVle/czrl8PpgzuxeiCbBLj4z4z78FvLq2tjm1V+1ldXsuGijo27dnHxt37WFNey976JpIT4pg2zMfEohySE+JJiDcS4+Pok5VCv5xUCnJSSU7QvgeRtlT8EpFaWh0LN1Xy6opyXl1RTnlNQ7vzmUF+diqjC7IZXZjN2MIejOmXTZaG6ZAYpuKXiOeco7axmabmVppbHY1NreysbWDLnnq2VNazvqKO5duq2bynHvC/GQzLzWTCgByKB+RwbJ5//4GOSpJYcaji10ZTiRhm9rk1+P690phY1PMz06rqD7CsrJolW6tYtHkvryzbzlMLtgD+L+EV9UpnWJ9MRuRnMSIvixH5WeRlp2i8JIkZKn6JOj3Skpg2zMe0YT7Av/9gw+46VpfX8vHOOj4ur2V1eQ2zV5XzyQfenLRERhVkM6rAv5no5GNySUrQqOUSnVT8EvXi4owhuZmf+/byvsZmVpfXsmp7NSu21bBiezV/fmcDTS2OwpxUbjh5KDMmFHxmx7RINNA2fpE2GptbeG/dbn7/77UsK6tmQK80bpo+lHPGFmjMI4k4h9rGr1UZkTaSE+I5+Zg+vHjd8fz5qyWkJyVw8zNLOfe+9yjdVOl1PJGgUPGLtMPMmD6iD69c/wV+e+FYdtY0cMH987juicVsq9rvdTyRLlHxixxGXJxxfnEhb916EjeeMpQ3Vu/ktN+9zWPzNtHaGv6bSUXao+IX6YC0pARuPnUYr998IhMG5PDDF1dy8QPzWF9R53U0kaOm4hc5Cv16pvHoNZO444IxrCmv5fTfz+V7zy1j0+72RzMVCUc6nFPkKJkZF5b048ThPu55Yx3PlG7lr6VbOXNMPjdOH8pgX4bXEUUOS4dzinTRrtoGHnxnI4/P30yrgzsuHMOZY/K9jiWiwzlFQiU3M4Xvn3Esb916EiPzs/jOkx/yy399RIt2/kqYClnxm9lDZrbLzFa0mXahma00s1Yz+9y7kEgky81K4clrJ3P55P7MensDVz284D/nJBAJJ6Fc4/8LcPpB01YA5wFzQ7hcEc8kJcTx83NHc/v5Y/hgYyVf/M0crnjwA15ftVOfACRshGznrnNurpkVHTTtI0CjIErUu2hiP046xsczC7byxAdbuPbRUobkZvDINZMo6JHqdTyJcWG7jd/MZppZqZmVVlRUeB1H5KjlZqZw/SlDefe/v8gfLpvAzpoGLp41j62V9V5HkxgXtsXvnHvAOVfinCvx+XxexxHptIT4OL48Jo8nvn4cNfubuOSB+Sp/8VTYFr9ItBlT2IMnr53MvgPNXDxrHpv3aMeveEPFL9KNRhVk8+TXJ7O/qYWLZmnIB/FGKA/nfAqYBww3szIz+5qZzTCzMmAK8A8zmx2q5YuEqxH5WTw9cwotrY6LZ81n7c5aryNJjNE3d0U8sm5XHZf9aT4trY7Hv34cx+ZleR1Jooy+uSsSZobkZvDMN6aQlBDHpX+az7pdWvOX7qHiF/HQwN7pPDNzCglxxrWPLqJ6f5PXkSQGqPhFPNa/Vxp/vLyYsr313PDUh/qGr4Scil8kDEws6sltZ4/k7Y8ruH32aq/jSJTTePwiYeIrxw1g1fYaZr29gRF5WZwzrsDrSBKltMYvEkZ+fNZIJhX15HvPLddhnhIyKn6RMJKUEMe9l40nPTmebz+xmPoDzV5Hkiik4hcJM7lZKdx58TjWVdTx4xdXeh1HopCKXyQMnTDUx/VfHMLfFpXx7KIyr+NIlFHxi4SpG6cPY/Kgnvzw7yu0vV+CSsUvEqbi44y7LxlPSmIc33t+Oa06vl+CRMUvEsZys/wncl+0eS/PLtYmHwkOFb9ImLtgQiHFA3L41b9WU1V/wOs4EgVU/CJhLi7O+Pm5o6je38Tts9d4HUeigIpfJAIcm5fFVVOLeGrBFpZsrfI6jkQ4Fb9IhLhp+lB8Gcn8v78v10Bu0iUqfpEIkZmSyP+ccSwrttXw6opyr+NIBFPxi0SQs8bmM8iXzj1vriUSzp4n4UnFLxJB4uOM604awuryWt74aJfXcSRCqfhFIszZ4/Lp1zOVe95ap7V+6RQVv0iESYyP41snDmHp1ireXbfb6zgSgVT8IhHo/OIC+malcM+b67yOIhFIxS8SgZIT4vnGiYNYsLGSDzbs8TqORBgVv0iEumRif3pnJPHHt9d7HUUijIpfJEKlJsVz+eQBzFlTwYaKOq/jSARR8YtEsMuO609ivPHY/M1eR5EIouIXiWC5mSmcMTqPZ0vL2Neo8/NKx6j4RSLclVOLqG1s5nmN1y8dpOIXiXDj+/VgTGE2j8zbrC90SYeo+EUinJnx1SlFrNtVx/vrdWinHJmKXyQKnDkmj57pSfzl/U1eR5EIELLiN7OHzGyXma1oM62nmb1uZmsDP3NCtXyRWJKSGM+lk/rxxkc72VpZ73UcCXOhXOP/C3D6QdO+B7zhnBsKvBG4LSJB8JXjBuCA57STV44gZMXvnJsLVB40+RzgkcD1R4BzQ7V8kViT3yOVqYN78cKH27STVw6ru7fx93HO7QAI/Mw91IxmNtPMSs2stKKiotsCikSyc8cVsHlPPR/qvLxyGGG7c9c594BzrsQ5V+Lz+byOIxIRTh/Vl+SEOP7+4Tavo0gY6+7i32lmeQCBnzqFkEgQZaYkcuqIPry8dDtNLa1ex5Ew1d3F/xJwZeD6lcCL3bx8kag3Y3wBe+ubmPuxNpFK+0J5OOdTwDxguJmVmdnXgF8Bp5rZWuDUwG0RCaJpw3z0TE/iBW3ukUNICNUDO+cuPcSvTgnVMkXEf2rGs8bk8fTCrdQ0NJGVkuh1JAkzYbtzV0Q679zxBTQ2t/LqinKvo0gYUvGLRKFx/XpQ1CuNFxZrc498nopfJAqZGTPGFzJ/4x62V+33Oo6EGRW/SJSaMb4A59BOXvkcFb9IlOrfK41JRT15fnGZhnCQz1Dxi0Sx84sLWF+xj6Vl1V5HkTCi4heJYl8anUdyQpxOyyifoeIXiWJZKYmcNrIvLy3dTmNzi9dxJEyo+EWi3PkTCqiqb+Kt1RrCQfxU/CJR7gtDeuPLTNbmHvkPFb9IlEuIj+Pccfm8tWYXlfsOeB1HwoCKXyQGnF9cSFOL46UlOqZfVPwiMeGYvlmMyMvieX2ZS1Dxi8SM84sLWVZWzdqdtV5HEY+p+EVixDnj8kmIM57VTt6Yp+IXiRG9M5I5abiPv3+4jZZWDeEQy1T8IjHkvAmF7Kxp5L11u72OIh5S8YvEkFOOzSU7NZHntLknpnWo+M1ssJklB66fZGY3mFmP0EYTkWBLTojnrLF5zF5ZTm1Dk9dxxCMdXeN/DmgxsyHAg8BA4MmQpRKRkDl/QiENTa38c/kOr6OIRzpa/K3OuWZgBvB759zNQF7oYolIqIzr14NBvdN5TqdljFkdLf4mM7sUuBJ4JTAtMTSRRCSUzIzziwtZsLGSzXv2eR1HPNDR4r8amAL8wjm30cwGAo+HLpaIhNL5EwqJjzOe/GCL11HEAx0qfufcKufcDc65p8wsB8h0zv0qxNlEJET6Zqdw2og+PFO6lYYmjdMfazp6VM8cM8sys57AUuBhM/tdaKOJSChdMWUAVfVNvLx0u9dRpJt1dFNPtnOuBjgPeNg5VwxMD10sEQm1KYN6MTQ3g8fmb/Y6inSzjhZ/gpnlARfx6c5dEYlgZsYVUwawrKyaJVurvI4j3aijxf9TYDaw3jm30MwGAWtDF0tEusOM8QWkJ8Xz6LxNXkeRbtTRnbt/c86Ncc59K3B7g3Pu/NBGE5FQy0xJ5LwJhbyybIfOzhVDOrpzt9DMXjCzXWa208yeM7PCUIcTkdC7YsoADjS38szCrV5HkW7S0U09DwMvAflAAfByYJqIRLhhfTKZPKgnj7y/SYd2xoiOFr/POfewc645cPkL4AthLhHpRjecPJTymgae0Be6YkJHi3+3mV1uZvGBy+XAns4u1MxuNLMVZrbSzG7q7OOISHBMHdKb44f04r631rGvsdnrOBJiHS3+a/AfylkO7AAuwD+Mw1Ezs1HAtcAkYCxwppkN7cxjiUjw3HracPbsO8DD7230OoqEWEeP6tninDvbOedzzuU6587F/2WuzjgWmO+cqw+M+Pk2/lE/RcRD4/vnMP3YPsyau4Gqeh3hE826cgauWzp5vxXANDPrZWZpwBlAv4NnMrOZZlZqZqUVFRVdiCkiHfXd04ZR19jMrLkbvI4iIdSV4rfO3Mk59xHwa+B14FX8Y/98bqOic+4B51yJc67E59N+ZJHucGxeFmePzefh9zayq7bB6zgSIl0pftfpOzr3oHNugnNuGlCJvgUsEjZunj6M5hbH71772OsoEiKHLX4zqzWzmnYutfiP6e8UM8sN/OyPf1/BU519LBEJrqLe6Vx9fBHPlG7VGD5R6rDF75zLdM5ltXPJdM4ldGG5z5nZKvxfBLvOObe3C48lIkF24/Rh+DKS+dGLK2hp7fSHewlTXdnU02nOuROccyOcc2Odc294kUFEDi0jOYEffPlYlpVV8/RCfakr2nhS/CIS/s4em89xA3tyx+w17NUAblFFxS8i7TIzfnrOKGobmrl99hqv40gQqfhF5JCG983kyilFPL1wC4u3aFdctFDxi8hh3XzqUPpkpvA/zy+nqaXV6zgSBCp+ETmszJREbjt7JKvLa3noXY3jEw1U/CJyRKeP6supI/pw578/ZmtlvddxpItU/CLSIT85eyTxZvzwxRU4p2P7I5mKX0Q6JL9HKt89bThz1lTwyrIdXseRLlDxi0iHXTm1iDGF2fzwxRVsr9rvdRzpJBW/iHRYfJxx1yXjaWpu5canP6RZR/lEJBW/iByVgb3T+cWM0SzctJe73tDAupFIxS8iR+3c8QVcWFzIvW+t4711u72OI0dJxS8infKTc0Yy2JfBjU8vYXddo9dx5Cio+EWkU9KSErj3svHU7G/iRy+u8DqOHAUVv4h02jF9s7hx+lD+ubycV1foEM9IoeIXkS6ZOW0QI/Ky+H9/X0lVvYZvjgQqfhHpksT4OG6/YAx76w/ws1c+8jqOdICKX0S6bFRBNt88cRDPLS5jzppdXseRI1Dxi0hQXH/yUAb70vnec8vZWdPgdRw5DBW/iARFSmI8d10ynpqGJq5+eCG1DU1eR5JDUPGLSNCMKsjmvq9MYM3OWr79xGKduCVMqfhFJKhOGp7LL88bzTtrd/O955ZrCOcwlOB1ABGJPheV9GNHVQN3/vtjCnqkcMtpw72OJG2o+EUkJG44ZQjbquq5+8119OuZxoUl/byOJAEqfhEJCTPjFzNGs72qge8/v5z8HqkcP6S317EEbeMXkRBKjI/jvssnMMiXzjcfX8THO2u9jiSo+EUkxLJSEnnoqomkJMZz9cMLqajVSJ5eU/GLSMgV5qTx0JUT2V3XyHVP6jBPr6n4RaRbjC7M5tfnj2HBxkp+8Q+N6eMl7dwVkW5z7vgClm+r5sF3NzIyP0tH+nhEa/wi0q2+/6VjmDq4Fz/4+wqWbq3yOk5M8qT4zexmM1tpZivM7CkzS/Eih4h0v4T4OO69bAK+jGS++fgi7ez1QLcXv5kVADcAJc65UUA8cEl35xAR7/RMT+KBrxazt/4A33p8EY3NLV5HiilebepJAFLNLAFIA7Z7lENEPDIyP5vfXDiW0s17+fGLKzWmTzfq9uJ3zm0DfgNsAXYA1c651w6ez8xmmlmpmZVWVFR0d0wR6QZnjsnnui8O5umFW3l8/mav48QMLzb15ADnAAOBfCDdzC4/eD7n3APOuRLnXInP5+vumCLSTb576nCmH5vLT15exfvrd3sdJyZ4salnOrDROVfhnGsCngemepBDRMJAXJxx58XjKOqdzrefWMyWPfVeR4p6XhT/FmCymaWZmQGnAPo2h0gMy0xJ5M9fLQHga4/o7F2h5sU2/g+AZ4HFwPJAhge6O4eIhJei3uncd9kENuzex41PL6GlVTt7Q8WTo3qccz92zh3jnBvlnLvCOacDeUWEqUN6c9vZI3lz9S5uf3W113GiloZsEJGwcsXkAXxcXsusuRsY5Evn4on9vY4UdVT8IhJ2fnTWCDbt2ccPXlhBQY80vjBUJ3AJJo3VIyJhJzE+jvu+MoEhuRl8SydwCToVv4iEpczACVxSk/wncNlV2+B1pKih4heRsJXfI5UHr5xI5b4DXPXQQqr36zDPYFDxi0hYG12Yzf1XFLN2Vy1XPbyAusZmryNFPBW/iIS9E4f5uOfSCSwrq+brjyykoUmjeXaFil9EIsLpo/ryu4vG8sHGSr7xmIZy7goVv4hEjHPGFfDLGaN5++MKvvrgAm3z7yQVv4hElEsm9eeuS8axeMteLvjj+2yr2u91pIij4heRiHPOuAIeuWYS5TUNzPjDe6zcXu11pIii4heRiDR1cG+e/eZUEuKMS2bNp3RTpdeRIoaKX0Qi1vC+mTz37an4MpO54sEFvL9OJ3LpCBW/iES0vOxUnv7GZPr1TOXqvyxkzppdXkcKeyp+EYl4uZkpPD1zCkNyM7j20VJeX7XT60hhTcUvIlGhZ3oST147mRH52Vz3xGLeWVvhdaSwpeIXkaiRnZrIo1dPYpAvnZmPLmJhmx2+dY3N3P/2eh6Yu57WGD+7l8bjF5Gokp2WyGNfO46LZ83jmocXMuurxSzatJcH39tIVb3/C18LN+3lzovHkZEcmxWoNX4RiTq+zGSeuPY4stMSuexPH/Db1z+muH8OL153PLedNYI3V+/ivPveY/OefV5H9YQ5F/4feUpKSlxpaanXMUQkwmzZU8/D72/kvPGFjC7M/s/099bt5ttPLMYM7rl0PCcM9XmYMnTMbJFzruRz01X8IhKLNu/Zx8xHF7F2Vy3fPW043zpxMHFx5nWsoDpU8WtTj4jEpAG90nnhuqmcNTafO2avYeZjpTEz6JuKX0RiVlpSAr+/eBy3nTWCOWsqOPved1m1vcbrWCGn4heRmGZmXHX8QJ75xmQamlo474/v8fziMq9jhZSKX0QEKB7Qk1euP4GxhT245a9L+eHfV3CgudXrWCGh4hcRCfBlJvPE149j5rRBPDZ/Mxfc/z6bdkffIZ8qfhGRNhLi4/ifM47l/suL2bR7H1+++52o2/Sj4hcRacfpo/ryr5umMTI/m1v+upSbnv6Q2oboOOpHxS8icggFPVJ5auZkbp4+jJeWbudLd73zmfF/IpWKX0TkMOLjjBunD+Vv35xKnBkXz5rHb2avoaklcnf8qvhFRDqgeEAO/7zxBC4oLuTet9Zx3n3vs6a81utYndLtxW9mw81sSZtLjZnd1N05RESOVkZyArdfMJb7Ly9me9V+zrrnXf7w1jqaI2ztv9uL3zm3xjk3zjk3DigG6oEXujuHiEhnnT6qL6/dPI1TR/bhjtlrOO+P70fUN3693tRzCrDeObfZ4xwiIkelV0Yyf7hsAvd9ZQLb9u7nzHve4fvPL2dPXaPX0Y7I6+K/BHiqvV+Y2UwzKzWz0ooKnUJNRMLTGaPzePO7J3Hl1CL+VrqVk+6Yw5/f2RDWm388G5bZzJKA7cBI59xhz4ysYZlFJBKs21XHz/+xijlrKhhbmM1vLxrLkNxMz/KE47DMXwIWH6n0RUQixZDcDP5y9ST+cNkEtlTWc8bd7/LA3PW0hNk5fr0s/ks5xGYeEZFI9uUxebx284mcNMzH//5zNRfNmhdWY/54UvxmlgacCjzvxfJFRELNl5nMrCuKufPisazdWcuX7nqHR+dtojUM1v49KX7nXL1zrpdzrtqL5YuIdAczY8b4QmbfPI2JA3vyoxdXcsVDH7C1st7TXF4f1SMiEvXyslN55OqJ/PK80SzZUsVpd87l/rfXezbsg4pfRKQbmBmXTurPa7ecyBeG9uZX/1rNWfe8S+lhBn3bWdMQkh3Dnh3OeTR0OKeIRJvZK8u57aWV7KhuYGR+FheV9OOccfk0tTj+tWIHryzdwcLNlTzxteOYOqR3p5ZxqMM5VfwiIh6pa2zm2dKt/G1RGSu315AUH0dzayutDob1yeDLo/O5sKSQ/B6pnXr8QxV/QpeTi4hIp2QkJ3DV8QO56viBrNpew4tLtpGcGM+ZY/IY1id0X/xS8YuIhIER+VmMyM/qlmVp566ISIxR8YuIxBgVv4hIjFHxi4jEGBW/iEiMUfGLiMQYFb+ISIxR8YuIxJiIGLLBzCqAKuDgYZyzjzDtSNc/+dkb2N2JaO0t/2jyHSlze1nb/r4zuTuT+XC52rvdXtauPNfdmbnt9XB/fYRL5vam6/VxZN3x+ujhnPN97pGdcxFxAR442mlHut7mZ2mwMgUz8yGytp33qHN3JvPhcnXk+e3qc92dmSPp9REumfX6CP/Xx8GXSNrU83Inph3penv372qmI/3+aDK3ve1l5vamH+52e1m7krs7M7e9Hu6vj3DJ3N50vT6OrDtfH58REZt6Qs3MSl07I9iFu0jMrczdIxIzQ2TmjsTMkbTGH0oPeB2gkyIxtzJ3j0jMDJGZO+Iya41fRCTGaI1fRCTGqPhFRGJM1BW/mT1kZrvMbEUn7ltsZsvNbJ2Z3W1m1uZ315vZGjNbaWa3h3tmM7vNzLaZ2ZLA5YxgZg5V7ja/v9XMnJl17mSjh15uKJ7rn5nZssDz/JqZ5UdA5jvMbHUg9wtm1iMCMl8Y+P/XamZB25nalayHeLwrzWxt4HJlm+mHfc13q84cfxrOF2AaMAFY0Yn7LgCmAAb8C/hSYPoXgX8DyYHbuRGQ+Tbg1kh7rgO/6wfMBjYDvcM9M5DVZp4bgPsjIPNpQELg+q+BX0dA5mOB4cAcoMTrrIEcRQdN6wlsCPzMCVzPOdzf5cUl6tb4nXNzgcq208xssJm9amaLzOwdMzvm4PuZWR7+/8DznP9f6VHg3MCvvwX8yjnXGFjGrgjIHHIhzH0n8H+BoB95EIrMzrmaNrOmBzt3iDK/5pxrDsw6HyiMgMwfOefWBDNnV7Iewv8BXnfOVTrn9gKvA6d7/X/1YFFX/IfwAHC9c64YuBW4r515CoCyNrfLAtMAhgEnmNkHZva2mU0MaVq/rmYG+E7go/xDZpYTuqif0aXcZnY2sM05tzTUQdvo8nNtZr8ws63AV4AfhTDrJ4Lx+vjENfjXQEMtmJlDrSNZ21MAbG1z+5P84fJ3ATFwsnUzywCmAq757boAAAUkSURBVH9rs0ktub1Z25n2yZpbAv6PbZOBicBfzWxQ4J076IKU+Y/AzwK3fwb8Fv9/8JDpam4zSwN+gH8zRLcI0nONc+4HwA/M7PvAd4AfBznqp0GClDnwWD8AmoEngpnxc0GCmDnUDpfVzK4GbgxMGwL808wOABudczM4dH7P/662or748X+qqXLOjWs70czigUWBmy/hL8q2H3cLge2B62XA84GiX2BmrfgHZqoI18zOuZ1t7vcn4JUQZW2rq7kHAwOBpYH/cIXAYjOb5JwrD9PMB3sS+AchLH6ClDmw4/FM4JRQrcS0EeznOZTazQrgnHsYeBjAzOYAVznnNrWZpQw4qc3tQvz7Asrw/u/6lFc7F0J5AYpos6MGeB+4MHDdgLGHuN9C/Gv1n+x8OSMw/ZvATwPXh+H/KGdhnjmvzTw3A09HwnN90DybCPLO3RA910PbzHM98GwEZD4dWAX4QvG6COVrgyDv3O1sVg69c3cj/i0EOYHrPTv6mu+uiycLDekfBE8BO4Am/O+yX8O/FvkqsDTwYv/RIe5bAqwA1gP38uk3m5OAxwO/WwycHAGZHwOWA8vwr0nlBTNzqHIfNM8mgn9UTyie6+cC05fhHxirIAIyr8O/ArMkcAn2kUihyDwj8FiNwE5gtpdZaaf4A9OvCTy/64Crj+Y1310XDdkgIhJjYuWoHhERCVDxi4jEGBW/iEiMUfGLiMQYFb+ISIxR8UtEMrO6bl7en81sRJAeq8X8I3muMLOXjzQyppn1MLNvB2PZIqAzcEmEMrM651xGEB8vwX06aFlItc1uZo8AHzvnfnGY+YuAV5xzo7ojn0Q/rfFL1DAzn5k9Z2YLA5fjA9Mnmdn7ZvZh4OfwwPSrzOxvZvYy8JqZnWRmc8zsWfOPVf/EJ2OmB6aXBK7XBQZlW2pm882sT2D64MDthWb20w5+KpnHpwPUZZjZG2a22Pzjtp8TmOdXwODAp4Q7AvP+V2A5y8zsJ0F8GiUGqPglmtwF3OmcmwicD/w5MH01MM05Nx7/yJn/2+Y+U4ArnXMnB26PB24CRgCDgOPbWU46MN85NxaYC1zbZvl3BZZ/xHFYAuPUnIL/m9UADcAM59wE/OeA+G3gjed7wHrn3Djn3H+Z2WnAUGASMA4oNrNpR1qeyCdiYZA2iR3TgRFtRlTMMrNMIBt4xMyG4h8RMbHNfV53zrUdi32Bc64MwMyW4B/D5d2DlnOATwe9WwScGrg+hU/HWH8S+M0hcqa2eexF+MdsB/8YLv8bKPFW/J8E+rRz/9MClw8DtzPwvxHMPcTyRD5DxS/RJA6Y4pzb33aimd0DvOWcmxHYXj6nza/3HfQYjW2ut9D+/5Em9+nOsUPNczj7nXPjzCwb/xvIdcDd+Mfy9wHFzrkmM9sEpLRzfwN+6ZybdZTLFQG0qUeiy2v4x8IHwMw+GVY3G9gWuH5VCJc/H/8mJoBLjjSzc64a/6kabzWzRPw5dwVK/4vAgMCstUBmm7vOBq4JjBuPmRWYWW6Q/gaJASp+iVRpZlbW5nIL/hItCezwXIV/OG2A24Ffmtl7QHwIM90E3GJmC4A8oPpId3DOfYh/BMhL8J8MpcTMSvGv/a8OzLMHeC9w+OcdzrnX8G9Kmmdmy4Fn+ewbg8hh6XBOkSAJnEFsv3POmdklwKXOuXOOdD+R7qZt/CLBUwzcGzgSp4oQn+pSpLO0xi8iEmO0jV9EJMao+EVEYoyKX0Qkxqj4RURijIpfRCTG/H9Plgr7n9LvaQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47min 44s, sys: 2min 49s, total: 50min 33s\n",
      "Wall time: 44min 32s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for i in range(3):\n",
    "    learn.fit_one_cycle(5, 1e-3, wd=0.2)\n",
    "    create_submission_filename(learn, f'submission_custom_stage_{i+1}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# learn.fit_one_cycle(10, 1e-3, wd=0.2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}