ml_25m_deepML.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML_25M Deep ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "import json\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.decomposition import PCA\n",
    "import itertools\n",
    "from sklearn.metrics import r2_score, mean_squared_error\n",
    "\n",
    "def fix_random(seed: int) -> None:\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    torch.backends.cudnn.deterministic = True  # slower\n",
    "fix_random(42)\n",
    "\n",
    "if not os.path.exists('results'):\n",
    "    os.mkdir('results')\n",
    "\n",
    "if not os.path.exists('results/pca'):\n",
    "    os.mkdir('results/pca')\n",
    "\n",
    "if not os.path.exists('results/no_pca'):\n",
    "    os.mkdir('results/no_pca')\n",
    "\n",
    "pca = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: mps\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('dataset.csv')\n",
    "\n",
    "# PyTorch Device\n",
    "if torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "elif torch.backends.mps.is_available():\n",
    "    device = torch.device('mps')\n",
    "else:\n",
    "    device = torch.device('cpu')\n",
    "\n",
    "print('Using device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "delete_previous = True\n",
    "\n",
    "if delete_previous:\n",
    "    try:\n",
    "        os.system('rm -rf deepML')\n",
    "        os.system('rm -rf models')\n",
    "        os.system('rm -rf best_model.pth')\n",
    "        os.system('rm -rf best_model_config.json')\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop('rating', axis=1)\n",
    "Y = df['rating']\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)\n",
    "X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size=0.1, random_state=42)\n",
    "\n",
    "X_train = X_train.to_numpy()\n",
    "X_val = X_val.to_numpy()\n",
    "X_test = X_test.to_numpy()\n",
    "Y_train = Y_train.to_numpy()\n",
    "Y_val = Y_val.to_numpy()\n",
    "Y_test = Y_test.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=0.95)\n",
    "pca.fit(X_train)\n",
    "X_train = pca.transform(X_train)\n",
    "X_val = pca.transform(X_val)\n",
    "X_test = pca.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training samples: 9946\n",
      "Number of validation samples: 1106\n",
      "Number of testing samples: 2764\n",
      "\n",
      "Number of features: 552\n"
     ]
    }
   ],
   "source": [
    "print(f'Number of training samples: {X_train.shape[0]}')\n",
    "print(f'Number of validation samples: {X_val.shape[0]}')\n",
    "print(f'Number of testing samples: {X_test.shape[0]}')\n",
    "print(f'\\nNumber of features: {X_train.shape[1]}')\n",
    "\n",
    "val_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(torch.tensor(X_val, dtype=torch.float32), torch.tensor(Y_val, dtype=torch.float32)), batch_size=Y_val.shape[0], shuffle=False)\n",
    "test_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.float32), torch.tensor(Y_test, dtype=torch.float32)), batch_size=Y_test.shape[0], shuffle=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(input_size, hidden_size, dropout_prob=0, depth=1):\n",
    "    model = [\n",
    "        torch.nn.Linear(input_size, hidden_size),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.Dropout(dropout_prob)\n",
    "    ]\n",
    "\n",
    "    for i in range(depth):\n",
    "        model.append(torch.nn.Linear(hidden_size, hidden_size))\n",
    "        model.append(torch.nn.ReLU())\n",
    "        model.append(torch.nn.Dropout(dropout_prob))\n",
    "\n",
    "    model.append(torch.nn.Linear(hidden_size, 1))\n",
    "    return torch.nn.Sequential(*model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of hyperparameter combinations: 108\n"
     ]
    }
   ],
   "source": [
    "# Hyperparameters\n",
    "hidden_sizes =  [256, 512, 1024]\n",
    "nums_epochs = [200]\n",
    "depth = [3, 4, 5]\n",
    "batch_sizes = [8, 16, 32]\n",
    "learning_rate = [0.01, 0.001]\n",
    "step_size_lr_decay = [10, 20]\n",
    "momentum = [0.9]\n",
    "dropout_prob = 0.2\n",
    "patience = 10\n",
    "\n",
    "hyperparameters = itertools.product(hidden_sizes, depth, nums_epochs, batch_sizes, learning_rate, step_size_lr_decay, momentum)\n",
    "n_comb = len(hidden_sizes)*len(depth)*len(nums_epochs)*len(batch_sizes)*len(learning_rate)*len(step_size_lr_decay)*len(momentum)\n",
    "print (f'Number of hyperparameter combinations: {n_comb}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model(model, criterion, loader):\n",
    "    model.eval()\n",
    "    y_pred = torch.tensor([]).to(device)\n",
    "    y_true = torch.tensor([]).to(device)\n",
    "\n",
    "    total_loss = 0.0\n",
    "    for data in loader:\n",
    "        data, targets = data[0].to(device), data[1].to(device)\n",
    "        pred = model(data)\n",
    "        loss = criterion(pred.squeeze(), targets)\n",
    "        total_loss += loss.item()\n",
    "        y_pred = torch.cat((y_pred, pred.squeeze()))\n",
    "        y_true = torch.cat((y_true, targets.detach()))\n",
    "\n",
    "    return total_loss / len(loader), y_pred.squeeze(), y_true.squeeze()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, criterion, optimizer, scheduler, epochs, data_loader, val_loader, device, writer, log_name):\n",
    "    n_iter = 0\n",
    "\n",
    "    best_model = None\n",
    "    best_val_loss = float('inf')\n",
    "    epochs_since_last_improvement = 0\n",
    "\n",
    "    start = time.time()\n",
    "\n",
    "    loss_history = []\n",
    "    val_loss_history = []\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        model.train()\n",
    "\n",
    "        start_epoch = time.time()\n",
    "\n",
    "        loss_train = 0\n",
    "        for data, targets in data_loader:\n",
    "            data, targets = data.to(device), targets.to(device)\n",
    "            y_pred = model(data)\n",
    "            loss = criterion(y_pred.squeeze(), targets)\n",
    "            optimizer.zero_grad()\n",
    "            writer.add_scalar(\"Loss/train_iter\", loss, n_iter)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            n_iter += 1\n",
    "            loss_train += loss.item()\n",
    "\n",
    "        scheduler.step()\n",
    "        loss_train /= len(data_loader)\n",
    "        writer.add_scalar(\"Loss/train\", loss_train, epoch)\n",
    "\n",
    "        # Compute Val Loss\n",
    "        val_loss, _ , _ = test_model(model, criterion, val_loader)\n",
    "\n",
    "        writer.add_scalar(\"Loss/val\", val_loss, epoch)\n",
    "\n",
    "        loss_history.append(loss_train)\n",
    "        val_loss_history.append(val_loss)\n",
    "\n",
    "        # Early stopping\n",
    "        if val_loss < best_val_loss:\n",
    "            best_val_loss = val_loss\n",
    "            best_model = copy.deepcopy(model)\n",
    "            epochs_since_last_improvement = 0\n",
    "        elif epochs_since_last_improvement >= patience:\n",
    "            break\n",
    "        else:\n",
    "            epochs_since_last_improvement += 1\n",
    "\n",
    "        print('Epoch [{}/{}] - {:.2f} seconds - val_loss: {:.6f} - patience: {}'.format(epoch+1,\n",
    "              epochs, time.time() - start_epoch, val_loss, epochs_since_last_improvement), end='\\r')\n",
    "\n",
    "    print('\\nTraining ended after {:.2f} seconds - Best val_loss: {:.6f}'.format(time.time() - start, best_val_loss))\n",
    "\n",
    "    return best_model, loss_history, val_loss_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Iterations 1/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [29/200] - 1.42 seconds - val_loss: 0.005351 - patience: 10\n",
      "Training ended after 52.32 seconds - Best val_loss: 0.005342\n",
      "Model MSE: 0.005263 - Best MSE: 0.005263\n",
      "\n",
      "Iterations 2/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [48/200] - 1.50 seconds - val_loss: 0.005386 - patience: 10\n",
      "Training ended after 82.14 seconds - Best val_loss: 0.005319\n",
      "Model MSE: 0.005174 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 3/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [45/200] - 1.40 seconds - val_loss: 0.007258 - patience: 10\n",
      "Training ended after 67.43 seconds - Best val_loss: 0.007256\n",
      "Model MSE: 0.007207 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 4/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [65/200] - 1.40 seconds - val_loss: 0.006170 - patience: 10\n",
      "Training ended after 92.50 seconds - Best val_loss: 0.006167\n",
      "Model MSE: 0.005946 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 5/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [34/200] - 0.74 seconds - val_loss: 0.005575 - patience: 10\n",
      "Training ended after 26.83 seconds - Best val_loss: 0.005568\n",
      "Model MSE: 0.005419 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 6/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [46/200] - 0.78 seconds - val_loss: 0.005459 - patience: 10\n",
      "Training ended after 35.64 seconds - Best val_loss: 0.005356\n",
      "Model MSE: 0.005291 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 7/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [44/200] - 0.78 seconds - val_loss: 0.009242 - patience: 10\n",
      "Training ended after 34.86 seconds - Best val_loss: 0.009236\n",
      "Model MSE: 0.008768 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 8/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [130/200] - 0.77 seconds - val_loss: 0.007355 - patience: 10\n",
      "Training ended after 97.26 seconds - Best val_loss: 0.007355\n",
      "Model MSE: 0.007258 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 9/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [29/200] - 0.45 seconds - val_loss: 0.005799 - patience: 10\n",
      "Training ended after 12.92 seconds - Best val_loss: 0.005733\n",
      "Model MSE: 0.005622 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 10/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [44/200] - 0.42 seconds - val_loss: 0.005399 - patience: 10\n",
      "Training ended after 19.28 seconds - Best val_loss: 0.005360\n",
      "Model MSE: 0.005446 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 11/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [70/200] - 0.44 seconds - val_loss: 0.012184 - patience: 10\n",
      "Training ended after 30.86 seconds - Best val_loss: 0.012184\n",
      "Model MSE: 0.011428 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 12/108 - Training with hidden_size=256, depth=3, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [50/200] - 0.43 seconds - val_loss: 0.008994 - patience: 10\n",
      "Training ended after 22.06 seconds - Best val_loss: 0.008976\n",
      "Model MSE: 0.008646 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 13/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [25/200] - 1.66 seconds - val_loss: 0.005681 - patience: 10\n",
      "Training ended after 43.28 seconds - Best val_loss: 0.005607\n",
      "Model MSE: 0.005684 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 14/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [40/200] - 1.66 seconds - val_loss: 0.005445 - patience: 10\n",
      "Training ended after 67.96 seconds - Best val_loss: 0.005332\n",
      "Model MSE: 0.005275 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 15/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [70/200] - 1.69 seconds - val_loss: 0.008002 - patience: 10\n",
      "Training ended after 121.54 seconds - Best val_loss: 0.008002\n",
      "Model MSE: 0.007830 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 16/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [67/200] - 1.61 seconds - val_loss: 0.006197 - patience: 10\n",
      "Training ended after 111.87 seconds - Best val_loss: 0.006195\n",
      "Model MSE: 0.006124 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 17/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [39/200] - 0.88 seconds - val_loss: 0.005465 - patience: 10\n",
      "Training ended after 35.61 seconds - Best val_loss: 0.005420\n",
      "Model MSE: 0.005536 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 18/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [31/200] - 0.90 seconds - val_loss: 0.005314 - patience: 10\n",
      "Training ended after 28.41 seconds - Best val_loss: 0.005272\n",
      "Model MSE: 0.005308 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 19/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [48/200] - 0.90 seconds - val_loss: 0.009694 - patience: 10\n",
      "Training ended after 43.35 seconds - Best val_loss: 0.009693\n",
      "Model MSE: 0.009588 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 20/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [67/200] - 0.90 seconds - val_loss: 0.007712 - patience: 10\n",
      "Training ended after 60.16 seconds - Best val_loss: 0.007699\n",
      "Model MSE: 0.007614 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 21/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [35/200] - 0.52 seconds - val_loss: 0.006019 - patience: 10\n",
      "Training ended after 18.47 seconds - Best val_loss: 0.006009\n",
      "Model MSE: 0.005909 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 22/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [40/200] - 0.50 seconds - val_loss: 0.005455 - patience: 10\n",
      "Training ended after 20.57 seconds - Best val_loss: 0.005439\n",
      "Model MSE: 0.005592 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 23/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [74/200] - 0.53 seconds - val_loss: 0.014317 - patience: 10\n",
      "Training ended after 38.11 seconds - Best val_loss: 0.014317\n",
      "Model MSE: 0.013376 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 24/108 - Training with hidden_size=256, depth=4, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [134/200] - 0.52 seconds - val_loss: 0.010289 - patience: 10\n",
      "Training ended after 69.60 seconds - Best val_loss: 0.010289\n",
      "Model MSE: 0.010056 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 25/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [21/200] - 1.89 seconds - val_loss: 0.007235 - patience: 10\n",
      "Training ended after 41.54 seconds - Best val_loss: 0.006286\n",
      "Model MSE: 0.006511 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 26/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [33/200] - 1.85 seconds - val_loss: 0.005947 - patience: 10\n",
      "Training ended after 63.57 seconds - Best val_loss: 0.005507\n",
      "Model MSE: 0.005581 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 27/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [32/200] - 1.93 seconds - val_loss: 0.008548 - patience: 10\n",
      "Training ended after 63.49 seconds - Best val_loss: 0.008526\n",
      "Model MSE: 0.008340 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 28/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [81/200] - 1.80 seconds - val_loss: 0.006375 - patience: 10\n",
      "Training ended after 150.95 seconds - Best val_loss: 0.006375\n",
      "Model MSE: 0.006163 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 29/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [30/200] - 1.00 seconds - val_loss: 0.005582 - patience: 10\n",
      "Training ended after 30.60 seconds - Best val_loss: 0.005489\n",
      "Model MSE: 0.005584 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 30/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [49/200] - 0.98 seconds - val_loss: 0.005426 - patience: 10\n",
      "Training ended after 50.36 seconds - Best val_loss: 0.005352\n",
      "Model MSE: 0.005433 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 31/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [71/200] - 1.02 seconds - val_loss: 0.012121 - patience: 10\n",
      "Training ended after 71.53 seconds - Best val_loss: 0.012121\n",
      "Model MSE: 0.011323 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 32/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [69/200] - 0.99 seconds - val_loss: 0.008470 - patience: 10\n",
      "Training ended after 71.22 seconds - Best val_loss: 0.008469\n",
      "Model MSE: 0.008499 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 33/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [28/200] - 0.55 seconds - val_loss: 0.006357 - patience: 10\n",
      "Training ended after 16.58 seconds - Best val_loss: 0.006354\n",
      "Model MSE: 0.006563 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 34/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [47/200] - 0.58 seconds - val_loss: 0.005666 - patience: 10\n",
      "Training ended after 27.38 seconds - Best val_loss: 0.005620\n",
      "Model MSE: 0.005688 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 35/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [46/200] - 0.58 seconds - val_loss: 0.017372 - patience: 10\n",
      "Training ended after 26.99 seconds - Best val_loss: 0.017370\n",
      "Model MSE: 0.016854 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 36/108 - Training with hidden_size=256, depth=5, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [130/200] - 0.54 seconds - val_loss: 0.010695 - patience: 10\n",
      "Training ended after 75.85 seconds - Best val_loss: 0.010695\n",
      "Model MSE: 0.010580 - Best MSE: 0.005174\n",
      "\n",
      "Iterations 37/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [40/200] - 4.04 seconds - val_loss: 0.004975 - patience: 10\n",
      "Training ended after 157.90 seconds - Best val_loss: 0.004952\n",
      "Model MSE: 0.005073 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 38/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [20/200] - 3.17 seconds - val_loss: 0.005206 - patience: 10\n",
      "Training ended after 67.36 seconds - Best val_loss: 0.005187\n",
      "Model MSE: 0.005474 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 39/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [29/200] - 3.16 seconds - val_loss: 0.006922 - patience: 10\n",
      "Training ended after 93.99 seconds - Best val_loss: 0.006890\n",
      "Model MSE: 0.006801 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 40/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [50/200] - 3.13 seconds - val_loss: 0.005792 - patience: 10\n",
      "Training ended after 158.44 seconds - Best val_loss: 0.005766\n",
      "Model MSE: 0.005926 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 41/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [37/200] - 1.69 seconds - val_loss: 0.005375 - patience: 10\n",
      "Training ended after 63.77 seconds - Best val_loss: 0.005352\n",
      "Model MSE: 0.005302 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 42/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [37/200] - 1.72 seconds - val_loss: 0.005270 - patience: 10\n",
      "Training ended after 64.29 seconds - Best val_loss: 0.005268\n",
      "Model MSE: 0.005092 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 43/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [49/200] - 1.66 seconds - val_loss: 0.008222 - patience: 10\n",
      "Training ended after 83.15 seconds - Best val_loss: 0.008220\n",
      "Model MSE: 0.008080 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 44/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [49/200] - 1.65 seconds - val_loss: 0.006929 - patience: 10\n",
      "Training ended after 84.14 seconds - Best val_loss: 0.006924\n",
      "Model MSE: 0.006825 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 45/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [38/200] - 0.91 seconds - val_loss: 0.005799 - patience: 10\n",
      "Training ended after 36.60 seconds - Best val_loss: 0.005793\n",
      "Model MSE: 0.005683 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 46/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [46/200] - 0.94 seconds - val_loss: 0.005111 - patience: 10\n",
      "Training ended after 44.27 seconds - Best val_loss: 0.005098\n",
      "Model MSE: 0.005167 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 47/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [69/200] - 0.96 seconds - val_loss: 0.010933 - patience: 10\n",
      "Training ended after 66.24 seconds - Best val_loss: 0.010933\n",
      "Model MSE: 0.010587 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 48/108 - Training with hidden_size=512, depth=3, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [88/200] - 0.97 seconds - val_loss: 0.008316 - patience: 10\n",
      "Training ended after 83.54 seconds - Best val_loss: 0.008315\n",
      "Model MSE: 0.008045 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 49/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [36/200] - 3.90 seconds - val_loss: 0.005107 - patience: 10\n",
      "Training ended after 144.85 seconds - Best val_loss: 0.005097\n",
      "Model MSE: 0.005107 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 50/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [44/200] - 3.75 seconds - val_loss: 0.004978 - patience: 10\n",
      "Training ended after 172.38 seconds - Best val_loss: 0.004933\n",
      "Model MSE: 0.005087 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 51/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [36/200] - 3.60 seconds - val_loss: 0.007203 - patience: 10\n",
      "Training ended after 140.02 seconds - Best val_loss: 0.007188\n",
      "Model MSE: 0.006999 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 52/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [47/200] - 4.04 seconds - val_loss: 0.006213 - patience: 10\n",
      "Training ended after 184.41 seconds - Best val_loss: 0.006176\n",
      "Model MSE: 0.005969 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 53/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [34/200] - 2.06 seconds - val_loss: 0.005340 - patience: 10\n",
      "Training ended after 71.94 seconds - Best val_loss: 0.005334\n",
      "Model MSE: 0.005341 - Best MSE: 0.005073\n",
      "\n",
      "Iterations 54/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [49/200] - 2.04 seconds - val_loss: 0.005286 - patience: 10\n",
      "Training ended after 101.37 seconds - Best val_loss: 0.005253\n",
      "Model MSE: 0.005037 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 55/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [39/200] - 2.07 seconds - val_loss: 0.009243 - patience: 10\n",
      "Training ended after 79.76 seconds - Best val_loss: 0.009217\n",
      "Model MSE: 0.008884 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 56/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [64/200] - 2.02 seconds - val_loss: 0.007176 - patience: 10\n",
      "Training ended after 134.12 seconds - Best val_loss: 0.007145\n",
      "Model MSE: 0.007147 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 57/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [45/200] - 1.07 seconds - val_loss: 0.005647 - patience: 10\n",
      "Training ended after 51.92 seconds - Best val_loss: 0.005646\n",
      "Model MSE: 0.005756 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 58/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [42/200] - 1.06 seconds - val_loss: 0.005408 - patience: 10\n",
      "Training ended after 47.58 seconds - Best val_loss: 0.005378\n",
      "Model MSE: 0.005341 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 59/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [42/200] - 1.08 seconds - val_loss: 0.011963 - patience: 10\n",
      "Training ended after 48.38 seconds - Best val_loss: 0.011960\n",
      "Model MSE: 0.011789 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 60/108 - Training with hidden_size=512, depth=4, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [66/200] - 1.07 seconds - val_loss: 0.008983 - patience: 10\n",
      "Training ended after 73.42 seconds - Best val_loss: 0.008966\n",
      "Model MSE: 0.008893 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 61/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [26/200] - 4.30 seconds - val_loss: 0.005227 - patience: 10\n",
      "Training ended after 116.06 seconds - Best val_loss: 0.005216\n",
      "Model MSE: 0.005360 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 62/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [50/200] - 4.13 seconds - val_loss: 0.005305 - patience: 10\n",
      "Training ended after 223.96 seconds - Best val_loss: 0.005238\n",
      "Model MSE: 0.005129 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 63/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [40/200] - 4.19 seconds - val_loss: 0.007632 - patience: 10\n",
      "Training ended after 181.42 seconds - Best val_loss: 0.007626\n",
      "Model MSE: 0.007378 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 64/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [52/200] - 4.22 seconds - val_loss: 0.006476 - patience: 10\n",
      "Training ended after 235.72 seconds - Best val_loss: 0.006453\n",
      "Model MSE: 0.006333 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 65/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [33/200] - 2.36 seconds - val_loss: 0.005309 - patience: 10\n",
      "Training ended after 79.58 seconds - Best val_loss: 0.005290\n",
      "Model MSE: 0.005366 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 66/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [59/200] - 2.27 seconds - val_loss: 0.005302 - patience: 10\n",
      "Training ended after 143.37 seconds - Best val_loss: 0.005288\n",
      "Model MSE: 0.005279 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 67/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [31/200] - 2.33 seconds - val_loss: 0.009964 - patience: 10\n",
      "Training ended after 77.59 seconds - Best val_loss: 0.009959\n",
      "Model MSE: 0.009898 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 68/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [62/200] - 2.44 seconds - val_loss: 0.007910 - patience: 10\n",
      "Training ended after 150.33 seconds - Best val_loss: 0.007885\n",
      "Model MSE: 0.007518 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 69/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [34/200] - 1.34 seconds - val_loss: 0.005917 - patience: 10\n",
      "Training ended after 47.79 seconds - Best val_loss: 0.005910\n",
      "Model MSE: 0.006001 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 70/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [49/200] - 1.32 seconds - val_loss: 0.005361 - patience: 10\n",
      "Training ended after 66.32 seconds - Best val_loss: 0.005328\n",
      "Model MSE: 0.005264 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 71/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [65/200] - 1.35 seconds - val_loss: 0.015182 - patience: 10\n",
      "Training ended after 87.67 seconds - Best val_loss: 0.015182\n",
      "Model MSE: 0.014168 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 72/108 - Training with hidden_size=512, depth=5, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [47/200] - 1.30 seconds - val_loss: 0.010105 - patience: 10\n",
      "Training ended after 63.04 seconds - Best val_loss: 0.010077\n",
      "Model MSE: 0.009964 - Best MSE: 0.005037\n",
      "\n",
      "Iterations 73/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [28/200] - 10.53 seconds - val_loss: 0.004973 - patience: 10\n",
      "Training ended after 286.52 seconds - Best val_loss: 0.004953\n",
      "Model MSE: 0.004978 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 74/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [17/200] - 9.50 seconds - val_loss: 0.008616 - patience: 10\n",
      "Training ended after 169.31 seconds - Best val_loss: 0.005324\n",
      "Model MSE: 0.005376 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 75/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [40/200] - 9.46 seconds - val_loss: 0.006461 - patience: 10\n",
      "Training ended after 397.44 seconds - Best val_loss: 0.006451\n",
      "Model MSE: 0.006486 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 76/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [47/200] - 9.59 seconds - val_loss: 0.005845 - patience: 10\n",
      "Training ended after 487.81 seconds - Best val_loss: 0.005822\n",
      "Model MSE: 0.005653 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 77/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [36/200] - 5.05 seconds - val_loss: 0.005177 - patience: 10\n",
      "Training ended after 191.59 seconds - Best val_loss: 0.005167\n",
      "Model MSE: 0.005204 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 78/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [45/200] - 5.11 seconds - val_loss: 0.005093 - patience: 10\n",
      "Training ended after 234.90 seconds - Best val_loss: 0.005083\n",
      "Model MSE: 0.005041 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 79/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [49/200] - 5.11 seconds - val_loss: 0.007745 - patience: 10\n",
      "Training ended after 273.16 seconds - Best val_loss: 0.007744\n",
      "Model MSE: 0.007613 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 80/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [49/200] - 5.31 seconds - val_loss: 0.006518 - patience: 10\n",
      "Training ended after 253.52 seconds - Best val_loss: 0.006467\n",
      "Model MSE: 0.006563 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 81/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [41/200] - 3.05 seconds - val_loss: 0.005531 - patience: 10\n",
      "Training ended after 116.23 seconds - Best val_loss: 0.005523\n",
      "Model MSE: 0.005622 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 82/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [47/200] - 2.70 seconds - val_loss: 0.005036 - patience: 10\n",
      "Training ended after 121.97 seconds - Best val_loss: 0.005025\n",
      "Model MSE: 0.005073 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 83/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [72/200] - 2.46 seconds - val_loss: 0.010824 - patience: 10\n",
      "Training ended after 188.95 seconds - Best val_loss: 0.010824\n",
      "Model MSE: 0.010544 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 84/108 - Training with hidden_size=1024, depth=3, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [61/200] - 2.40 seconds - val_loss: 0.007762 - patience: 10\n",
      "Training ended after 156.20 seconds - Best val_loss: 0.007749\n",
      "Model MSE: 0.007566 - Best MSE: 0.004978\n",
      "\n",
      "Iterations 85/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [34/200] - 13.24 seconds - val_loss: 0.005060 - patience: 10\n",
      "Training ended after 452.41 seconds - Best val_loss: 0.005048\n",
      "Model MSE: 0.004943 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 86/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [34/200] - 13.81 seconds - val_loss: 0.004983 - patience: 10\n",
      "Training ended after 480.48 seconds - Best val_loss: 0.004938\n",
      "Model MSE: 0.005050 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 87/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [30/200] - 13.44 seconds - val_loss: 0.007332 - patience: 10\n",
      "Training ended after 409.66 seconds - Best val_loss: 0.007276\n",
      "Model MSE: 0.006860 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 88/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [30/200] - 12.89 seconds - val_loss: 0.005926 - patience: 10\n",
      "Training ended after 395.45 seconds - Best val_loss: 0.005883\n",
      "Model MSE: 0.006032 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 89/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [38/200] - 7.35 seconds - val_loss: 0.005060 - patience: 10\n",
      "Training ended after 274.17 seconds - Best val_loss: 0.005054\n",
      "Model MSE: 0.005263 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 90/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [51/200] - 6.65 seconds - val_loss: 0.004929 - patience: 10\n",
      "Training ended after 350.54 seconds - Best val_loss: 0.004926\n",
      "Model MSE: 0.005009 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 91/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [39/200] - 6.83 seconds - val_loss: 0.008390 - patience: 10\n",
      "Training ended after 255.39 seconds - Best val_loss: 0.008359\n",
      "Model MSE: 0.008331 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 92/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [30/200] - 6.41 seconds - val_loss: 0.007237 - patience: 10\n",
      "Training ended after 195.70 seconds - Best val_loss: 0.007159\n",
      "Model MSE: 0.006917 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 93/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [69/200] - 3.55 seconds - val_loss: 0.005658 - patience: 10\n",
      "Training ended after 233.17 seconds - Best val_loss: 0.005658\n",
      "Model MSE: 0.005647 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 94/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [43/200] - 3.21 seconds - val_loss: 0.005180 - patience: 10\n",
      "Training ended after 142.00 seconds - Best val_loss: 0.005150\n",
      "Model MSE: 0.005356 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 95/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [56/200] - 3.07 seconds - val_loss: 0.011586 - patience: 10\n",
      "Training ended after 180.50 seconds - Best val_loss: 0.011586\n",
      "Model MSE: 0.011011 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 96/108 - Training with hidden_size=1024, depth=4, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [58/200] - 3.29 seconds - val_loss: 0.008352 - patience: 10\n",
      "Training ended after 192.01 seconds - Best val_loss: 0.008322\n",
      "Model MSE: 0.008078 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 97/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=8, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [37/200] - 16.18 seconds - val_loss: 0.005000 - patience: 10\n",
      "Training ended after 614.00 seconds - Best val_loss: 0.004992\n",
      "Model MSE: 0.005030 - Best MSE: 0.004943\n",
      "\n",
      "Iterations 98/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=8, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [45/200] - 16.07 seconds - val_loss: 0.004880 - patience: 10\n",
      "Training ended after 688.66 seconds - Best val_loss: 0.004846\n",
      "Model MSE: 0.004927 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 99/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=8, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [39/200] - 15.34 seconds - val_loss: 0.007354 - patience: 10\n",
      "Training ended after 649.96 seconds - Best val_loss: 0.007328\n",
      "Model MSE: 0.007153 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 100/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=8, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [46/200] - 16.45 seconds - val_loss: 0.005941 - patience: 10\n",
      "Training ended after 760.85 seconds - Best val_loss: 0.005921\n",
      "Model MSE: 0.006064 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 101/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=16, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [38/200] - 8.20 seconds - val_loss: 0.005149 - patience: 10\n",
      "Training ended after 316.41 seconds - Best val_loss: 0.005142\n",
      "Model MSE: 0.005251 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 102/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=16, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [43/200] - 8.91 seconds - val_loss: 0.004885 - patience: 10\n",
      "Training ended after 384.27 seconds - Best val_loss: 0.004880\n",
      "Model MSE: 0.005090 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 103/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=16, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [30/200] - 8.62 seconds - val_loss: 0.008767 - patience: 10\n",
      "Training ended after 260.28 seconds - Best val_loss: 0.008681\n",
      "Model MSE: 0.008565 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 104/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=16, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [30/200] - 7.59 seconds - val_loss: 0.007507 - patience: 10\n",
      "Training ended after 233.05 seconds - Best val_loss: 0.007175\n",
      "Model MSE: 0.007303 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 105/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=32, lr=0.01, step_size=10, momentum=0.9\n",
      "Epoch [30/200] - 4.01 seconds - val_loss: 0.005651 - patience: 10\n",
      "Training ended after 121.45 seconds - Best val_loss: 0.005644\n",
      "Model MSE: 0.005864 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 106/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=32, lr=0.01, step_size=20, momentum=0.9\n",
      "Epoch [50/200] - 3.55 seconds - val_loss: 0.005273 - patience: 10\n",
      "Training ended after 185.83 seconds - Best val_loss: 0.005249\n",
      "Model MSE: 0.005171 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 107/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=32, lr=0.001, step_size=10, momentum=0.9\n",
      "Epoch [40/200] - 3.82 seconds - val_loss: 0.013024 - patience: 10\n",
      "Training ended after 159.72 seconds - Best val_loss: 0.013021\n",
      "Model MSE: 0.012677 - Best MSE: 0.004927\n",
      "\n",
      "Iterations 108/108 - Training with hidden_size=1024, depth=5, num_epochs=200, batch=32, lr=0.001, step_size=20, momentum=0.9\n",
      "Epoch [46/200] - 4.07 seconds - val_loss: 0.009679 - patience: 10\n",
      "Training ended after 183.77 seconds - Best val_loss: 0.009655\n",
      "Model MSE: 0.008871 - Best MSE: 0.004927\n"
     ]
    }
   ],
   "source": [
    "current_iter = 0\n",
    "\n",
    "best_model = None\n",
    "best_mse = float('inf')\n",
    "criterion = torch.nn.MSELoss()\n",
    "history_loss = []\n",
    "history_val_loss = []\n",
    "\n",
    "if os.path.exists('best_model.pth'):\n",
    "    # read best model config\n",
    "    with open('best_model_config.json', 'r') as f:\n",
    "        best_model_config = json.load(f)\n",
    "\n",
    "    # load best model\n",
    "    best_model = get_model(X_train.shape[1], best_model_config['hidden_size'], dropout_prob, best_model_config['depth'])\n",
    "    best_model.load_state_dict(torch.load('best_model.pth'))\n",
    "    best_model.to(device)\n",
    "    best_model.eval()\n",
    "\n",
    "    # evaluate best model\n",
    "    best_mse, _, _ = test_model(best_model, criterion, test_loader)\n",
    "    \n",
    "    print(\"Best model - MSE: {:.6f}\".format(best_mse))\n",
    "\n",
    "if not os.path.exists('models'):\n",
    "    os.makedirs('models')\n",
    "\n",
    "for hidden_size, depth, num_epochs, batch, lr, step_size, momentum in hyperparameters:\n",
    "    current_iter += 1\n",
    "\n",
    "    print(\"\\nIterations {}/{} - Training with hidden_size={}, depth={}, num_epochs={}, batch={}, lr={}, step_size={}, momentum={}\".format(\n",
    "        current_iter, n_comb, hidden_size, depth, num_epochs, batch, lr, step_size, momentum))\n",
    "    \n",
    "    log_name = \"dim:\"+str(hidden_size)+\"_depth:\"+str(depth)+\"_batch:\" + \\\n",
    "        str(batch)+\"_lr:\"+str(lr)+\"_step_size:\" + \\\n",
    "        str(step_size)\n",
    "\n",
    "    if os.path.exists('deepML/'+log_name):\n",
    "        print(\"Model already trained, skipping...\")\n",
    "        continue\n",
    "\n",
    "    writer = SummaryWriter('deepML/'+log_name)\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(torch.tensor(X_train, dtype=torch.float32), torch.tensor(Y_train, dtype=torch.float32)), batch_size=batch, shuffle=True)\n",
    "\n",
    "    model = get_model(X_train.shape[1], hidden_size, dropout_prob, depth=depth)\n",
    "    model = model.to(device)\n",
    "\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum)\n",
    "    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=0.1)\n",
    "\n",
    "    model, hl, hvl = train_model(model, criterion, optimizer, scheduler, num_epochs, train_loader, val_loader, device, writer, log_name)\n",
    "\n",
    "    # validate model on test set\n",
    "    mse, _, _ = test_model(model, criterion, test_loader)\n",
    "    writer.add_hparams({'hidden_size': hidden_size, 'depth': depth, 'batch': batch,'lr': lr, 'step_size': step_size, 'momentum': momentum}, {'hparam/mse': mse})\n",
    "\n",
    "    if mse < best_mse:\n",
    "        best_mse = mse\n",
    "        best_model = model\n",
    "        \n",
    "        history_loss = hl\n",
    "        history_val_loss = hvl\n",
    "\n",
    "        torch.save(model.state_dict(), 'best_model.pth')\n",
    "        # save config\n",
    "        with open('best_model_config.json', 'w') as f:\n",
    "            json.dump({'name':log_name,'hidden_size': hidden_size, 'depth': depth, 'num_epochs': num_epochs, 'batch': batch,\n",
    "                       'lr': lr, 'step_size': step_size}, f)\n",
    "            \n",
    "    print(\"Model MSE: {:.6f} - Best MSE: {:.6f}\".format(mse, best_mse))\n",
    "    torch.save(model.state_dict(), 'models/'+log_name+'.pth')\n",
    "    writer.flush()\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network Evalutation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model.eval()\n",
    "_, y_pred, y_true = test_model(best_model, criterion, test_loader)\n",
    "y_pred = y_pred.squeeze().cpu().detach().numpy()\n",
    "y_true = y_true.squeeze().cpu().detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2: 0.977773 - MSE: 0.004927\n"
     ]
    }
   ],
   "source": [
    "r2 = r2_score(y_true, y_pred)\n",
    "mse = mean_squared_error(y_true, y_pred)\n",
    "print(\"R2: {:.6f} - MSE: {:.6f}\".format(r2, mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame({'loss': [h for h in history_loss], 'val_loss': [h for h in history_val_loss]})\n",
    "\n",
    "if pca is not None:\n",
    "    results.to_csv('results/pca/deepML.csv', index=False)\n",
    "else:\n",
    "    results.to_csv('results/deepML.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "08eac26dc1f80337b7d87e94e7159a5bad95c2e85f47efef91c61359b3afbfe9"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}