02_15_2023 Time Series Forecasting Using FCNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "70f0e86e",
   "metadata": {},
   "source": [
    "# Time series forecasting example (FCNN)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c42e94d",
   "metadata": {},
   "source": [
    "## generate the time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf14dad3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from math import sin, cos\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def get_time_series_data(length):\n",
    "    a = .2\n",
    "    b = 300\n",
    "    c = 20\n",
    "    ls = 5\n",
    "    ms = 20\n",
    "    gs = 100\n",
    "\n",
    "    ts = []\n",
    "\n",
    "    for i in range(length):\n",
    "        ts.append(b + a * i + ls * sin(i / 5) + ms * cos(i / 24) + gs * sin(i / 120) + c * random.random())\n",
    "\n",
    "    return ts\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    data = get_time_series_data(3_000)\n",
    "    plt.plot(data)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55667c4d",
   "metadata": {},
   "source": [
    "## Learn the train_test_split\n",
    "\n",
    "- Split arrays or matrices into random train and test subsets.\n",
    "\n",
    "- **sklearn.model_selection.train_test_split(*arrays, test_size=None, train_size=None, random_state=None, shuffle=True, stratify=None)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e6f44d28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1],\n",
       "       [2, 3],\n",
       "       [4, 5],\n",
       "       [6, 7],\n",
       "       [8, 9]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> import numpy as np\n",
    ">>> from sklearn.model_selection import train_test_split\n",
    ">>> X, y = np.arange(10).reshape((5, 2)), range(5)\n",
    ">>> X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "68043e54",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3, 4]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> list(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d05dcaa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0, 1],\n",
       "        [2, 3],\n",
       "        [4, 5],\n",
       "        [6, 7],\n",
       "        [8, 9]]),\n",
       " array([[4, 5],\n",
       "        [0, 1],\n",
       "        [6, 7]]),\n",
       " array([[2, 3],\n",
       "        [8, 9]]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.33, random_state=42)\n",
    "X, X_train,X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3ace3f66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(range(0, 5), [2, 0, 3], [1, 4])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y,y_train,y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daf31ec2",
   "metadata": {},
   "source": [
    "## Split the dataset to train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10f17701",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from ch2.nn.time_series import get_time_series_data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "def get_time_series_datasets(features, ts_len):\n",
    "    ts = get_time_series_data(ts_len)\n",
    "\n",
    "    X = []\n",
    "    Y = []\n",
    "    for i in range(features + 1, ts_len):\n",
    "        X.append(ts[i - (features + 1):i - 1])\n",
    "        Y.append([ts[i]])\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.3, shuffle = False)\n",
    "    X_val, X_test, Y_val, Y_test = train_test_split(X_test, Y_test, test_size = 0.5, shuffle = False)\n",
    "\n",
    "    x_train = torch.tensor(data = X_train)\n",
    "    y_train = torch.tensor(data = Y_train)\n",
    "\n",
    "    x_val = torch.tensor(data = X_val)\n",
    "    y_val = torch.tensor(data = Y_val)\n",
    "\n",
    "    x_test = torch.tensor(data = X_test)\n",
    "    y_test = torch.tensor(data = Y_test)\n",
    "\n",
    "    return x_train, x_val, x_test, y_train, y_val, y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b89b7d25",
   "metadata": {},
   "source": [
    "## define this network by class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c97bd116",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class FCNN(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, n_inp, l_1, l_2, n_out):\n",
    "        super(FCNN, self).__init__()\n",
    "        self.lin1 = torch.nn.Linear(n_inp, l_1)\n",
    "        self.lin2 = torch.nn.Linear(l_1, l_2)\n",
    "        self.lin3 = torch.nn.Linear(l_2, n_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1 = F.relu(self.lin1(x))\n",
    "        x2 = F.relu(self.lin2(x1))\n",
    "        y = self.lin3(x2)\n",
    "        return y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2d968cf",
   "metadata": {},
   "source": [
    "## construct another model to compare with FCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "607544e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "class DummyPredictor(torch.nn.Module):\n",
    "\n",
    "    def forward(self, x):\n",
    "        last_values = []\n",
    "        for r in x.tolist():\n",
    "            last_values.append([r[-1]])\n",
    "        return torch.tensor(data = last_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9ab06ce",
   "metadata": {},
   "source": [
    "## constract linear interpolation for comparision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "870c418b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import interpolate\n",
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class InterpolationPredictor(torch.nn.Module):\n",
    "\n",
    "    def forward(self, x):\n",
    "        last_values = []\n",
    "        values = x.tolist()\n",
    "        for v in values:\n",
    "            x = np.arange(0, len(v))\n",
    "            y = interpolate.interp1d(x, v, fill_value = 'extrapolate')\n",
    "            last_values.append([y(len(v)).tolist()])\n",
    "        return torch.tensor(data = last_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae3fa5f4",
   "metadata": {},
   "source": [
    "## construct exponential smoothing for comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "249909cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from statsmodels.tsa.holtwinters import ExponentialSmoothing\n",
    "import torch\n",
    "\n",
    "\n",
    "class HwesPredictor(torch.nn.Module):\n",
    "\n",
    "    def forward(self, x):\n",
    "        last_values = []\n",
    "        for r in x.tolist():\n",
    "            model = ExponentialSmoothing(r)\n",
    "            results = model.fit()\n",
    "            forecast = results.forecast()\n",
    "            last_values.append([forecast[0]])\n",
    "        return torch.tensor(data = last_values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9452bef1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "137cdd05",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "## Construct different methods\n",
    "from ch2.nn.dataset import get_time_series_datasets\n",
    "from ch2.nn.dummy_model import DummyPredictor\n",
    "from ch2.nn.fcnn_model import FCNN\n",
    "from ch2.nn.hwes_model import HwesPredictor\n",
    "from ch2.nn.linear_interpolation_model import InterpolationPredictor\n",
    "\n",
    "## set the seed for reproducity\n",
    "random.seed(1)\n",
    "torch.manual_seed(1)\n",
    "\n",
    "\n",
    "## generate the time series data\n",
    "features = 256\n",
    "ts_len = 3_000\n",
    "\n",
    "x_train, x_val, x_test, y_train, y_val, y_test = get_time_series_datasets(features, ts_len)\n",
    "\n",
    "\n",
    "\n",
    "## define the net = FCNN(parameters)\n",
    "net = FCNN(n_inp = features, l_1 = 64, l_2 = 32, n_out = 1)\n",
    "net.train()\n",
    "dummy_predictor = DummyPredictor()\n",
    "interpolation_predictor = InterpolationPredictor()\n",
    "hwes_predictor = HwesPredictor()\n",
    "\n",
    "## define the Adam and the loss function\n",
    "## the net parameters could be obtained by net.parameters()\n",
    "optimizer = torch.optim.Adam(params = net.parameters())\n",
    "loss_func = torch.nn.MSELoss()\n",
    "\n",
    "best_model = None\n",
    "min_val_loss = 1_000_000\n",
    "\n",
    "training_loss = []\n",
    "validation_loss = []\n",
    "\n",
    "for t in range(10_000):\n",
    "    \n",
    "    \n",
    "    prediction = net(x_train)\n",
    "    loss = loss_func(prediction, y_train)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    val_prediction = net(x_val)\n",
    "    val_loss = loss_func(val_prediction, y_val)\n",
    "\n",
    "    training_loss.append(loss.item())\n",
    "    validation_loss.append(val_loss.item())\n",
    "     \n",
    "        \n",
    "    ## Must!! return the find-tuned network model\n",
    "    ## since we need to compute the metric on the testing dataset\n",
    "    \n",
    "    ## we return the performance metric on the valization dataset\n",
    "    ## not the training dataset\n",
    "    if val_loss.item() < min_val_loss:\n",
    "        best_model = copy.deepcopy(net) # Return a deep copy of x\n",
    "        min_val_loss = val_loss.item()\n",
    "\n",
    "    if t % 1000 == 0:\n",
    "        print(f'epoch {t}: train - {round(loss.item(), 4)}, \\\n",
    "              val: - {round(val_loss.item(), 4)}')\n",
    "\n",
    "net.eval()\n",
    "\n",
    "print('Testing')\n",
    "print(f'FCNN Loss: {loss_func(best_model(x_test), y_test).item()}')\n",
    "print(f'Dummy Loss: {loss_func(dummy_predictor(x_test), y_test).item()}')\n",
    "print(f'Linear Interpolation Loss: \\\n",
    "      {loss_func(interpolation_predictor(x_test), y_test).item()}')\n",
    "print(f'HWES Loss: {loss_func(hwes_predictor(x_test), y_test).item()}')\n",
    "\n",
    "plt.title(\"Training progress\")\n",
    "plt.yscale(\"log\")\n",
    "plt.plot(training_loss, label = 'training loss')\n",
    "plt.plot(validation_loss, label = 'validation loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "plt.title(\"FCNN on Train Dataset\")\n",
    "plt.plot(y_train, label = 'actual')\n",
    "plt.plot(best_model(x_train).tolist(), label = 'predicted')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "plt.title('Test')\n",
    "plt.plot(y_test, '--', label = 'actual')\n",
    "plt.plot(best_model(x_test).tolist(), label = 'FCNN')\n",
    "plt.plot(hwes_predictor(x_test).tolist(), label = 'HWES')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "test_n = len(y_test)\n",
    "net_abs_dev = (best_model(x_test) - y_test).abs_()\n",
    "hwes_abs_dev = (hwes_predictor(x_test) - y_test).abs_()\n",
    "diff_pos = F.relu(hwes_abs_dev - net_abs_dev).reshape(test_n).tolist()\n",
    "diff_min = (-F.relu(net_abs_dev - hwes_abs_dev)).reshape(test_n).tolist()\n",
    "plt.title('HWES Predictor VS FCNN Predictor')\n",
    "plt.hlines(0, xmin = 0, xmax = test_n, linestyles = 'dashed')\n",
    "plt.bar(list(range(test_n)), diff_pos, color = 'g', label = 'FCNN Wins')\n",
    "plt.bar(list(range(test_n)), diff_min, color = 'r', label = 'HWES Wins')\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}