Martin_Lukan_RandomForest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/mlukan/GDA3B2021/blob/main/Martin_Lukan_RandomForest.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "55CvTNrtu2sE"
   },
   "source": [
    "## Random Forests\n",
    "Here, we're going to learn how to use Random Forests through the use of the example given in \"An Introduction to Statistical Learning\" but adapted for Python rather than R. If given the chance, I can't recommend this book enough. It's the easier, introductory version of what's often called the \"Bible\" of Machine Learning - \"The Elements of Statistical Learning\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "DKq15qbGuuhu"
   },
   "outputs": [],
   "source": [
    "# Import the necessary library and datasets\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "data = pd.DataFrame(load_boston().data)\n",
    "data.columns = load_boston().feature_names\n",
    "data['MEDV'] = load_boston().target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "6HP9t861v84b",
    "outputId": "d7be14a8-6052-43ff-c1c8-40062abd3c2b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>MEDV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX  ...    TAX  PTRATIO       B  LSTAT  MEDV\n",
       "0  0.00632  18.0   2.31   0.0  0.538  ...  296.0     15.3  396.90   4.98  24.0\n",
       "1  0.02731   0.0   7.07   0.0  0.469  ...  242.0     17.8  396.90   9.14  21.6\n",
       "2  0.02729   0.0   7.07   0.0  0.469  ...  242.0     17.8  392.83   4.03  34.7\n",
       "3  0.03237   0.0   2.18   0.0  0.458  ...  222.0     18.7  394.63   2.94  33.4\n",
       "4  0.06905   0.0   2.18   0.0  0.458  ...  222.0     18.7  396.90   5.33  36.2\n",
       "\n",
       "[5 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3MIW3gjDwoeK"
   },
   "source": [
    "A description of each of the variables can be seen below:\n",
    "- CRIM - per capita crime rate by town\n",
    "- ZN - proportion of residential land zoned for lots over 25,000 sq.ft.\n",
    "- INDUS - proportion of non-retail business acres per town.\n",
    "- CHAS - Charles River dummy variable (1 if tract bounds river; 0 otherwise)\n",
    "- NOX - nitric oxides concentration (parts per 10 million)\n",
    "- RM - average number of rooms per dwelling\n",
    "- AGE - proportion of owner-occupied units built prior to 1940\n",
    "- DIS - weighted distances to five Boston employment centres\n",
    "- RAD - index of accessibility to radial highways\n",
    "- TAX - full-value property-tax rate per 10,000 dollars\n",
    "- PTRATIO - pupil-teacher ratio by town\n",
    "- B - $1000(B_k - 0.63)^2$ where $B_k$ is the proportion of blacks by town\n",
    "- LSTAT - % lower status of the population\n",
    "- MEDV - Median value of owner-occupied homes in 1000's of dollars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CqQDwS52xJG_"
   },
   "source": [
    "We're going to be using the dataset to predict MEDV."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "fyp1OVF3wAD4"
   },
   "outputs": [],
   "source": [
    "# Using a random state of 42, create a train/test set with a test set ratio of 0.20\n",
    "X_train,X_test,y_train,y_test=train_test_split(data.iloc[:,:-1],data['MEDV'],test_size=0.2,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4l-RzvIXweqg",
    "outputId": "acffa2f4-46c6-4b97-f1bd-a540f43efdec"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',\n",
       "                      max_depth=10, max_features='auto', max_leaf_nodes=None,\n",
       "                      max_samples=100, min_impurity_decrease=0.0,\n",
       "                      min_impurity_split=None, min_samples_leaf=1,\n",
       "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                      n_estimators=100, n_jobs=None, oob_score=False,\n",
       "                      random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit a Random Forest with the the following hyperparameters:\n",
    "# max_samples = 100, max_depth = 10, n_estimators = 100\n",
    "#for i in [X_train,x_test,y_train,y_test]:\n",
    "#  print(i.describe())\n",
    "# Instantiate model with 1000 decision trees\n",
    "# Train the model on training data\n",
    "RFR=RandomForestRegressor(n_estimators=100,max_samples=100,max_depth=10,random_state=42)\n",
    "RFR.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "zImZVZTryHDU"
   },
   "outputs": [],
   "source": [
    "# Predict on the test set\n",
    "predicts=RFR.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "X1TZb3hQyTct",
    "outputId": "127b6b40-e25b-4cb7-8248-8d31068fcac1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 2.21\n",
      "Mean Squared Error: 13.16\n",
      "R-squared scores: 0.82\n"
     ]
    }
   ],
   "source": [
    "# Calculate both the MSE and the R2 score`\n",
    "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\n",
    "\n",
    "# Mean absolute error (MAE)\n",
    "mae = mean_absolute_error(y_test.values.ravel(), predicts)\n",
    "\n",
    "# Mean squared error (MSE)\n",
    "mse = mean_squared_error(y_test.values.ravel(), predicts)\n",
    "\n",
    "# R-squared scores\n",
    "r2 = r2_score(y_test.values.ravel(), predicts)\n",
    "\n",
    "# Print metrics\n",
    "print('Mean Absolute Error:', round(mae, 2))\n",
    "print('Mean Squared Error:', round(mse, 2))\n",
    "print('R-squared scores:', round(r2, 2))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 297
    },
    "id": "o0H53rTyzCFR",
    "outputId": "20ada5a9-9d60-4cab-a25d-9a7ad4534595"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEYCAYAAABMVQ1yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAdBUlEQVR4nO3de5xU5Z3n8c+PBmxGkATsEBSVSwhoBAGb1pXxiiKuDpBIVHTiYILEKEZj4oaJu2bX7EyMJl5CGC+JWYIvWVDXOESZaEwk3qLSAoKIIrCttjETQhxlVZTLb/94TmN1U91d3f1UdfXj9/168XpVnXP6/J4qqr51Ls95jrk7IiLS9XXr7AaIiEgcCnQRkUQo0EVEEqFAFxFJhAJdRCQRCnQRkUR076zC+++/vw8ePLizyouIdEnPPffcX9y9Kt+8Tgv0wYMHU1tb21nlRUS6JDN7tbl5OuQiIpIIBbqISCIU6CIiiei0Y+giUr527NhBfX0927dv7+ymfGxVVlYyaNAgevToUfDfKNBFZC/19fX06dOHwYMHY2ad3ZyPHXdn69at1NfXM2TIkIL/TodcRGQv27dvp3///grzTmJm9O/fv817SAp0EclLYd652vP+K9BFpCz17t27pPXq6upYtGhRSWvGpmPoUnYGz32wzX9Td+3pRWiJNGjP/0lLyu3/a+fOnXsC/dxzz+3s5rSbttBFpKwtX76c448/nqlTpzJ06FDmzp3LXXfdRU1NDaNGjWLTpk0AzJw5k4suuojq6mo++9nP8sADDwDhfMAFF1zAqFGjGDt2LI8++igACxYsYMqUKZx00klMnDiRuXPn8vjjjzNmzBhuvPFG6urqOPbYYxk3bhzjxo3jqaee2tOeE044genTpzNy5EjOO+88Gu78tmLFCo455hiOOOIIampq2LZtG7t27eLKK69k/PjxjB49mttuuw2AN998k+OOO44xY8Zw+OGH8/jjj3f4vdIWuoiUveeff57169fTr18/hg4dyqxZs3j22We5+eabmTdvHjfddBMQDps8++yzbNq0iRNPPJGNGzcyf/58zIy1a9fy0ksvMWnSJDZs2ADAypUrWbNmDf369WP58uX88Ic/3PND8N577/Gb3/yGyspKXnnlFWbMmLFnuJJVq1axbt06DjjgACZMmMCTTz5JTU0NZ599NkuWLGH8+PG888479OrVizvuuIO+ffuyYsUKPvjgAyZMmMCkSZO47777OPXUU7nqqqvYtWsX7733XoffJwW6iJS98ePHM3DgQACGDRvGpEmTABg1atSeLW6As846i27dujF8+HCGDh3KSy+9xBNPPMGll14KwMiRIznkkEP2BPopp5xCv3798tbcsWMHc+bMYfXq1VRUVOz5G4CamhoGDRoEwJgxY6irq6Nv374MHDiQ8ePHA7DffvsB8PDDD7NmzRruvfdeAN5++21eeeUVxo8fz5e//GV27NjBtGnTGDNmTIffJwW6iJS9ffbZZ8/jbt267XnerVs3du7cuWde054hrfUU2XfffZudd+ONNzJgwACef/55du/eTWVlZd72VFRUNGpDU+7OvHnzOPXUU/ea99hjj/Hggw8yc+ZMrrjiCs4///wW29saHUMXkWTcc8897N69m02bNrF582ZGjBjBsccey1133QXAhg0beO211xgxYsRef9unTx+2bdu25/nbb7/NwIED6datG3feeSe7du1qsfaIESN48803WbFiBQDbtm1j586dnHrqqdxyyy3s2LFjTxveffddXn31VQYMGMCFF17IrFmzWLlyZYdfv7bQRSQZBx98MDU1NbzzzjvceuutVFZWcvHFF/O1r32NUaNG0b17dxYsWNBoC7vB6NGjqaio4IgjjmDmzJlcfPHFnHnmmSxcuJDJkye3uDUP0LNnT5YsWcKll17K+++/T69evXjkkUeYNWsWdXV1jBs3DnenqqqK+++/n+XLl3P99dfTo0cPevfuzcKFCzv8+q3h7GypVVdXu8ZDl3zUbbHzrV+/nkMPPbSzm9EmM2fO5IwzzmD69Omd3ZRo8v0/mNlz7l6db3kdchERSYQOuYhIEhYsWNDZTeh02kIXEUmEAl1E8uqs82sStOf9V6CLyF4qKyvZunWrQr2TNIyHntv3vRA6hi4iexk0aBD19fVs2bKls5vysdVwx6K2UKCLyF569OjRpjvlSHnQIRcRkUQo0EVEEqFAFxFJhAJdRCQRCnQRkUQo0EVEEqFAFxFJhAJdRCQRCnQRkUQo0EVEEqFAFxFJhAJdRCQRCnQRkUQo0EVEElFQoJvZZDN72cw2mtncFpY708zczPLekVpERIqn1UA3swpgPnAacBgww8wOy7NcH+Ay4JnYjRQRkdYVsoVeA2x0983u/iGwGJiaZ7nvAT8Atkdsn4iIFKiQQD8QeD3neX02bQ8zGwcc5O4PRmybiIi0QYdPippZN+AG4JsFLDvbzGrNrFb3KhQRiauQQH8DOCjn+aBsWoM+wOHAcjOrA44GluY7Merut7t7tbtXV1VVtb/VIiKyl0ICfQUw3MyGmFlP4BxgacNMd3/b3fd398HuPhh4Gpji7rVFabGIiOTVaqC7+05gDvAQsB64293Xmdk1Zjal2A0UEZHCdC9kIXdfBixrMu3qZpY9oePNEhGRttKVoiIiiVCgi4gkQoEuIpIIBbqISCIU6CIiiVCgi4gkQoEuIpIIBbqISCIU6CIiiVCgi4gkQoEuIpIIBbqISCIU6CIiiVCgi4gkQoEuIpIIBbqISCIU6CIiiVCgi4gkQoEuIpIIBbqISCIU6CIiiVCgi4gkQoEuIpIIBbqISCIU6CIiiVCgi4gkontnN0BEJHWD5z7YpuXrrj29XXW0hS4ikggFuohIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJKCjQzWyymb1sZhvNbG6e+ReZ2VozW21mT5jZYfGbKiIiLWk10M2sApgPnAYcBszIE9iL3H2Uu48BrgNuiN5SERFpUSFb6DXARnff7O4fAouBqbkLuPs7OU/3BTxeE0VEpBCFXPp/IPB6zvN64KimC5nZJcAVQE/gpCitExGRgkU7Keru8919GPBt4L/mW8bMZptZrZnVbtmyJVZpERGhsEB/Azgo5/mgbFpzFgPT8s1w99vdvdrdq6uqqgpvpYiItKqQQF8BDDezIWbWEzgHWJq7gJkNz3l6OvBKvCaKiEghWj2G7u47zWwO8BBQAfzc3deZ2TVArbsvBeaY2cnADuAt4B+K2WgREdlbQeOhu/syYFmTaVfnPL4scrtERKSNdKWoiEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJKKgQDezyWb2spltNLO5eeZfYWYvmtkaM/utmR0Sv6kiItKSVgPdzCqA+cBpwGHADDM7rMliq4Bqdx8N3AtcF7uhIiLSskK20GuAje6+2d0/BBYDU3MXcPdH3f297OnTwKC4zRQRkdYUEugHAq/nPK/PpjXnK8C/daRRIiLSdt1jrszM/h6oBo5vZv5sYDbAwQcfHLO0iMjHXiFb6G8AB+U8H5RNa8TMTgauAqa4+wf5VuTut7t7tbtXV1VVtae9IiLSjEICfQUw3MyGmFlP4Bxgae4CZjYWuI0Q5n+O30wREWlNq4Hu7juBOcBDwHrgbndfZ2bXmNmUbLHrgd7APWa22syWNrM6EREpkoKOobv7MmBZk2lX5zw+OXK7RESkjXSlqIhIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJUKCLiCRCgS4ikggFuohIIhToIiKJUKCLiCSioEA3s8lm9rKZbTSzuXnmH2dmK81sp5lNj99MERFpTauBbmYVwHzgNOAwYIaZHdZksdeAmcCi2A0UEZHCdC9gmRpgo7tvBjCzxcBU4MWGBdy9Lpu3uwhtFBGRAhRyyOVA4PWc5/XZNBERKSMlPSlqZrPNrNbMards2VLK0iIiySsk0N8ADsp5Piib1mbufru7V7t7dVVVVXtWISIizSgk0FcAw81siJn1BM4Blha3WSIi0latBrq77wTmAA8B64G73X2dmV1jZlMAzGy8mdUDXwRuM7N1xWy0iIjsrZBeLrj7MmBZk2lX5zxeQTgUIyIinURXioqIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSiIJuQVdqg+c+2Kbl6649vUgtEem4tn6eQZ9paR9toYuIJEKBLiKSCAW6iEgiFOgiIolQoIuIJEKBLiKSCAW6iEgiFOgiIokoywuLREpBF/xIarSFLiKSCAW6iEgidMhFRAqmw1TlTVvoIiKJ0Ba6tIlGwpRSKNWeQGp7HNpCFxFJhAJdRCQRBR1yMbPJwM1ABfAzd7+2yfx9gIXAkcBW4Gx3r4vb1K4ptV06ESlfrQa6mVUA84FTgHpghZktdfcXcxb7CvCWu3/GzM4BfgCcXYwGi0h+Or8hhRxyqQE2uvtmd/8QWAxMbbLMVOAX2eN7gYlmZvGaKSIirSnkkMuBwOs5z+uBo5pbxt13mtnbQH/gLzEaWQypHQpJ7fWISNuZu7e8gNl0YLK7z8qefwk4yt3n5CzzQrZMffZ8U7bMX5qsazYwO3s6Ani5je3dn9L8SJSiTkqvJbU6Kb2W1Oqk9FraW+cQd6/KN6OQLfQ3gINyng/KpuVbpt7MugN9CSdHG3H324HbC2lxPmZW6+7V7f37cqqT0mtJrU5KryW1Oim9lmLUKeQY+gpguJkNMbOewDnA0ibLLAX+IXs8Hfidt7bpLyIiUbW6hZ4dE58DPETotvhzd19nZtcAte6+FLgDuNPMNgJ/JYS+iIiUUEH90N19GbCsybSrcx5vB74Yt2l5tftwTRnWSem1pFYnpdeSWp2UXkv0Oq2eFBURka5Bl/6LiCRCgS4ikggFeicwsx5mNtbMPtXZbYkt67baZZjZfi3MO7iUbekqzGx8C/O+VMq2dCWl+DyV5TF0M/tCS/Pd/b5Idc5vpc7CSHVuBeZlvYP6An8AdgH9gG+5+/+OVOdCYLm7v5INvfBz4EygDpjp7isj1fkVMMfdX20y/WTgJnc/PFKdH7c0392/HqHGSncflz3+rbtPzDcvNjPrDxwHvObuz0Ved3fgNGBkNmk98Gt33xlp/WuAJ4F/dPf/yKYdDvwL8Fd3nxajTrbeTwGXAJ/LJq0D/sXd/z1WjWbq7g9sjdn9upifpwblujV1L7A6+weQOy6MA1ECHWhuS2MKYTiDKIEOHOvuF2WPLwA2uPs0M/s08G9AlEAHLgMWZI9nAKOBIcBYwmiZx0aqsxh41MzuAK4DqoCbgEP46HqEGC4CXgDuBv5I489BLLnr7NfCvI4VMXsAmOvuL5jZQGAlUAsMM7Pb3f2mSHUOBH4HvAmsIryGM4AfmdmJ7v7HCGXGAVcCq8zse8Ao4D8D33T3ByKsHwAzmwAsInymG76LRwLPmNl57v5kpDpHA9cSulx/D7iTcAVnNzM7391/HaMOxfn8NubuZfcPmEYIjVrgvwGfKUFNA/4eWAssAUZHXPeqnMcPEraW95oXoc7qnMeLgMtynq+M/H71BW4DNgKvEoZ0sMg1+hNC/VHgN8As4BORa6zM9zj2ewasy3n8HWBh9rgPsCZinQXA5Xmmfx34ReT37kpgN2F8pwNirjtb/9PA2DzTxwDPRKxTC0widL1+Czg6mz4y8vfzz8CPm/sXo0ZZbqG7+/3A/Wa2L2Ekxx9lu6hXufvvY9bKdk9nAt8ifICmu3tbx5hpzX+Y2RmEIRImEIYbbqjdK2Kd3dnW31vAROCfcubFrANwGGEkzmeBamAAYY9vR6wC7r4VuBW41cwGES5Ye9HMvu3ud0Yq8ykzu4Lwg97wmOx53vEy2in3fZkI/BTA3beZ2e6IdY5295lNJ7r7j80syufazIYRhtR24FDC4Z3HzOyf3P1/xaiR2c/dVzWd6O6rzaxPxDrd3f1hADO7xt2fzuq8FHnQ2PeBqIfXmirLQM+xHXgbeIewO18Zc+VmdgnhMMVvCYOL1cVcf46vEn6FP03YevpTNn0iYYs9lqsJWxsVwFJ3XwdgZscDm2MVyQ61jAUudvc/ZD+8/wN43swub/hyRKw3jnAI6RTCIaqYX4qfEraSmz4G+FnEOq+b2aWErdlxwK8BzKwX0CNinfdbmPdepBoPEQ4f3Zs9f9nM7gZuMLNZ7j4hUh0zs0+6+1tNJvYjboeO3B/Upu9fzJOMW939F60v1n7lelL0JMLWWA3wCLDY3WuLUGc3YTdoC43/4wxwdx8du2axZVv9fXK/BGb2N0CFu2+LVOMbhF3EXU2mjyKcsIpyrD4bXuJ0wkm9xUQ8sVdq2cm9a4CBwPycLcITgSPd/YeR6mwm7G3uNQu4zt2HRajR293/XzPzTnb3RzpaI1vXbOBCwutpOKF/JOEGOj9399si1dkFvEt4j3rx0Q+fAZXuHuUH18yedvej80z/W2CGu1/S4RplGui7gTXAE4SgbdRIj9C7IatzEWFLKd+bcLa7XxepzrwmNZwwZOaj7v5EjBrN1DXgJOBc4Ax3HxBx3UXvfZB9Dv4vH33BGt7DaD+4ZvY5YJiHMYkwsxsJ5wcAfuKRegaVipm1eMjD3S8oUt1hhM/ZOe7+udaWb8N6zwD+C40/Z9e7+69i1egMZjaW8H59kfAZv8/d53V4vWUa6DNpYVcn1m5L9sv8e+BL7v5Gk3nRuhiZWb6eH/2As4AlHqmHQ069owkflmlZnUsIh2DeavEPC19/bu+DhsMfRxJ6uMTsfXBIS/O9SbfJdtb4FfB9d38qe/4i4UT83wBneqQueFmdlj7TU2LUaaUNAyL/4B5AuNXkuYSeLt8nBNPaWDVSYmafJRw6nEHYoFtC6Lbc4ue8TTXKMdBLxcxWEfrOXg18I+eYIGa2yt3HFrl+L+CpWHXM7J8Jv/ivEbpC/pIwIuaQGOvPqfM08LWmJ6zMbAxwm7s3vaNVVGbWjbCLeleEdTUajzp3t9jMnnD3v+1ojWxdx7c0P/bJ/py6nyBci3AucKi7HxBhnbMJoXQgoUvp3cC/FuFz1nTPtpFYe+qlku1xPg58xd03ZtM2u/vQWDXK8qRoCbdm3N1/ama/B+4ys9OBS9z9vZbqx+Lu70c+iz4L2ADcAvzK3T8ws2K8jpL0PrBwFeclhOBYSui6OAf4JvA80OFAp/FJUJoc44x2JW9uYJtZVTZtS6z158o2FKYSQnws4TVOAx6LVOInhIvjzm04t1Wkz1n082ad7AuEc4OPmtmvCeeFogZAWQY6EOUEUaHcfYOZ/SfgfxIulmjxCtIYspOXXyL0eohlIKEnyAzgJjN7FOhlZt0jn0wsVe+DOwldMP9A+LH6DuELMM3dV7f0h23wRzM7yt2fyZ2YHbaKcRFO7jq/C1xKeI/MzHYSriC+JmKNRYQLyB4G5hEuMtro7stj1SD8wJ5J6E78acIWesyeOg1GuPt3irDeTpGnO/blhK6ytwC/jNI7LEZn9lL+AyZEXNdeFw0AJxC6+G2LWGcboevltpx//074IkS/ICOruQ/hS3dvVmtRxHXPJtzJ6njC1l+f7H17BvhqxDprcx5XEHokVUZ+n2oIJ6W+C/xd9u+/Z9NqIta5grCHMSRn2lBCF8BvRKyzmtCh4FvAoGza5sjvWe7FWIMIe0y1hN5I/1yMOin8AxbkmfbJ7Pv02xg1yvIYuplVEE4YHkjoqvZCdrb7O0Avj3fMeZqHX82m0z9JCKZrY9TpbNlhkM97pLFpsnUWvfdB0xPTMU9UN6kzgI967DjhtTxC6LHR4a5kWY1VwCm+943Tq4CHY32ms3WOJOylnU04+TYCONwjnRBt7vySmQ0nnNuIssdhZs8TNhTyHpZw97/GqFMqxfr8NqpRpoG+gHDT6WeBowi7vtWEixn2CuCuwPYeMOlF4CGPeCgk5yrHvNz9hli1SiGnfzA07iPc0G2x2ZES21mv4QKmhq5k/8fdfxJp3S94M4OWtTQvQt0j+ah7XL27HxNhnfVAs5+lWJ8zM/uAcHV1vkB3j3gysRTM7CXC56u5H6gOd5Et12Po1YSxVHabWSXwJ0Jf4a2d3K52seYHTLrB4g2YBI1P8H2VMNZKg5ijxl3dwmx39+/FqOPuFTHW05JmupKZu58YudSH7ZzXIR5GcnzOzOYC/xhptRVAb5oJ2kg1AF6MuedSBg4EfkTz79tJHS1QrlvoJdnVLpVsj2O1N+lvbmZfJ1wlGHOEwoZ1F63bpZl9M8/kfQlj1PR3997FqFsMpehKlq0zd2+j0SziXo2Yr2fQJYTj3GvcfWqEGiX5Prb0GY7dp74UStEVuly30EdaGHMZwgd+WM5zvOtdkl/0AZPyKNovtbv/qOFxdnz+MsKwwIsJWyBdSdG7kkFp9jYyzfUM+rzH6xlU/GFgg5sbFW3Spx7ocJ/61JRroB9BGL3v9SbTDyIcfulqSjFgUkllXRSvAM4DfgGM80hXopaSl6IrWWkNdfdRAGb2M8JhvoPdfXvEGhNbX6Tj3H1BCfrUl9K3c5+YWQ/gcOANd/9zjALlGug3Eu6G0vSOOPtl8/6uU1rVfn0t/12YDIh2Ys/M1vLRlvlncvdqIN6ejZldT9iyvR0Y5c0M1NSVuPu7hOEMFmW9nL5I+AJ2tUDfM0yvu+8ys/rIYV6y3iUl6lNfSl8wszc8z53LzCzKncvK9Rj6CnfPezchM1vbsAXSVViJBkzKuo01u2fTcIw4Qp3dwAfATvKPUhm194kUrtQ9g4rJzFYTLsJaSBhxtb4Y5zdKxczWeTZwmZldDpzgOXcui3F8vVy30D/RwrzYN2oouliBXYCS7Nm4u24uXqZKeKy+6Nx9TE6f+kfM7C9An654QjST25vpFOAeAHf/U6whQMo10GvN7EJ3/2nuRDObRZHv+FEMrQwl4B7v7jsDPM9Id+6+1swGR6ohUjLu/hLhKt7v5vSpX5EdSupwn/oSK/qdy8o10C8Hfmlm5/FRgFcDPYHPd1qr2q+1m1HHCvSk9mxEchWpT30pFf3OZWUZ6Nnu1DEW7ubScAXdg+7+u05sVru5+6UNjy3sW51HOOH2NI3v+9lRSe3ZyMdba33qO7Fp7eLuG4DJeaY/ZGaHxqhRlidFU2R734z6+x75ZtTZmCS/JByr22vPJmeLQKTsmdm/8lGf+omE4YwNuCxin/qyYGavufvBHV6PAr34rPHNqH/gxbsZdUO93D2bdV11z0Y+3nJ7tGUD9hWjT31ZMLPX3f2gDq9HgV58luDNqEWKLbUhQFqiLfQuxEpwb0yR1KTUpx7AzLaRf0gOIwwL3uFzmgp0ESlLZtbD3Xe0vqQ0KMteLqlp5Ze5y21piJTIM0CSh1iKRYFeAu4e7cbJIh8jpRrVMRkKdBEpV1Ut3YWrq92BqxQU6CJSrlq6M5LkoZOiIlKWUu6mWCwaNU9EypW2zNtIW+giUpbM7ADgLOAzwFrgDnff2bmtKm8KdBEpS2a2hHAHpseB04BX3f2yzm1VeVOgi0hZajKWS3fgWR1Tb5mOoYtIucq9P6oOtRRAW+giUpZSG8ulFBToIiKJ0CEXEZFEKNBFRBKhQBcRSYQCXUQkEQp0EZFE/H/E7CgEXbzGUgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the variable importance with the variable importance on the y-axis and the feature names on the x-axis\n",
    "# INSERT CODE HERE\n",
    "import pandas as pd\n",
    "dfimp=pd.DataFrame(RFR.feature_importances_, columns=['Importances'], index=X_train.columns)\n",
    "dfimp.plot.bar();\n",
    "# What are the two most important variables?\n",
    "RM and LSTAT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HpWBBs5F1iDr"
   },
   "source": [
    "# Hyperparameter Optimization\n",
    "Now we're going to try to use the optimal hyperparameters of the Random Forest algorithm using Grid Search in addition to cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OMG9lSAJ2n5z",
    "outputId": "7e36a450-4017-4f5c-d0a9-ad5f23592c2c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506, 14)"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KNGaLH2C1VZ8",
    "outputId": "4f6959c8-54a8-4e36-826a-ad65ee420072"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'criterion': 'mse',\n",
       " 'max_depth': 10,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': 100,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': 42,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 33,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# View all the possible hyperparameters to use\n",
    "RFR.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "ECWn00E52C4L"
   },
   "outputs": [],
   "source": [
    "# Create the parameter grid for only max_depth, n_estimators (number of trees), max_features (number of features) and max_samples (maximum number of observations)\n",
    "param_grid = {\n",
    "    'max_depth': list(range(0, 51, 10)),\n",
    "    'n_estimators': list(range(60, 161, 20)),\n",
    "    'max_features': ['auto', 'sqrt', None],\n",
    "    'max_samples': list(range(6, 507, 100))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "i9LI3Eui3YkD"
   },
   "outputs": [],
   "source": [
    "# Initialize a random forest model\n",
    "RFmodel=RandomForestRegressor(param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LzRN1PRz2tKT",
    "outputId": "185e40ab-c784-46a4-df77-447f0687094e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 648 candidates, totalling 1944 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  50 tasks      | elapsed:    3.3s\n",
      "[Parallel(n_jobs=-1)]: Done 498 tasks      | elapsed:   30.8s\n",
      "[Parallel(n_jobs=-1)]: Done 1033 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1785 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1944 out of 1944 | elapsed:  3.0min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score=nan,\n",
       "             estimator=RandomForestRegressor(bootstrap=True, ccp_alpha=0.0,\n",
       "                                             criterion='mse', max_depth=None,\n",
       "                                             max_features='auto',\n",
       "                                             max_leaf_nodes=None,\n",
       "                                             max_samples=None,\n",
       "                                             min_impurity_decrease=0.0,\n",
       "                                             min_impurity_split=None,\n",
       "                                             min_samples_leaf=1,\n",
       "                                             min_samples_split=2,\n",
       "                                             min_weight_fraction_leaf=0.0,\n",
       "                                             n_estimators={'max_depth': [0, 10,\n",
       "                                                                         20, 30,\n",
       "                                                                         40,\n",
       "                                                                         50],\n",
       "                                                           'max...\n",
       "                                                                            140,\n",
       "                                                                            160]},\n",
       "                                             n_jobs=None, oob_score=False,\n",
       "                                             random_state=None, verbose=0,\n",
       "                                             warm_start=False),\n",
       "             iid='deprecated', n_jobs=-1,\n",
       "             param_grid={'max_depth': [0, 10, 20, 30, 40, 50],\n",
       "                         'max_features': ['auto', 'sqrt', None],\n",
       "                         'max_samples': [6, 106, 206, 306, 406, 506],\n",
       "                         'n_estimators': [60, 80, 100, 120, 140, 160]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=2)"
      ]
     },
     "execution_count": 36,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perform a grid search using the following parameter list.\n",
    "grid_search = GridSearchCV(\n",
    "    estimator = RFmodel,\n",
    "    param_grid = param_grid,\n",
    "    cv = 3,\n",
    "    n_jobs = -1,\n",
    "    verbose = 2\n",
    ")\n",
    "\n",
    "# Fit the model to the data (this may take a couple minutes)\n",
    "grid_search.fit(data.iloc[:,:-1],data['MEDV'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NqBsg3rz2ufh",
    "outputId": "a87a5b89-7eb5-4374-8f13-55508b236234"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 50,\n",
       " 'max_features': 'sqrt',\n",
       " 'max_samples': 206,\n",
       " 'n_estimators': 80}"
      ]
     },
     "execution_count": 37,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Output the best parameters\n",
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uX8TQifH30t2",
    "outputId": "12e9ad43-10dc-4efb-ae67-432449a7436f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',\n",
       "                      max_depth=50, max_features='sqrt', max_leaf_nodes=None,\n",
       "                      max_samples=206, min_impurity_decrease=0.0,\n",
       "                      min_impurity_split=None, min_samples_leaf=1,\n",
       "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                      n_estimators=80, n_jobs=None, oob_score=False,\n",
       "                      random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 41,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using the best parameters, evaluate the MSE and R2 of a new random forest model.\n",
    "RFRbest=RandomForestRegressor(max_depth=50,max_features='sqrt',max_samples=206, n_estimators=80,random_state=42)\n",
    "RFRbest.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NaCLhTOovHmV",
    "outputId": "5c700774-6547-4f7d-f112-2a0373553f76"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Absolute Error: 2.18\n",
      "Mean Squared Error: 11.32\n",
      "R-squared scores: 0.85\n"
     ]
    }
   ],
   "source": [
    "newpredicts=RFRbest.predict(X_test)\n",
    "# Mean absolute error (MAE)\n",
    "mae = mean_absolute_error(y_test.values.ravel(), newpredicts)\n",
    "\n",
    "# Mean squared error (MSE)\n",
    "mse = mean_squared_error(y_test.values.ravel(), newpredicts)\n",
    "\n",
    "# R-squared scores\n",
    "r2 = r2_score(y_test.values.ravel(), newpredicts)\n",
    "\n",
    "# Print metrics\n",
    "print('Mean Absolute Error:', round(mae, 2))\n",
    "print('Mean Squared Error:', round(mse, 2))\n",
    "print('R-squared scores:', round(r2, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TLSEC3rbv9qH"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "Martin Lukan RandomForest.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}