grad_boost_reg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9bc8c69e-97a5-4efb-ada9-aee77d12f707",
   "metadata": {},
   "source": [
    "# Gradient Boosting Regression from Scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63766e43-de58-4c98-a683-4130f5663461",
   "metadata": {},
   "source": [
    "## Prepare the Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8051ca41-9bef-4aed-85d2-2a02e29ba077",
   "metadata": {},
   "source": [
    "### Loading the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0d908b2d-a1cb-41a9-82ee-602e6d020049",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "import pandas as pd\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "\n",
    "# Load the california_housing toy dataset\n",
    "ca_housing = datasets.fetch_california_housing(as_frame = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "04801ecc-8e92-4408-8647-e3bea7f0f85e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>MedHouseVal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.3252</td>\n",
       "      <td>41.0</td>\n",
       "      <td>6.984127</td>\n",
       "      <td>1.023810</td>\n",
       "      <td>322.0</td>\n",
       "      <td>2.555556</td>\n",
       "      <td>37.88</td>\n",
       "      <td>-122.23</td>\n",
       "      <td>4.526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.3014</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.238137</td>\n",
       "      <td>0.971880</td>\n",
       "      <td>2401.0</td>\n",
       "      <td>2.109842</td>\n",
       "      <td>37.86</td>\n",
       "      <td>-122.22</td>\n",
       "      <td>3.585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.2574</td>\n",
       "      <td>52.0</td>\n",
       "      <td>8.288136</td>\n",
       "      <td>1.073446</td>\n",
       "      <td>496.0</td>\n",
       "      <td>2.802260</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.24</td>\n",
       "      <td>3.521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.6431</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.817352</td>\n",
       "      <td>1.073059</td>\n",
       "      <td>558.0</td>\n",
       "      <td>2.547945</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.25</td>\n",
       "      <td>3.413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.8462</td>\n",
       "      <td>52.0</td>\n",
       "      <td>6.281853</td>\n",
       "      <td>1.081081</td>\n",
       "      <td>565.0</td>\n",
       "      <td>2.181467</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.25</td>\n",
       "      <td>3.422</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  Latitude  \\\n",
       "0  8.3252      41.0  6.984127   1.023810       322.0  2.555556     37.88   \n",
       "1  8.3014      21.0  6.238137   0.971880      2401.0  2.109842     37.86   \n",
       "2  7.2574      52.0  8.288136   1.073446       496.0  2.802260     37.85   \n",
       "3  5.6431      52.0  5.817352   1.073059       558.0  2.547945     37.85   \n",
       "4  3.8462      52.0  6.281853   1.081081       565.0  2.181467     37.85   \n",
       "\n",
       "   Longitude  MedHouseVal  \n",
       "0    -122.23        4.526  \n",
       "1    -122.22        3.585  \n",
       "2    -122.24        3.521  \n",
       "3    -122.25        3.413  \n",
       "4    -122.25        3.422  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Take a look at the dataframe\n",
    "data = ca_housing[\"frame\"]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "56b4b0a0-5d2d-4cbd-8490-c57436f183df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".. _california_housing_dataset:\n",
      "\n",
      "California Housing dataset\n",
      "--------------------------\n",
      "\n",
      "**Data Set Characteristics:**\n",
      "\n",
      "    :Number of Instances: 20640\n",
      "\n",
      "    :Number of Attributes: 8 numeric, predictive attributes and the target\n",
      "\n",
      "    :Attribute Information:\n",
      "        - MedInc        median income in block\n",
      "        - HouseAge      median house age in block\n",
      "        - AveRooms      average number of rooms\n",
      "        - AveBedrms     average number of bedrooms\n",
      "        - Population    block population\n",
      "        - AveOccup      average house occupancy\n",
      "        - Latitude      house block latitude\n",
      "        - Longitude     house block longitude\n",
      "\n",
      "    :Missing Attribute Values: None\n",
      "\n",
      "This dataset was obtained from the StatLib repository.\n",
      "http://lib.stat.cmu.edu/datasets/\n",
      "\n",
      "The target variable is the median house value for California districts.\n",
      "\n",
      "This dataset was derived from the 1990 U.S. census, using one row per census\n",
      "block group. A block group is the smallest geographical unit for which the U.S.\n",
      "Census Bureau publishes sample data (a block group typically has a population\n",
      "of 600 to 3,000 people).\n",
      "\n",
      "It can be downloaded/loaded using the\n",
      ":func:`sklearn.datasets.fetch_california_housing` function.\n",
      "\n",
      ".. topic:: References\n",
      "\n",
      "    - Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,\n",
      "      Statistics and Probability Letters, 33 (1997) 291-297\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print info about the dataframe\n",
    "print(ca_housing[\"DESCR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c4dcded-5be1-4e72-b57e-88bfc395e394",
   "metadata": {},
   "source": [
    "### Selecting Features and Target Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "705ae686-1854-446d-b427-2ccabfe003f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We will only use the following 3 features\n",
    "features = [\"MedInc\", \"HouseAge\", \"AveRooms\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2e1f7491-d2bb-4479-8b15-5a21f24f890c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate features and target variable\n",
    "X = data[features].copy()\n",
    "y = data[\"MedHouseVal\"].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2b73a29-1759-44e4-a8eb-2686872bc14d",
   "metadata": {},
   "source": [
    "### Splitting Training and Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "afca81e4-bcb4-43f1-b2ef-1274f28bda02",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Permute the dataframe to kill any possible ordering used\n",
    "np.random.seed(0)      # Fix a random seed to allow reproducibility\n",
    "old_index = data.index\n",
    "new_index = np.random.permutation(old_index)\n",
    "\n",
    "X = X.loc[new_index].reset_index(drop = True)\n",
    "y = y.loc[new_index].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cb5f1467-66f2-4293-892e-e46be7e0f968",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use 80% of the data for training purposes and the remaining 20% for testing purposes\n",
    "train_test_cut = int(np.floor(len(X)*0.8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3842dab2-7f3b-47a7-a38d-06fb761b3a64",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the training set\n",
    "X_train = X.iloc[: train_test_cut].copy()\n",
    "y_train = y.iloc[: train_test_cut].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "07f85004-b1a0-4a73-a468-381ebc84a2ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the test set\n",
    "X_test = X.iloc[train_test_cut :].copy()\n",
    "y_test = y.iloc[train_test_cut :].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3ff52ec7-530a-4725-962f-80aa82949c62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The training set has 16512 samples\n",
      "The test set has 4128 samples\n"
     ]
    }
   ],
   "source": [
    "print(\"The training set has {} samples\".format(len(X_train)))\n",
    "print(\"The test set has {} samples\".format(len(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c499970f-b9ef-417c-aedc-d8fc10ecc0c5",
   "metadata": {},
   "source": [
    "## Implement a Gradient Boosting Regressor from Scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85b87f38-0ceb-4f83-9ffe-8e1681f86a77",
   "metadata": {},
   "source": [
    "### Pseudo Code of the Training Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5843c30c-a4dc-440b-b6cd-34bb6564c9b1",
   "metadata": {},
   "source": [
    "In the following we denote by $\\bf x$ the collection of training samples, by $\\bf y$ the corresponding target values and by $\\overline {\\bf y}$ the average of these values."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36e7a518-1d85-4e40-9493-49c8cfb74a9b",
   "metadata": {},
   "source": [
    "0. Create an empty list `fitted_gb_reg` to store the output data.\n",
    "1. Initialize the algorithm with the average of the target values. In other words, set $F_0(\\bf x) = \\overline {\\bf y}$, where $F_k(\\bf x)$ are the values predicted by the algorithm during the $k$-th iteration (at this point we haven't done any iteration so $k=0$).\n",
    "2. Append the value $\\overline {\\bf y}$ to the list `fitted_gb_reg`.\n",
    "3. Calculate the residuals $\\bf r$, i.e., the difference between the true values and the predicted values. In other words, $\\textbf {r} = \\textbf {y} - F_k(\\textbf {x})$.\n",
    "4. Train a decision tree $T_k$ on the samples $\\bf x$ with target variable the residuals $\\bf r$.\n",
    "5. Calculate the leaves' output values using the average of the samples they contain.\n",
    "6. Append the fitted tree $T_k$ to the list `fitted_gb_reg`.\n",
    "7. Feed the samples $\\bf x$ to $T_k$ and let $T_k(\\bf x)$ be the outputted values. The new predicted values are $F_{k+1}(\\textbf {x}) =  F_{k}(\\textbf {x}) + \\alpha T_k(\\textbf {x})$, where $\\alpha$ is the `learning rate` (an hyperparameter fixed at the beginning). \n",
    "8. Increase $k$ by 1.\n",
    "9. If $k >$ `n_estimators` (where `n_estimators` is an integer hyperparameter fixed at the beginning), return `fitted_gb_reg`; else go back to step 3."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28cef331-05cb-4c11-89d0-c2425aed5425",
   "metadata": {},
   "source": [
    "The list `fitted_gb_reg` returned by this algorithm represents a fitted instance of the Gradient Boosting Regressor."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84128e04-2fad-49f0-b882-9575a7717019",
   "metadata": {},
   "source": [
    "### Coding the Training Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8ac2aa26-f89c-48ff-ad54-a3d3afdd4459",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "def grad_boost_fit(X, y, n_estimators = 100, learning_rate = 0.1):  # We set the same hyperparameter default values used by\n",
    "                                                                    # sklearn's instance of Gradient Boosting Regressor \n",
    "    \"\"\"\n",
    "    Fit my Gradient Boosting Regressor.\n",
    "    \n",
    "    :param X: the samples we use to train the model\n",
    "    :type X: pandas.DataFrame of shape (n_samples, n_features)\n",
    "    :param y: the target values of the training data\n",
    "    :type y: pandas.Series of shape (n_samples,)\n",
    "    :param n_estimators: the number of boosting stages to perform\n",
    "    :type n_estimators: int, default = 100\n",
    "    :param learning_rate: the learning rate used in the algorithm. It must be in range (0, infty)\n",
    "    :type learning_rate: float, default = 0.1\n",
    "    \n",
    "    :return: gb_reg, a dictionary with the following attributes\n",
    "    \n",
    "                 fitted_gb_reg: a list containing all the decision trees created during the boosting stages\n",
    "                            This list represents a fitted instance of the Gradient Boosting Regressor.\n",
    "             \n",
    "                 n_estimators: the max number of boosting stages performed\n",
    "                 \n",
    "                 learning_rate: the learning rate used in the algorithm\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    # STEP 0: initialize a list which will represent a fitted instance of the Gradient Boosting Regressor \n",
    "    fitted_gb_reg = []\n",
    "    \n",
    "    # STEP 1: initialize the algorithm with the mean of the target variable\n",
    "    predictions = y.mean()\n",
    "    \n",
    "    # STEP 2: add this initial value to the list representing the fitted Gradient Boosting Regressor\n",
    "    fitted_gb_reg.append(predictions)\n",
    "    \n",
    "    # Initialize a count of the number of boosting stages performed\n",
    "    n_estimators_trained = 0\n",
    "    \n",
    "    # test STEP 9 condition: keep boosting until we reach the max number of stages\n",
    "    while (n_estimators_trained < n_estimators):\n",
    "        \n",
    "        # STEP 3: calculate the residuals\n",
    "        residuals = y - predictions\n",
    "        \n",
    "        # STEP 4: instantiate and fit a decision tree which predicts the residuals\n",
    "        # STEP 5: because of the way sklearn's decision trees work, Step 5 is automatically performed at this point  \n",
    "        tree = DecisionTreeRegressor(max_depth = 3, random_state = 1) # We set the same hyperparameter default values used by\n",
    "                                                                      # sklearn's instance of Gradient Boosting Regressor \n",
    "                                                                      # We also set a random_state for reproducibility purposes\n",
    "        _ = tree.fit(X, residuals)\n",
    "        \n",
    "        # STEP 6: add the fitted tree to the list representing the fitted Gradient Boosting Regressor\n",
    "        fitted_gb_reg.append(tree)\n",
    "        \n",
    "        # STEP 7: use the tree to predict the residuals and update the predictions for the target values\n",
    "        predicted_residuals = tree.predict(X)\n",
    "        predictions += learning_rate * predicted_residuals\n",
    "        \n",
    "        # STEP 8: increase the number of boosting stages performed\n",
    "        n_estimators_trained += 1\n",
    "    \n",
    "    # exit STEP 9 condition: the list we created represents a fitted instance of the Gradient Boosted Regressor\n",
    "    gb_reg = {\"model\" : fitted_gb_reg,        # We return a dictionary containing the model and the hyperparameters we used\n",
    "              \"n_estimators\" : n_estimators,\n",
    "              \"learning_rate\" : learning_rate}\n",
    "    \n",
    "    return gb_reg"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b60579f-7a51-4c48-8bbb-64f8291e9857",
   "metadata": {},
   "source": [
    "### Pseudo Code for Making Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2352ed2-8102-4596-966d-d75dc1471bb9",
   "metadata": {},
   "source": [
    "In the following `fitted_gb_reg` is the list representing the fitted Gradient Boosting Regressor we constructed above. Moreover,  we denote by $\\hat {\\bf x}$ the collection of new samples we are trying to predict and with $\\hat {\\bf y}$ the corresponding target values predicted by the model."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6125cc19-d025-438d-b1c1-4cebf6c5d80b",
   "metadata": {},
   "source": [
    "1. Set the initial predictions to be the value we used to initialize the training algorithm. In other words, set $\\hat {\\bf y} = $ `fitted_gb_reg[0]`.\n",
    "2. For each decision tree $T_k$ in `fitted_gb_reg`, feed the samples $\\hat {\\bf x}$ to $T_k$ and get the outputted values $T_k(\\hat {\\bf x})$\n",
    "    1. Feed the samples $\\hat {\\bf x}$ to $T_k$ and get the outputted values $T_k(\\hat {\\bf x})$.\n",
    "    2. Set the new predicted values to be $\\hat {\\bf y} = \\hat {\\bf y} + \\alpha T_k(\\hat {\\bf x})$, where              $\\alpha$ is the `learning rate` that was established at the beginning of the training algorithm.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "733605f5-16f5-406d-ac50-fb69c54fa6c6",
   "metadata": {},
   "source": [
    "From the description above it is clear that the final predictions are $\\hat {\\bf y} = \\overline {\\bf y} + \\sum_ T\\alpha T(\\hat {\\bf x})$, where the sum is taken over all the trees $T$ in `fitted_gb_reg[1:]`."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6c17720-3b2c-4070-acc0-26a73732578b",
   "metadata": {},
   "source": [
    "### Coding the Making Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d984499e-1fc8-4589-9d23-e68335f87d72",
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_boost_predict(X, gb_reg):\n",
    "    \"\"\"\n",
    "    Use a fitted instance of my Gradient Boosting Regressor to make predictions.\n",
    "    \n",
    "    :param X: the samples we want to predict\n",
    "    :type X: pandas.DataFrame of shape (n_queries, n_features)\n",
    "    :param gb_reg: a fitted instance of the Gradient Boosting Regressor\n",
    "    :type gb_reg: dictionary\n",
    "    \n",
    "    :return: output, a dictionary with the following keys\n",
    "    \n",
    "                 \"predictions\", the predicted classes\n",
    "                 \n",
    "                 \"n_estimators\", the number of boosting stages performed during the training phase\n",
    "                 \n",
    "                 \"learning_rate\", the learning rate used in the algorithm\n",
    "    \"\"\"\n",
    "    \n",
    "     # Extract the model and hyperparameters from the input data\n",
    "    fitted_gb_reg = gb_reg[\"model\"]\n",
    "    learning_rate = gb_reg[\"learning_rate\"]\n",
    "    n_estimators = gb_reg[\"n_estimators\"]\n",
    "    \n",
    "    # STEP 1: set the initial predictions to be the value we used to initialize the training algorithm \n",
    "    predictions = fitted_gb_reg[0]\n",
    "    \n",
    "    # Loop through all the boosting stages\n",
    "    for i in range(1, len(fitted_gb_reg)):\n",
    "        \n",
    "        # STEP 2A: feed the samples to the tree trained at the current stage and get the output values\n",
    "        predicted_residuals = fitted_gb_reg[i].predict(X)\n",
    "        \n",
    "        # STEP 2B: update the model's predictions \n",
    "        predictions += learning_rate * predicted_residuals\n",
    "    \n",
    "    # Return a dataframe with the the predicted values and the hyperparameters used\n",
    "    output = {\"predictions\" : predictions,\n",
    "              \"n_estimators\" : n_estimators,\n",
    "              \"learning_rate\" : learning_rate,\n",
    "               }\n",
    "    \n",
    "    return output    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6a41ea9-33e7-4209-a465-628f1d7d8bd3",
   "metadata": {},
   "source": [
    "## Evaluating my Gradient Boosting Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "17a9578f-deeb-4b53-8203-bc4db8b9190e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize a dictionary to evaluate the performance of my Gradient Boosting Regressor on the test data\n",
    "my_test_errors = dict()\n",
    "# Initialize the samples we want to predict (in this case, the test data)\n",
    "X_hat = X_test\n",
    "\n",
    "# Fit an instance of my Gradient Boosting Regressor and use it to predict the test data at different learning rates\n",
    "# Evaluate the model using RMSE and collect all these values into a dictionary\n",
    "for alpha in [x/10 for x in range(1, 11)]:  # We use learning the rates 0.1, 0.2, 0.3, ..., 1\n",
    "    fitted_gb_reg = grad_boost_fit(X_train, y_train, learning_rate = alpha)\n",
    "    my_test_predictions = grad_boost_predict(X_hat, fitted_gb_reg)[\"predictions\"]\n",
    "    my_test_errors[alpha] = np.sqrt((y_test - my_test_predictions).apply(lambda x : x**2).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "285491eb-8c6d-43b5-85d8-e3f740e755ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAElCAYAAADp4+XfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAr+0lEQVR4nO3deZgdZZn38e+PACq7kHZLwIAEAWWTEFFUQF8lIE5AUYNrEGXQFzTjgCyOkBnHGR03VNCYF5m4jKAiOgED6KhhFUmQsAQIhrC1GSBhDyKQcL9/PE/Rlco53aeTru5O1+9zXec6p6qeU3XXU3XqrvU5igjMzKy5NhjqAMzMbGg5EZiZNZwTgZlZwzkRmJk1nBOBmVnDORGYmTWcE0EvJG0r6feSnpQUkg4d6piGO0lflLQs19f5Qx3PYJA0Nc/vgtw9Lnf73uzMdTK8rfeJQNJdxQqWX8slXSppwgCM/hTgAGAx8A1gyQCMc8SSNBE4CdgM+A5w0RDGMkbSdyTdKekpSQ9KmifplEGY/GOk9eUbAzXCTjekpaRUvFZIWijp+IGKpYNYZ+VpTy/1HvA6aTPt6ZX5f1TSNZIm1Tnd9d2GQx3AALoIuBPYH3gbsI+knSPigf6OSNJGEfEMsFPu9fWIOGdtAyuNb6Qr6mteRHxiqIKQtBNwFTAaWAb8HHgG2B34NPDvbb63YUSsXNfpR8RDwLR1Hc86Wg78F9AFHAl8U9INEXH5UAQzBHWyhLRN2At4I3CBpDER8XBdExyo9Weg9Gu7ExHr9Qu4CwjgsNw9OncHMDn3ezXwK+ABejYM25XGUZSfRkomS4C5pf7FaxywKfBl4A5gBbAA+GBpXNNz2fOBnwJPAlNL/S8GfgT8FbgW2BGYmcd1M/Ca0rh+DHQDTwGPA78Ddmsx7ycD1wNPAHOAF5bK7Av8Os/3CuAaYJNO6qVFXXcBZwP3kPbwrgEm5WFTW9TX9BbjOCAPewT4DPAwcDcpeX8ix7EU+FAu//5c/tLSOKbkfr9uE+fFefhtwNaVYa3q77PAQmBV7v/VPOxveTldAxxQ+t7Lcp0+AVwB/HMez4I8fFxRB6XvbAecB/wlz/uvgVd3sizL46u8xrWY92I5LCj1uy73O6bU73BgHmm9uhs4C9iqNHx34BJSQlkGXAi8sjR8Guk38Lc8fC7wSmBWizhntamTYvhxwO05lh8BG+fhAv6FtH7+Bfhg6Tt7tln20/PwX+bujUi/nwAmlsr9Hen391ie/6+Sfxd5+CeAe/P8f4Y1tzPFfH4X+A3wNGnd3hA4Ebg1L8NbgI+Vxvsa0jrzGD2/+Y+X1ptLSL+JJ4FFwD+Xvvsm4HLS+rOUlOhf1tt2rOPt6FBvyNf1VV5ApFNd7ypVyH7AS4CH8oK6gLSXEHlBPa9SgX8Fvg/MyCtnd+7/a+AMYGvSxj3yintOXtgBHFlZEYP0A5wJTCr1f5a0obqNng3iPODq3H1lad6uygv7LOCyIu4W817E/WDu/nwe/irSDzXyCvQ90mmurTqpl0o9bwD8IZf5E/BD0l72qlzPE3M9Ra63M8hJojKeA0r1cCPw29z9KCnB/Dx3PwFsCTw/x7kSeGkex49zmaktxv+CHFcAx3a47qwkbaR/lvtfAPwk1/uvcpllwOZ5+NxSXf2gVMctEwGwSa73Ytn/LNf7MmB0X8uStN6dQ896dUZ+bd1inqaW4j2DtGF9lrSe7ZDLHJzLPJWndXPuviQPfylpYxR5vSiW6/+SEtOOpWnMyMtjSV627yNt/IKUQM/I/Vark8rv7kHShvXJ3H10Hn5U7v5bHn5H6Tt9JgJSItm3VK9b5zIHleL/IT2J8j9brKM/Bm4ireetEkGQ1odzSBv5f6dnJ2QWaZ0O4MP5e1fSs6P4/0hJ4ew87Ef0bG++C/xPaZnsTlpnniWtq8Vv8SZgo3bbsSYmguprNmnjdWLuvoWeH9ADuV+xN1t85yOVcc+ltMEBXlQq+/Lc71O5++rKingHsGGLFfTPeQWdmrufJm3wdsvdK0rfGQMcD3wR+FZp2i+rzPuJubvYM70od5+Zu/+7NM5RndZLpS4m5mGPA5vmfl/P/X5c2QjN7WV5HVCaj7Gsvrd7SC6zPHfvk7uLef80aY/rYdJGY4sW4x9TGl+xfCeV+gV5775Uf/9SGcfWwDHAv+V6KZL963PMxXi2zeW/Su+J4N2sniDPICWGICerDpblauPspX6LZVB9XQF05TJzcr/Tc/doepLnTqQ94AB+Xxrv9bnfMcAu+fMNpCO5scW6VdlITi99f434S7G9O3d/P3efmbv/pxLnXqXv9JUIyq+HgbeVyhTJvdjB+zY9G/5NSEe9QU9i6CrVTzURXFYar0i/jyAlhjNI26EArsll/pi7P0I6It+oVG8/ycNOyfP6/NKwIsYipo2A+3O/t/W2HevkNdKuESwm7V1cR8qkIWlcHr5LfpXtWOm+qo9pFON6MiLuzp9vy+8vr5S9NlqfL7wtx/VI7r4/Ih6V9Hju3hRA0njSnvdmLcbRRTo0LFyf34txFt/ZPr9fUxSMiFV5/MW8dFIv0DPv90bEE8W85PfqvHdiRUR0S9qq1G9RMQzYhlwXpB/mccAHSKfitgLOj4jHWoy3OHrYENg297uLdJHy48DGLb7z3HKXtA1pL+ulLcp1kfYMIa0D9+bPt7coWzYuv48h7TiUVeu63bLsrxsiYk9Jm5LqbwrwJdIGqIjnVoCIWC5pOeko8eXV4dltwJ6kHaCZkk4HPglcCiBpEXAE6eiiv9rN85hKHLf0Y5xLSEfBk0lHMe8gbfihZ/7eml8FATtUpxsRy0r1U3V16fPoUuxHVcoVy/nTpI362Xl6K4DTSDtV00k7Gp8n7YQ8RdoJOpE1l9kzkpaQdk6rv7++tmNrWO/vGir5XkT8Q0T8a0RcXOyKkTYCABdEhIoX6Yf+vco4nupjGsW4XiBpu/z5lfn97krZduNa1Ud34e2kleom0obvxaVhqpQtEk5U+t+Z31/73BelDSSJ/tULpfLbStokf243751oNd8t6yIibiAl971I59AhHbK3Kvsk6XQTwPGSNo+I2yJiGukoopXysnojqQ6WkX74z6NnAyXSuWpI60CRaHaid3fl9+uADUp1/ULgC5Wy7Zblc3UjqePfbU7a8ytxFvHsnMe3DWkjBmlZrjY8e25ZSxoFfCEiRpM2Ql/Kw/+hEmuncbab56Kux7eIpy83RcRRpGshAP9X0h758135/ZOVdf8VEXFzdbqSRtNTP1XldWc56egRYPfSeDcAirsY50fEHqRlfwBpz/6LkjYkndPfj3SGYCJpp+aEvJ4VMRfLbCNS0oLOtz1tjaQjgnb+CzgVeKekS0kV+grS3UXj6angPkXEA/ne+COA30i6CnhPHnzmAMYM6bAPUozfIO2N9dcM4KPAZElzSXuubwReR//rZT7psPa1wBWSFpLuRgnSHk7dzgb2Ju3BPUo6vdHONNJe0W7ArZKK6xCb9vKdQlHvXaS9tB0o7ZXno5jLSRfufi1pHvDePsY5h7SHujdwlaQbSRePDwAOIZ2C7CSup0lHND+WdHdEnNRL+TGSziDNc7GO/iG/n0W6TnCqpB1yXBsCv4mI2yX9iLRuHChpdp7mXjmG80lHWn/M9fAA6RoR9CTM4kjpA5K2JJ2vL3ZK+uNHwFuAz0rakbTu9ktEXCbpf4D/Q9qJOJL0Wz0E+A9JryftIOxOOgrdnnTd4GjgqLzTsxsdJLV8pH8W6dTabyRdSFp39iVd45sKXJgT6R2kDf7zSGcxVgHflfRK0tHXhqTks4p01DAT+BjwYUkvICXgF5Fucpjb33ppFfx6/aJyNb9Nmd1Jdz3cT8rYt5LO321WObc2rvK9uZSuEeR+mwNfI63YT5AueJaHT8/fmVUZV9H/l7n7sNx9V7Q+rzyKtPF7jHRu+b2lOPeszHtxznsalXP0pJXwN/TsrZTvGuq1XlrU44tI5z7vJZ0LvRY4tDR8anX6LcZxQC7zSO7eqlr/1fnK/bag51z99zpYL16e66+btAF9IM/7KeS7Y1pNJ/f/V9Le2DLSqZyiXHF+eGyu0+KOouIC4YJWy7LUr7gL7EnShmAmPRfAO1mW0+i5jrOizXwXy6B4PZ3HfQbwglK5d5OOUFaQLmjOYPW7zfYinfZ5kLTuXATskodtTVpvlpbq9tzi+6RTK1fRc7fOCW3qpLrcz6D022H1u4aWknZqiu/s0mb+p1P6neV+b8r9VpL2+iH9/q4h7VQ8QlqXp5W+U9w19CDp2ZileRwH5+GzqFwHyf03IiWCW/Jyvp90J1DxvVPpuaNoRZ7uW/Kwo0mnPh8nrVs3Ae8tjftA0sXmR0kX7s8lX5/pbTvWyUt5BGbDnqSLSRd+3xIRvxvqeKxeec/5+ZGvSUl6Hemc/CrSDQv9PgXSj2lvGRGP5s9jSadfNgB2jIg76pruUGnCqSFbz0nal5QADiQdNv9+aCOyQbI5cLOkn5JuIf1w7v/dOpNAdr2kOaQjgimkJDBnJCYBcCKw9cMk0p0Vi4APhA9jm+Ip0u3WHyFdp7iLdH3jq4Mw7T+REsBmpFNnXyGdMhyRfGrIzKzhRtLto2ZmthacCIaQpANyC4l3DXUs1rtWrX+qRTPlkkbl1jcfyf2+MpRxr+9KranOHepYRjIngkEiaW5eoafWOI0isbR73TUA4+51HPmBtdMl3aHVm38+uh/TatWM8drE3FGdl+pnlaTHJf1Z0n9KKj/A1KoZ5VbNlL+LdFFzJemp0CFp7bOgnmaZZ/VRrqiraYMTWcduIdVtrf9tUZr/kPSspAckzVZqybbTcay3ScsXi0eWbno2VK8ADiXdk1w0of3QIMTwKdK93PcA/0m6/39v0kNsrZ5YHk7OI7U3sz/pfvz3SDooIq6M1s0or9FMuaR35n5zIuKTaxuIRnjT5Z3OX0RcS7rXfrBcTrp//yBSsxRdpHV3ZOvvgwdNeNFLc8uVh0lmkB6s+SvpwbI924xvLms2hDWdnoer7iK1QXIf6eGZE0vf7bVZ217m4bBi3J2OizZN5LJ6Q3HPvdpMt2hk6/BK/61Ln7chta54FylRXQW8sVK35desNtNq20x3uzpvM57qg3qb0NPa622537jyfLcZf6vYp+byHyE10raCdCfMqeRGCel5COxK0h/6PF7ESi/NJbP6+nMqad15bv2hdQNsc/tYR6e1Gf6GXOZh0sNV5wDb5GEbkR6wu4/0gNkjeT3YtkUdT6PURHKpf7umqIu6mdvJPOcyW5IacHuM9Lv8NKWHGDuZf9LOQAB/LZX5AOk383iez9uBT1TiLL/u6mt9z8OLFlufJO2s/QF4w6Bu8wZzYuvDiz6aW85lZpUW9i9Ij3kHcEWbcbZq0noSqzd3ewup+eOie3z+bq/N2vYyH4eVV8ZOxkWbJnJJDWadn4c9luM/o810v5vLLc919wlKTzrm+i2mczmpqYvH84/glbRpxrjNtNo2092uztuMZ7VEkPuVWywdz5qJoNX435djDnpadZ0I/H3ud0+u96IJ8tPzuKaWprUo1+GH6Ly55GJ6q60/eR6q8RzXpg7m0iYRkFrJLJLtuaWyvyM9/fs80k7D9/Oy+FMefkmLOl6tieRS/3ZNURd1M7eTec5lilZMu0lHpY/Rj0RA2mE6IXdfUyrzWdIT1t/Jy+OvuczraN0M+2n0vb6/gJRUniD91n5I2p70+vse8O3eUG94h9uLzppbnpW7f5W7D8zdLR/7r6xoU0v9ipV6JfCS3O/u3O8IOmjWtpfpHcbqeyXr2kTuAeXx9TLdsaT/VyjvGa2iZw93HyoJhZ4Nxxcr9Tu9j2n11Uz3GnXeZjytEsEupf770bqJhFbLdDqVoxh6dhTOp+c/AgK4Lw+fWqqTrUrf66u55GKZtFx/2sXTx/o5rcWws4r1pLTMiv9g2DmXGU/a8/4PejbEfyM1sleu42pT70X/dk1RF3Uzt8PfzCh6mrbYPw//BzpPBOXXTeT/cMhlNs7TOJ20TViUy53aKtZO1nfScwqrSMnjUHr+M2LUYG73fI1gTePyeyfNLV+f3x/J7500atbKfRFxX2lc25FWkE6ate3UujaR25GI6Cb9TehupMPrKaQN6WmSzqSnfjen7yaZ21L/muleG+Vl3e+/O60Yl9/fVen/Yknl+BdGxCMtvteuueRCu/VnoBRxvJZSS7bZjpK6SE97j6oMex5pOT9a6teuieTqb6mv+Hv7zRRNja9N89WXk45mJwO7kloNLf6r/ELS/y9UdfUyvnH5veX6HhErJH2clFwuBJDUTfo3trn9iHud+K6hNd2V3ztpbrld87mt9NY0b/l/C8rj6qRZ206taxO5HTUtLGlfSZtExE0RcSZpLwfSxmtTeup3KakdmSKOTUinW+hwWp00093f5pCLediEdEcQpGsEf+7P91u4K7//Xaze7PEOEbGiVK7abELxvXbNJRfarT+wlnXQJo6vtYj/IlKCG0VqXG1TVk8W1SbT2zUN0Z/fUrl89TvLSadaYO2ar/5FRLyL1NjdBsDXJT1P6X8ziiRwYB52ce7ubX27K7/3tr5/PyLGkP4C9VOko+rP9SPmdeYjgjXV1dxy0TTvpyTtTjp32auIjpq17UiH4+qtidwi/rGSzgb+HBFfajGpE4A3KzXR3U26YwjShbV78nj+QDqvOk/S1aR2//cnHcLPokUzxhHx+8p0Ommme406j/TfBu2cKKm4a2hbUuL8aC/lO3Umad35kaRf0JN8HyAl3N6+11tzyZ0o6uBgSd8inbb4eS/lp0maUur+Oj1NIH9Kqdnq5aRTZ6/P81Isi9eSTtHt32FsAy4iVkk6l3QL77m5CerqkVgnvk7aKL+M9Ns4h3SEvBnpdNvDpCayy4q63lvSt0lHOd+j7/X9/nzL6VJSk9fQc2Q0OAbzPNT68qLv5pZnUTqHTdoIrXb+uMU4dyPdNVL85d0RtDjvTmqG9rnzzvTRrG0v0zusxbjXuoncPPzL9PyX7c1tpvtuUvPFS0l7f/eRLujtVCrTRbrgdhfpPPLdpItoxfnmNZoxbjGdTprpXqPO28RcfO/ZPN+3kxJ1+c/ax1WXMZ1fIxA9TQyvIG1ILye1mwS9NN9NL80ld7j+bJqXcXFh88w2dVDMS/VVTOtNpIvDD5J+EzcA/5aHbU66aeIJ0h1RR5W+v1Wljse1qftxufuMcv1V66bDed6S9N/ij5PuGjo1D7+/l99LMf/TSv1Oy/0Wk9a3d5LW1SdI68d/5eFn5PIb5H7Fxenib0b7Wt9/RtpJeop019BF1Xqq++W2hsxsRJG0OenGjcjdp5D++vHKiOj3n9s0gU8NmdlI8xbgn/L/V2xDz80R3xy6kIY3JwIzG2nuIZ3K+UfSheMbgK9GxM+GNKphzKeGzMwartbbRyVNkrRI0mJJJ7cYvqWkCyXdIGmhpOr97WZmVrPajgjybYi3kx6E6SY9bXpkRNxSKnMqsGVEnJQfSllEelrw6VbjBBg9enSMGzeulpjNzEaq6667bnlEtHz4rc5rBBOBxRGxBEDSeaSn9cpP+QWwuSSR7s99iNUfFFnDuHHjmD9/fj0Rm5mNUJKqD8Q+p85TQ2PoecAC0lHBmEqZM0kPpiwlPSH6qYh4tjoiScdImi9p/rJly+qK18yskepMBNVHy2HNx8cPIj0M8jLSQ1lnStpijS9FzIyICRExoaurt2Y9zMysv+pMBN2kx/QLY1mzIbCjgAsiWUxqp7w/7YKYmdk6qjMRzAPGS9pe0sakVihnV8rcQ26vQ9KLSY27LcHMzAZNbReLI2KlpONI7c6MAs6JiIWSjs3DZwCfB2ZJuol0KumkiFheV0xmZramWp8sjog5wJxKvxmlz0tp3b63mZkNEv8fgZlZwzkRmJk1nBOBmVnDufVRM7NBtMf5lw7atG444qCOyvmIwMys4ZwIzMwazqeGzKwRfvqziYM2rfe8+9pBm9ZA8BGBmVnDORGYmTWcTw2ZWe2mT58+Iqc1UviIwMys4XxEYDaC3fqF3w3atHb57JsHbVo2sHxEYGbWcE4EZmYN51NDNuKc+Y8XDtq0jvvqO9oO+8IHjhi0OD77o/MHbVo28viIwMys4XxEYAPqsjftP2jT2v/yywZtWmYjmY8IzMwartYjAkmTgG+Q/rP47Ij4YmX4icD7S7HsAnRFxEOdTmPvE38wQNH27rovf6jtsHv+ZbdBiQFgu9Nuajtsv2/tNygxXHX8VYMyHTMbHLUdEUgaBZwFHAzsChwpaddymYj4ckTsGRF7AqcAl/UnCZiZ2bqr89TQRGBxRCyJiKeB84DJvZQ/Eji3xnjMzKyFOhPBGODeUnd37rcGSZsAk4Cftxl+jKT5kuYvW7ZswAM1M2uyOhOBWvSLNmXfAVzV7rRQRMyMiAkRMaGrq2vAAjQzs3oTQTewbal7LLC0Tdkp+LSQmdmQqDMRzAPGS9pe0sakjf3saiFJWwL7A/9dYyxmZtZGbbePRsRKSccBl5JuHz0nIhZKOjYPn5GLHg78OiKeqCsWMzNrr9bnCCJiDjCn0m9GpXsWMKvOOMzMrD0/WWxm1nBOBGZmDedEYGbWcE4EZmYN50RgZtZwTgRmZg3nRGBm1nBOBGZmDedEYGbWcE4EZmYN50RgZtZwTgRmZg3nRGBm1nBOBGZmDedEYGbWcE4EZmYN50RgZtZwTgRmZg1XayKQNEnSIkmLJZ3cpswBkhZIWijpsjrjMTOzNdX2n8WSRgFnAW8FuoF5kmZHxC2lMlsB3wYmRcQ9kl5UVzxmZtZanUcEE4HFEbEkIp4GzgMmV8q8D7ggIu4BiIgHaozHzMxaqDMRjAHuLXV3535lOwEvlDRX0nWSPtRqRJKOkTRf0vxly5bVFK6ZWTPVmQjUol9UujcE9gbeDhwEfE7STmt8KWJmREyIiAldXV0DH6mZWYPVdo2AdASwbal7LLC0RZnlEfEE8ISky4E9gNtrjMvMzErqPCKYB4yXtL2kjYEpwOxKmf8G3ihpQ0mbAK8Fbq0xJjMzq6jtiCAiVko6DrgUGAWcExELJR2bh8+IiFslXQLcCDwLnB0RN9cVk5mZranOU0NExBxgTqXfjEr3l4Ev1xmHmZm15yeLzcwazonAzKzhnAjMzBrOicDMrOGcCMzMGs6JwMys4ZwIzMwazonAzKzhnAjMzBrOicDMrOGcCMzMGs6JwMys4ZwIzMwazonAzKzhnAjMzBrOicDMrOGcCMzMGs6JwMys4WpNBJImSVokabGkk1sMP0DSo5IW5NdpdcZjZmZrqu0/iyWNAs4C3gp0A/MkzY6IWypFr4iIQ+uKw8zMelfnEcFEYHFELImIp4HzgMk1Ts/MzNZCnYlgDHBvqbs796t6naQbJF0s6VWtRiTpGEnzJc1ftmxZHbGamTVWnYlALfpFpftPwMsjYg/gW8AvW40oImZGxISImNDV1TWwUZqZNVydiaAb2LbUPRZYWi4QEY9FxIr8eQ6wkaTRNcZkZmYVdSaCecB4SdtL2hiYAswuF5D0EknKnyfmeB6sMSYzM6uo7a6hiFgp6TjgUmAUcE5ELJR0bB4+AzgC+LiklcCTwJSIqJ4+MjOzGtWWCOC50z1zKv1mlD6fCZxZZwxmZtY7P1lsZtZwTgRmZg3nRGBm1nBOBGZmDedEYGbWcE4EZmYN50RgZtZwTgRmZg3nRGBm1nC9JgJJby593r4y7J11BWVmZoOnryOCr5Q+/7wy7J8GOBYzMxsCfSUCtfncqtvMzNZDfSWCaPO5VbeZma2H+mp9dAdJs0l7/8Vncvf27b9mZmbri74SQfnP5r9SGVbtNjOz9VCviSAiLit3S9oIeDXwl4h4oM7AzMxscPR1++gMSa/Kn7cEbgB+AFwv6chBiM/MzGrW18XiN0bEwvz5KOD2iNgN2Bv4TK2RmZnZoOgrETxd+vxW4JcAEXFfJyOXNEnSIkmLJZ3cS7l9JK2SdEQn4zUzs4HTVyJ4RNKhkvYC9gMuAZC0IfCC3r4oaRRwFnAwsCtwpKRd25T7EulP7s3MbJD1ddfQ3wPfBF4CTCsdCbwF+FUf350ILI6IJQCSziPdhXRLpdzxpKeW9+lH3GZmNkD6umvodmBSi/6X0vce/Bjg3lJ3N/DacgFJY4DDgTfTSyKQdAxwDMB2223Xx2TNzKw/ek0Ekr7Z2/CI+GRvX2/1lUr3GcBJEbFKat9iRUTMBGYCTJgwwU80m5kNoL5ODR0L3Az8FFhK/9oX6ga2LXWPzeMomwCcl5PAaOAQSSsj4pf9mI6Zma2DvhLBS4F3A+8FVgI/AX4eEQ93MO55wPjcfPVfgCnA+8oFIuK5ZiokzQIuchIwMxtcvd41FBEPRsSMiDgQmApsBSyU9MG+RhwRK4HjSNcSbgV+GhELJR0r6dh1jtzMzAZEX0cEAEh6DXAk6VmCi4HrOvleRMwB5lT6zWhTdmon4zQzs4HV18XifwYOJe3Rnweckvf0zcxshOjriOBzwBJgj/z6t3xhV0BExO71hmdmZnXrKxH4PwfMzEa4vh4ou7tV/9wsxBSg5XAzM1t/9NUM9RaSTpF0pqS3KTmedLroPYMTopmZ1amvU0M/BB4G/gB8FDgR2BiYHBEL6g3NzMwGQ5//WZz/fwBJZwPLge0i4vHaIzMzs0HRVzPUzxQfImIVcKeTgJnZyNLXEcEekh7LnwW8IHcXt49uUWt0ZmZWu77uGho1WIGYmdnQ6OvUkJmZjXBOBGZmDedEYGbWcE4EZmYN50RgZtZwTgRmZg3nRGBm1nBOBGZmDVdrIpA0SdIiSYslndxi+GRJN0paIGm+pDfUGY+Zma2po/8sXhv5PwvOIv3PcTcwT9LsiLilVOy3wOyICEm7Az8Fdq4rJjMzW1OdRwQTgcURsSQinib95/HkcoGIWBERkTs3BQIzMxtUdSaCMcC9pe7u3G81kg6XdBvwK+AjrUYk6Zh86mj+smXLagnWzKyp6kwEatFvjT3+iPhFROwMHAZ8vtWIImJmREyIiAldXV0DG6WZWcPVmQi6gW1L3WOBpe0KR8TlwCskja4xJjMzq6gzEcwDxkvaXtLGpD+7n10uIGlHScqfX0P6G8wHa4zJzMwqartrKCJWSjoOuBQYBZwTEQslHZuHzwDeBXxI0jPAk8B7SxePzcxsENSWCAAiYg4wp9JvRunzl4Av1RmDmZn1zk8Wm5k1nBOBmVnDORGYmTWcE4GZWcM5EZiZNZwTgZlZwzkRmJk1nBOBmVnDORGYmTWcE4GZWcM5EZiZNZwTgZlZwzkRmJk1nBOBmVnDORGYmTWcE4GZWcM5EZiZNZwTgZlZw9WaCCRNkrRI0mJJJ7cY/n5JN+bX1ZL2qDMeMzNbU22JQNIo4CzgYGBX4EhJu1aK3QnsHxG7A58HZtYVj5mZtVbnEcFEYHFELImIp4HzgMnlAhFxdUQ8nDuvAcbWGI+ZmbVQZyIYA9xb6u7O/do5Gri41QBJx0iaL2n+smXLBjBEMzOrMxGoRb9oWVA6kJQITmo1PCJmRsSEiJjQ1dU1gCGamdmGNY67G9i21D0WWFotJGl34Gzg4Ih4sMZ4zMyshTqPCOYB4yVtL2ljYAowu1xA0nbABcAHI+L2GmMxM7M2ajsiiIiVko4DLgVGAedExEJJx+bhM4DTgG2Ab0sCWBkRE+qKyczM1lTnqSEiYg4wp9JvRunzR4GP1hmDmZn1zk8Wm5k1nBOBmVnDORGYmTWcE4GZWcM5EZiZNZwTgZlZwzkRmJk1nBOBmVnDORGYmTWcE4GZWcM5EZiZNZwTgZlZwzkRmJk1nBOBmVnDORGYmTWcE4GZWcM5EZiZNZwTgZlZw9WaCCRNkrRI0mJJJ7cYvrOkP0h6StIJdcZiZmat1fafxZJGAWcBbwW6gXmSZkfELaViDwGfBA6rKw4zM+tdnUcEE4HFEbEkIp4GzgMmlwtExAMRMQ94psY4zMysF3UmgjHAvaXu7tyv3yQdI2m+pPnLli0bkODMzCypMxGoRb9YmxFFxMyImBARE7q6utYxLDMzK6szEXQD25a6xwJLa5yemZmthToTwTxgvKTtJW0MTAFm1zg9MzNbC7XdNRQRKyUdB1wKjALOiYiFko7Nw2dIegkwH9gCeFbSNGDXiHisrrjMzGx1tSUCgIiYA8yp9JtR+nwf6ZSRmZkNET9ZbGbWcE4EZmYN50RgZtZwTgRmZg3nRGBm1nBOBGZmDedEYGbWcE4EZmYN50RgZtZwTgRmZg3nRGBm1nBOBGZmDedEYGbWcE4EZmYN50RgZtZwTgRmZg3nRGBm1nBOBGZmDVdrIpA0SdIiSYslndxiuCR9Mw+/UdJr6ozHzMzWVFsikDQKOAs4GNgVOFLSrpViBwPj8+sY4Dt1xWNmZq3VeUQwEVgcEUsi4mngPGBypcxk4AeRXANsJemlNcZkZmYVioh6RiwdAUyKiI/m7g8Cr42I40plLgK+GBFX5u7fAidFxPzKuI4hHTEAvBJYtI7hjQaWr+M41tVwiAGGRxzDIQYYHnEMhxhgeMQxHGKA4RHHQMTw8ojoajVgw3UccW/Uol8163RShoiYCcwciKAAJM2PiAkDNb71NYbhEsdwiGG4xDEcYhgucQyHGIZLHHXHUOepoW5g21L3WGDpWpQxM7Ma1ZkI5gHjJW0vaWNgCjC7UmY28KF899C+wKMR8b81xmRmZhW1nRqKiJWSjgMuBUYB50TEQknH5uEzgDnAIcBi4K/AUXXFUzFgp5nWwXCIAYZHHMMhBhgecQyHGGB4xDEcYoDhEUetMdR2sdjMzNYPfrLYzKzhnAjMzBpuRCeCDpq42FnSHyQ9JemEIYrh/bl5jRslXS1pjyGKY3KOYYGk+ZLeMNgxlMrtI2lVfhZlUGOQdICkR3M9LJB02kDH0EkcpVgWSFoo6bLBjkHSiaV6uDkvk62HII4tJV0o6YZcFwN+LbGDGF4o6Rf5N3KtpFfXEMM5kh6QdHOb4fU1yRMRI/JFukB9B7ADsDFwA7BrpcyLgH2ALwAnDFEMrwdemD8fDPxxiOLYjJ5rRrsDtw12DKVyvyPdSHDEENTDAcBFw2Dd3Aq4BdiuWFeHYnmUyr8D+N0Q1cWpwJfy5y7gIWDjQY7hy8Dp+fPOwG9rqIs3Aa8Bbm4z/BDgYtLzV/sO5LZiJB8R9NnERUQ8EBHzgGeGMIarI+Lh3HkN6VmKoYhjReS1DdiUFg/21R1Ddjzwc+CBAZ5+f2KoWydxvA+4ICLugbSuDkEMZUcC5w5wDJ3GEcDmkkTaYXkIWDnIMewK/BYgIm4Dxkl68QDGQERcTpq3dmprkmckJ4IxwL2l7u7cbzjHcDQp4w9JHJIOl3Qb8CvgI4Mdg6QxwOHAjAGedscxZK/LpyEulvSqIYpjJ+CFkuZKuk7Sh4YgBgAkbQJMIiXogdZJHGcCu5AeNr0J+FREPDvIMdwAvBNA0kTg5dSz09ab2rZpIzkRdNR8xXCJQdKBpERw0lDFERG/iIidgcOAzw9BDGeQ2ppaNcDT7k8MfyK1ybIH8C3gl0MUx4bA3sDbgYOAz0naaZBjKLwDuCoiettbrTOOg4AFwMuAPYEzJW0xyDF8kZSYF5COWq9nYI9KOlHbNq3OtoaG2nBovqKjGCTtDpwNHBwRDw5VHIWIuFzSKySNjoiBamyrkxgmAOelMwCMBg6RtDIifjlYMUTEY6XPcyR9e4DroaM4cpnlEfEE8ISky4E9gNsHMYbCFOo5LdRpHEeRGqcMYLGkO0nn6a8drBjyenEUpIu2wJ35NZjq26YN9AWP4fIiJbklwPb0XAB6VZuy06nnYnGfMQDbkZ6sfv1Q1gWwIz0Xi18D/KXoHuzlkcvPYuAvFndSDy8p1cNE4J6BrId+xLEL6Zz0hsAmwM3Aqwd7eQBbks5bbzqE6+Z3gOn584vzujl6kGPYinyBGvgY6Vx9HfUxjvYXi9/O6heLrx2o6Y7YI4LooIkLSS8B5gNbAM9Kmka6W+CxduMd6BiA04BtgG/nPeGVMcCtDHYYx7tI7T49AzwJvDfy2jeIMdSqwxiOAD4uaSWpHqYMZD10GkdE3CrpEuBG4Fng7IhoeVthXTHkoocDv450ZDLgOozj88AsSTeRNoInxQAeoXUYwy7ADyStIt3NdfRATb8g6VzSXWujJXUDpwMblWKorUkeNzFhZtZwI/lisZmZdcCJwMys4ZwIzMwazonAzKzhnAjMzBrOicBGBEkrBnl6Vw/QeIrWTq+XdJukr3TwncMk7ToQ0zcDJwKzliT1+oxNRLx+ACd3RUTsBewFHCppvz7KH0ZqBM1sQIzYB8rMJL0COIvUdPFfgY9FxG2S3gH8E+kp0geB90fE/ZKmk9qzGQcsl3Q76cnvHfL7GRHxzTzuFRGxmaQDSE+mLwdeDVwHfCAiQtIhwNfysD8BO0TEoe3ijYgnc1s2Y/I0PgYck+NcDHyQ1NbO3wH7S/on0oOAtJrPta03ax4fEdhINhM4PiL2Bk4Avp37Xwnsm/fCzwM+U/rO3sDkiHhf7t6Z1OjZROB0SRu1mM5ewDTSXvoOwH6Sng98l9R+1BtIG+leSXohMB64PPe6ICL2idQA3q3A0RFxNTAbODEi9oyIO3qZT7OO+IjARiRJm5H+9OdnuekOgOfl97HAT3Jb7huzeuNhsyPiyVL3ryLiKeApSQ+Q2rrprkzu2ojoztNdQDqiWAEsiYhi3OeS9u5beaOkG4FXkhpXuy/3f7WkfyW1c7MZqQmE/synWUecCGyk2gB4JCL2bDHsW8DXImJ26dROodqmzlOlz6to/ZtpVaZVk8HtXBERh+Zmpq+U9IuIWEBqeO+wiLhB0lRSOzRVvc2nWUd8ashGpNxw4J2S3g3P/d9r8X/QW5JasAT4cE0h3AbsIGlc7n5vX1+IiNuBf6fnPyk2B/43n456f6no43lYX/Np1hEnAhspNpHUXXp9mrTxPFrSDcBCev5+cDrpVMoVpAu5Ay6fXvoEcImkK4H7gUc7+OoM4E2Stgc+B/wR+A0psRTOA07Mt5y+gvbzadYRtz5qVhNJm0XEivxHJmcBf46Irw91XGZVPiIwq8/H8sXjhaTTUd8d2nDMWvMRgZlZw/mIwMys4ZwIzMwazonAzKzhnAjMzBrOicDMrOH+P0a3yntYkKLUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Plot the errors we obtain\n",
    "sns.barplot(x = list(my_test_errors.keys()), y = list(my_test_errors.values()))\n",
    "plt.xlabel(\"Learning Rate\");\n",
    "plt.ylabel(\"RMSE\");\n",
    "plt.title(\"Performance of my Gradient Boosting Regressor\\n on the Test Set at Different Learning Rates\", weight = \"bold\");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1581beec-0ed5-4cd7-afc6-39299304cb7a",
   "metadata": {},
   "source": [
    "## Comparing my Model to sklearn's Implementation of Gradient Boosting Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "338e25c5-212b-4e9b-bf43-f6d29f4f28c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.2, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.3, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.4, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.5, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.6, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.7, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.8, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=0.9, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(learning_rate=1.0, random_state=1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "# Now do the same thing using the sklearn's Gradient Boosting Regressor\n",
    "\n",
    "sklearn_test_errors = dict()\n",
    "\n",
    "for alpha in [x / 10 for x in range(1, 11)]:\n",
    "    gb_regr = GradientBoostingRegressor(learning_rate = alpha, random_state = 1)\n",
    "    gb_regr.fit(X_train, y_train)\n",
    "    sklearn_test_predictions = gb_regr.predict(X_test)  \n",
    "    sklearn_test_errors[alpha] = np.sqrt((y_test - sklearn_test_predictions).apply(lambda x : x**2).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4a8b0694-0a07-4d32-8fd8-c2269cc25c80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAGeCAYAAAC97TYdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABCwUlEQVR4nO3debgcVZn48e9L2BcBJSqrAUURFVAj7mPGZQQHBtxGUHFwY9Afjs6MCu5xG3V0FBU0gwwyoIIrTkAEFQSVRQgKyG5kjaiEJWwiGHh/f5zTpNJ039tFuu+9yf1+nqef7lq6zqnTVW+9faqqOzITSZIkSYNZbbIrIEmSJK1MTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoKeZiNgyIn4aEXdFREbEbpNdp6kuIj4ZEYtre31nCMubU5d19RCqN+VExL51/U57MNPVW0TMqu2WEbFRHXd1HZ4zqZWbQmyT6aexX8ya7Lpo+jCBnoIaB4DO48aIODkiZg9h8e8B5gALgc8DVw5hmausiNgZOBBYH/gycEKPeZqJzZKIWK8x7QONaUfW0YsobX/E6NdAY2l8mZn1IN+/dkS8LyIujIg/R8TtEXFZRBwSERsPubq9HEHZlhYNa4GDJKBd23xGxN0RcU1d77WHVZdx6tnvi9jQ26RH2XO61v/P9XP/t1GVKYiIuV2xdFTl5DiPWUNY9pjLiIh/jIhfR8SdEXFbRFwaEZ9vUY4dFSO2+mRXQGM6AbgKeB7wd8DTImK7zLyh7YIiYo3M/Cvw2Drqc5n5oBO4xvJWdZ32Ojcz3zrA/BsCewOHR8QM4M3dM2TmQuAdQ6uhHmAits+IWAf4KfB04B7gJOAG4DHA/sA84JY+7109M5euaB0y8yMruowhmFef9wb+H6UNJq1eE9wm91C+WG8KvBL4r4i4ODNPHlWBUy32RsRqAJl532TXZYg6iepalH0Z4KvAbfX1bQ94xxBFxFOBY4C7gO8CfwW2B3YF3j7KstVCZvqYYg/gaiCBPevwJnU4gT3quCcCP6AcrBZTdrKtGsvozP8OShJ+JXBaY3znMQtYD/g08DvgDuB8YJ/GsubWeb8DfIuyU+/bGP9D4GvAn4FzKAnEYXVZFwFPaSzrG5SeobuB24FTgSf1WPeDgF8DdwInAhs35nkG8KO63ncAZwPrDtIuPdp6JnA4cC0lKJ4N7FKn7dujveb2WMasxvSbgfPq+H9ojEvgyDp+Th2+usf731DrcgvlS06znDcAF9R1/i3wXmD1HvWJ+plnV9tfWcfNrmWeVMu5C7gc+HCLbfQpwM9rm3U+57d0tdtpdXhPYClwI7Bj9/Q6z3Mo2+ctwPWUXsSH1WlrAD8G/khJWJYA84Etx9reu8YfAFxB2ea+BqzZ9VnMqsOvBi6pbXIzcBbwnD5tcGB9793Ndq7TtgI2rK+PrPP9d12Pe2q5r61l3V7HXQG8tbGMNSnJ2S2UM0b7NdZno679ZU4dXhf4ZJ3/TuBX1DjSVZd5wPGUffZCYKeu5TUf+46zzXfq8l91+BuN+XagbGc3UvbH44HHDbL/1ekvAs6r63JrXZ+X0XvfvLpPm5xWhz8B/Kyu8xnAoxrlvKy22W3AZ4HT63ve0eezn1OnL2mM+2kd9+7GuPHi9HOA39T1Oxo4ti7j4K596Rd1W7idGoMo8eWcWudravt34uDGwLdru/+Fsk/8d2O7+gplf7obuA6Y3/XZfhv4A2Xb+ynw9Mb0Tnt+CvglZd+e1aON1qCc8bysrt+lwL8Cq/VYt89R9uvfA6/p0+Zze3zmnRjTGX5TLWe5/Xy89hojzm3UWPasQZY1Ttt31//+7bSr3H+r0z7fNf6hjderA++q63snJZa8eYxj19WDxncfgz0mvQI+enwojQSacpnNyxs7wbOBR1IO7vcA36P0VGfdkdaqy+jM/2fgfykHzAMoyWtSEtCDgYdSkuKkHMCPqDtjAnvXZTUD13mU5HiXxvj7KEn0ZXV4CXAucGYd/kVj3c4Avg4cyrKD1KU91r1T75vq8Efr9CfUoJSUg+H/UA58Gw3SLl3tvBolQUrKgfloyjf9e2s771zbKWu7HUzj4N5YzqxG+3yuPu9c2+Q2SoKQDJZAX1vb5946/II6zz83ph/ZaOsP9dmGPlqnf6oOP60OX1KHv8ay7eC/gZ8AJ7XYRn/Bsi9VX6Ek04fXafvWaadREqC/UJPn7ul1+Iks+0J1DMsO0KdSvgysRUnQ/5ey3fyqTj+pUZ8HbO9d42+q7XZXHX5jj3Vah7Lt3Fk/s6OBi4F/6tMGZ9RlHTtOWx3ZqMdplH3sKcD7KNvol2tZf67zPLO+78ONuh9B+WLRWc5GXfvLnDp8DMv206MoSdt9jenNuhxX1y+Bn9fpH6Rss53P9mBg53G2+S/Xx821rFfWeTalJGBZ17OzL/2BkmSMuf/VZSyiJGhH1TZYUOvYa9/8YJ82Oa0O30vZt66tw0fX6Y+p5Sbwf7VOnf3vHX0+0zk0Eui6rlfVcZ0v4GPGI0rM6rTPT4GTG+Ue3LWvJOVL7n8DrwNeXMctru12Xh3+an3fx+rwWfWzORH4TZ32pjrtIuBLwPeBm+q09Vj2Rft0SsKflH3i0V3teR/lC9HRwKY92ug/63y/p8TpxXX4PT3W7ZzG53kH8JAey9uF8gUrKcniwcABXfv5Ynrs5+O11xj77kaNZc8aZFnjtP3BjeUdUYcf06Pcves8d1M6C94DPLVrnk/UeS6r69zZrv+JMfYPH8N7THoFfPT4UHr3AmXdkVajfOtsBpGDKT0czeDdec8bupbdCX771uGHN+Z9VB339jp8Zh2eW4d/R6PHszH+t5REZ986fA/lUoYn1eE7Gu/ZHHgbpZfsi42yN+ta93fV4U4ScUIdPqQO/19jmTMGbZeutti5TrsdWK+O+1wd94063Fmn08b4vGY11mN7ysH4J5SD4aEsC5pH1vnn0D+Bflod1/ly8c463El0OklNJwH+Y586bVunX1WHP12HD6rD36zD7wGeDKwNzGixjf6yvv8NlAR4jc77G222iHLgvT957tWmtY2ScnDsfG6dL0nbNdbn3ygH5f+t0/7Cst6sftt7Z3wnqeu895Ae67R+/cwWAbsB23S2rz5t8Nu6rE/W4bUb5SXLegqPrMOnd71/TeAVwIco293ldb731ukL6/A+dXj3xrI36tpf5lB6c7OuwxdrO55GI8lv1OUHdfhveeA+ev8yB9zmm49rqQk38O467qeN9/26jtuPwfa/P1ESqlcAj6Ps593b2WlddVuu/o02OLQOv74OX1SH39+sZ/1cOnHjHX3Wf06Pdb8POLAxz5jxiHIGIilxNep7zqd3An1b5zOv439Qx3c6Qr7UqMO6lN7hrNN2pmzbnXZ7S512DKWjYOPGtH/sUafj6rj/6GrPo8bYPqJ+bgk8r47bow5f37VuN1H2nTUoX5YSmN1nuXNpxNJB9/Px2muM9diosexZK9r2XXWdNUa5Mygx/j6W38ZOqu0UlP0mWZaIz6/DZ4+1f/gY3sNroKe2EygH0Zso33JPyszmzQePr4+mx3QNnzFOGZ1l3ZWZ19TXl9XnR3XNe072vm7zslqvJXX4T5l5a0TcXofXA4iIbSk9Tev3WMZMSg9bx6/rc2eZnfdsXZ/P7syYmffW5XfWZZB2gWXrfl1m3tlZl/rcve6Dup4SyF5Wh+cBb2zx/n7rPas+v7xr/kdExPqZeUdzZGb+NiLOAp4ZEU+nJCD3UYIylAPRFpSe6v+g9HR8kXLQH8S/UQ4ch7PsYPlBSgLUsXl9/hHlMoF+Ouv29PpoekxEzKT00M3omrYWsAHl1H5Hv+29X7veLzPviIi3UBLa4wEiYhGwDyVp6Na53nnLOryUcu3k7sA2PeY/s2v4eMq9Dd1m1udO+11en6/oMW/TrPq8GuVsU1P39t/dHuvx4G1M+fzfSln/Yynr36nPpY15LwN2ouxfnbLH2v/+mfLl79t1+CbKuh37IOrZbxvotPOlAJl5T0T8jmWfw1juofSu/h3waOCVEXFwZt7NsvXvF486bX551oyn1mHHHuVcnJlLGsOdZb+oPjqC0vYH1+W8ldIhci/wzYjYh9KbP4eS0O5FSbJ+EhEvbSy3Wad+MXGsY8vMxvp1Pv/OcjaNiDUb816amX8BiIg7gYfQ+xgxiPHiZ7/2uqhFGeMt62D6tH0OeJ14Paa9NiIOBJ5PufZ5b0rv9+6UM36ddXt919t7Hes0Av4Kx9T2P5n5r5n5scz8YSOgXV2fv5eZ0XlQTiP+T9cy7h6njM6y1omIrerrx9Xna7rm7bese8cZ7vh7yk7/G8o3+0c0pkXXvJ1EPbvGX1Wf70+0ImK1iAjatQuN+beMiHXr637r3saX6/MvMvM3bd7Y+ILSvd5X1+d/6Fq3bbqT54aj6vOnKUH/1Mzs/DLBlZn5bMqZgp0pp5rfGRFbAkTEdvWxVp9lL8jMHSnJ0xxKr8gnI6L5pfzXlM96T5ZPrLt11u2zPdbtBMqXhhmU3pf1WD7J7t5u+m2j/dq12/9m5ubAZpSD3xbAB/rMe3x9fllEPD4zl2bmOyjr3Mv9das/Q9dJnv+WEot/2Jlcn39fnzvbZOeG1n6urs/3ADMb7bgm8NKuecdqj87+O/DxoW63nYRqVk2QOvXZrjFrc//qTB9r//thZm5LuQ/kFcDDgI8/yHr2W+dOO28L5SY9en8B6uWuLDcX70xJ7p9K6V2H8eNRp9xmwtNsq6bu7bqz7H/pWvajM/Mi4ObM3IXyBXNHyhmsV1N6nJdm5qsoierjKWfLXkT50t9Z7mNrTIX2xwMolzd0vhR11qmznD9k5j2NeZudMuPtn+N95uPFz37t1cZ4yxqr7aF0ZIy1Dp34u2lm/j4zj87MV1Puf6Eu90aWte8OjTqsRrnHBR7Efqx27IFeOX2dcgPZyyLiZMoO/WjKr3Vsy7IdfFyZeUOU3zZ+BfDjiDiDchoPyuUSw/Sn+rwtpadqpwexjHmUa/j2qD/PcwXwXOCZtG+XBZRLEZ4O/DwiLmbZtWdfehB16zgFeCHLkv1hOIRSp69FxHEsC5Q3UBLYXr5J6Q15bh0+qjHtSxHxOEqv0OqUBOVeSk8iLOs1ejLltHK34+uvjPyOkoSvRUkgml+ebqOcpj4beHtELM7Mjz9gSeWa+jfXebahHBweDzyrrmdnu3k6pZf8eX3Wdxj+VLer6ymXIMGy3qxuX6Ak97OBBRFxIqUNnjFAOXdS2np9ytmAW4AXdM3zDUryfnD9SbmXjLXAzFwcEd+i7L+/jIgfUxLO51L2m7kD1AvKTWXbAB+JiH8A/iszrxtj/k9ExL2UyxKg3ER7T0R8jbI//m1EzKck8k+mfJ7fobTrePvfr6P8Xvq1LOvpX9KoJ8BTI+JLwK8z8ysDrmPTMZSzDi+s+9bDKfvDwDLz5oj4HOX613dFxDzGj0cn1HXZNiJ+Qkn+dhiwyEMo28N/RsSzKNf87kD5vLcGDqqf3W8oX6hm1ffdCuxdezYXULbB5nZ+aqOeP42IGylfvu6ixc9u1jOSXwbeCXwjIk6i3HjXqfuD1fnMd42IL1IuT/juAO8br73aWJG276zDo4BDIuIK4H2NMzAdLwQ+GxG/oJyFfjglGf8L5X6FjIhDKZdJ/TgijqfEkmdQLv/bl+HtH+pnsq8h8fHAB12/wtFnnh0oPWB/YtkdzgcD69fpPa+zousa6DpuA8qd51fVZV3YNX0uva8764z/fh3ekz7X9tbhGZRT/rdRrjN9VaOeO3Wt+5w6/A66ruOiBIkfs+xbePNXOMZslx7t+HDKgeE6yjVl5wC7Nabv211+j2Xcv540rlNsTD+42X6McQ104z3fZ/nraINyKcj5lIPejZSbKF87zrb0bbquM63jO8u6nXLz2m+AVzWmL/e59Fjue1l29/cdtd06Nzwu12b1M7mVZde+PqBNgb+hHLxvqnW6gGXXXG5AuQ7zTsp1x51rWO9vb/pv78uN7/4s+rTXtZTetZspSc6sMdp3Xcp1+pfW9yyhnA7+HPXXJlh23fHcrve+jNKrdyflJ7K+zvLXv65FuWlsCWXffFuP9b6a5feX9Sk3F/2WcrC9vrbdM3rVhfIltnvbm1Pf37mh7QHXo/LAa6Dvo/yqw3eo147X+Z5MuTnuJso2ewLw+Bb73xcoX9LuomxDPwWeXKetVtusc9PjCX3a5DQaMY+uONX4LDq/wvE5lt2s9pY+n/ucOn1JY9xDWHZT4D8NGKe7f4XjuPr+zs2/+9In/tT1OLu2y5Ladu+o03anfDlZUreDK4C31WnPpFwCcBMlwbuGcilX55rnbern+Mf6/tOoN7b2O4b0aaM1KdeXX0GJMZdREuq+17DX8u7/7Hoscz3KmajODbeda5zH3c/Haq8x1mGj7mWvSNvX6a+ibO+d65s36VHuUyiXKV1V13UJ5QzP3zXmWYOSQHd+NehPtW12HWv/8DG8R2eHkSRpWoqIDTPz1vp6PUqCszHwwsw8ZYLKXY1yun874E2Z2euyM0lThJdwSJKmux9GxKWUsw+7UZLnCyhneUbp8IhYSumZfh4leb6e8vNxkqYwLy6XJE13Cyg3Ob+Xcv3z4ZRT4aP+x79fUS7jeB/lJtFvUn72bcmIy5W0gryEQ5IkSWrBHmhJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoDU1EHBkRH6uvnxsRl092nSZSRLwmIn402fWQtGIiYm5EfK3ttOksIvaNiF80hu+IiG0ms04TKSK2qus8Y7LroolhAj3FRMTVEXFPRGzSNf78iMiImPUglzs7Ik6IiFsiYklEXBIRH4+IjYdS8S6Z+fPMfNwwllXb5IVjTJ8TEffV4HVHRPw+Ij48jLLHKHNW/TxW74zLzK9n5t+NqLyrI+Kuun5/rF9W1h9FWZKmphoHZj2I920aEV+JiOtrDLmyxpDtRlBNADJz/cy8ckWX0+yYGWOejIg767rdGBHHRMRGK1r2OGUud1zKzGvrOt87grKOrHnBHRFxc0T8eJSfnQZjAj01XQXs3RmIiCcB6zzYhUXEs4DTgDOA7TJzI2AXYCmwY5/3rN5r/BR2fQ1e6wPPAd4YEXtOcp2Gbfe6fjsBTwbeM+wCJvtzn+zypckyqm0/Ih4GnAmsCzwX2AB4CnA68KKJrMuI7Vjj4zbAxsDcya3O0P1nXb/Ngd8D/zPsAib7c49ipclLV5qKTjNHA69rDP8TcFRnICKeFhF/am7sEfHyiDi/z/L+E/hqZn4iM/8E939b/lBmnlbfv29EnBERn4uIm4G5EfHoiDg1Im6q3+q/3vxWHxFPjohfRcTtEfFNYO3GtDkRsagxvFlEfDciFkfEVRHxL41pcyPiWxFxVF3WxRExu047GtgKOL5++373eI2XmVdRDhjbN8p4VkScGxG31udnddVtfv1mvzAi3tyYtnNELIiI22qbf7ZO+ll9XlLr9cx44CnMjIj9I+K3tef/0IiIOm1GRPxXbderIuKA7h7tMdbvj8DJlES6U9YzIuLMKGcXLoiIOY1pW0fEz2rb/qTW42t1Wqcn/Y0RcS1wah3/hoi4tNb75Ih4VB0fdRu5obblhRHxxDrtJVHObNwe5SzAOxt1eHNt25trW2/W1U7/LyJ+C/x2vPWXhiUiDqzb6u0RcXlEvKDHPGtE6dH8bkSs2WP6WPve6+t+dHuUXt9/bkybExGLah3+CHx1rFjYo9y++1uXfwVuA/bJzN9lsSQzv5qZX6zL6hcHvh3ljNetNYY8oVH+w+q+fFtEnAM8uqt+GRGPqa/XiojPRMS1NY7Oi4h1utrh32tc+UNEvL5O2w94DfDuGmeP77OO98vM24D5LB//x4rxa0XEwVF656+vr9eq0zaJcuZ2SX3vzyNitehxXIqus5IRcVpEfDTKcfX2iPhRNM4sR8TrIuKaKMfXD8Q4Z1ob63cX8C2Wj/9jHV/XiYj/jRLLL611bR6br67b4IXAnRGx+jjb9L51W769lvWaOv4xEXF63VZujJITdN4z1vH3tChnw88A/kz5ArRyyEwfU+gBXA28ELgceDwwA7gOeBSQwKw63yXAro33HQf8e4/lrQfcC8wZp9x9KT3SbwNWp/R4P4bSQ7EWMJOSNB5c518TuIYSnNcAXgH8FfhYnT4HWFRfrwacB3ywvm8b4ErgxXX6XOAvwEvq+n4COLu7Tcao+/1l1eFtKd/Qn1+HHwrcAuxT123vOvywOv104EuULwA7AYuBF9RpZ1EOPADrA8+or2fVz2P1rjb8RWM4gROAjSjBdjGwS522f/0Mt6D0lvyke3m9tov6egvgN8Dn6/DmwE21/Varn9lNwMzGOnymtv1zKAfTr3Wtx1GUbWUdYE9gIWX7Wx14P3Bmnf/F9bPcCIg6z6Z12h+A59bXGwNPqa+fD9xI6fVaC/gi8LOudvpx/ZzWmex90Mf0eACPo8TWzerwLODR9fVc4Gt1f/gBcCQwozmtvh5v3/t7SmIZwPMoCUJnv5hDibmfqvvFOowTC7vq33N/6zHf2cDccdriAXGgjn8Dpcd6LeBg4PzGe46lJHLrAU+kxNzu+PeY+vpgSlL70Lq844FPdLXDRyjHkpfUdtq4Tj+SelwZo/7NsjYGfgR8pDF9rBj/kdpGD6cc584EPlqnfQKYV+u1BqUHP+q0q2kcl+g6JlDO+v4OeGz9bE8DPlmnbQ/cQYnHa1Li81/pc5xrtkFt76OBC+rweMfXT9b135hy7LiQ5Y+XVwPnA1vWevbdpmvZtwGPq+/dFHhCfX0M8L76nrWB59Tx4x1/TwOuBZ5Qp68x2bFh4Bgy2RXw0fWBLEug31933l0oycXqLJ9AHwh8vb5+KCXgbNpjeVvU923XGPefwBLgTuD9ddy+wLXj1G1P4Nf19d8A11ODSR13Jr0T6Kd3L5ty+cFX6+u5wE8a07YH7upukzHqNQe4r67TbXV9vwesWafvA5zT9Z6z6jpvSfmCsUFj2ieAI+vrnwEfBjbpev8sBkugn9MY/hZwUH19KvDPjWkv7F5ej+3iDuD2Ot8pwEaNbeHorvlPppy52IpycFq3Me1rPDCB3qYx/YfAGxvDq9Xt61GUZPgK4BnAal1lXgv8M/CQrvH/Qzn92Blen3KwmNVop+dP9r7nY3o9KB0EN9R9b42uaXMpCd/pwBdYPs7Nbew/ffe9PmV+H3h7fT0HuAdYu2vZfWNh17J67m895lsI7N8Y/gdKrLwd+FEd94A40GM5G9V5NqQk939l+ePKf9AjgaZ8ebiT+uWkTnsmcFWjHe5i+Vh6A8s6K45ksAT6trpe9wKXAZvXaePF+N8BL2lMezFwdX39EeD/qMl5V5lXM34C/f7G9LcCJ9XXHwSOaUxbt24LYyXQf6nrdx/lMs8d6rTxjq/3J9N1+E08MIF+Q2N4rOPJerUOL6ers4Py5eswYIuu8X2Pv412+kiv9Z7qDy/hmLqOBl5NScqO6jH9a8DuUW4k+0fg55n5hx7z3ULZ4TbtjMjMd2e5Dvo4SmLecV3zjRHx8Ig4tp4evK2W2TkFtRnw+6x7QHVNn3V5FLBZPR20JCKWAO8FHtGY54+N138G1o5212Ndn5kbZeZDKIH+LuB/G3Xtrts1lG/amwE3Z+btPaYBvJHSg3BZPfW0W4s6wQPXq3Pj32Ys397LtX0fe2bmBpQDznYs+yweBbyyq32fQ/nMO+v353HKao57FPD5xrJuphwEN8/MU4FDgEOBP0XEYRHxkPq+l1N6La6pp/Ke2VjX+9s/M++g9Gh02njQ9ZeGJjMXAu+gJK031Fi3WWOWZwA7UHoN84FLAMbe94iIXSPi7Hr6fwll/2jeIL44M//StcxBY2G//a3bTSwf/+fX+P+vlB7Lpvv3wyiXmX0yIn5X4//VddImlN7I1Vl+v+0X/2dSEsTzGm10Uh1/fx0zc2ljuBkrB/WUul5rA18Gfh4RazN+jO8+PlxTxwF8mvIF5Ef1soWDWtZpoPhf4/NN4yzrM3X9ZlGOb52b9Mc7vg5yrOmO/z236cy8E3gV5QzqHyLiB7HsZsZ3U44T50S59OgNjfL7HX/HqtOUZwI9RWXmNZRvmS+h9KZ2T/895VvcSynf8I7us5w7gV8CLxuk2K7hT9RxO9TE9LWUHQTK6cPNIyIa82/VZ7nXUXobNmo8NsjMlwxQp171GnvmzFuBbwC711HXU4JC01aUU47XAw+NiA16TCMzf5uZe1NO730K+E5ErNe2Tj38gXJ2oGPLQd+YmadTeiQ+U0ddR+kxaLbvepn5yVrOQyNi3XHKaq7PdZTe8eby1snMM2v5X8jMp1JOuT0WeFcdf25m7kFpq+9Tetyhq/1r+z2M2sY9ypcmRGZ+IzOfw7JL5D7VmPwjSgw8JSIe0ev9jLHvRbmO9ruU/fQRNfk5kWUxFFZgux9jf+t2CrBnDHZzVrM+rwb2oPTQb0hJ3KDUfzHlzFYzlvSL/zdSEr4nNNpowyw3xA2ibfz/K3A4sDXl0pIxYzwPPD5sVceRmbdn5r9n5jaU48m/xbLr5FckZi0X/6NcD/6wQd6YmdcCb6d0cqzD+MfXQY413fG/3/GEzDw5M19E+VJ2GfCVOv6PmfnmzNyMcmbkS1GugR/r+Nur/JWGCfTU9kbKqe07+0w/ivKt70mU3uR+3g28ISIOioiHA0TEFpQAM5YNKJcNLImIzamJUnUWJYD+S73p4GXAzn2Wcw5wW71RYZ3as/HEiHjaOOV3/IkWNxbUXvm9gIvrqBOBx0bEq2tdX0U5NXpCZl5HufTkExGxdkTsQGn3r9dlvTYiZmZm5xIRKKcDF1N69h/sDQ/fAt4eEZtHuTHzwJbvPxh4UUTsxLKzES+ubbt2lBtztqhfxBZQbgpds/ZS7d5/sUC55u89UW8YiogNI+KV9fXTIuLpEbEG5bTsX4B767JfExEb1gPYbZR2gvJl5vURsVNNKv4D+GVmXt1ynaWhiYjHRcTz6zb5F0qSt9xPkGXmf1K231Oi66dFq777HqV3dy1qshkRuwJD+ZnLcfa3bp+lXP96dJQbw6MmkzuNU8wGwN2UntF1KfstAFl+qu17lLiybkRsTznF/wA1dn4F+Fzj+LN5RLx4wNVtG/9nAK+nfJ5XjhfjKdfuvj8iZtbP+IOUz5WI2C3KzXHBsjbutHOrenX5DmW7eVaUG1M/zPJfrMaUmT+mJKb7Mf7x9VuUeL5xPY4fMM7i+27TEfGIiPiH2glyNyU/uBcgIl5Zt3soZ76zTut7/B10facqE+gpLMsd0wvGmOU4yje748ZIssnMX1CuXf0b4IrGKbTTKDd09fNhyo1ft1JupLm/Jzwz76H0au9L2VleRY+e8jrvvZSkbSdKr/qNlB6CDccou+kTlAC3JPrfab5Z1N+Bppweeijl7m0y8yZgN+DfKQeDdwO7ZeaN9b17U3pXrqe06YdqgIJyDfrFdbmfB/bKzL/UU24fB86o9XrGgOvS8RVKD9eFwK8pQWYp/Q+Cy8nMxZQvUB+oB4g9KKftFlN6EN7Fsv37NZRrDm8CPgZ8kxL8+i37OEpP3LFRTt1eBOxaJz+k1v0WSjvfxLKe8H2Aq+t79qecsSAzTwE+QOmN+wPlpqq9BllPaYTWotxgdSPlVPvDKfvQcjLzo5Qe3p9ExEO7pvXd9+olA/9CSWBuofTozh9i/Xvubz3qfyPlcpS/AL+gXPt8PiVBfssYyz+Kso//nnLD89ld0w+gXJLwR8oZsa+OsawDKZdCnF3r+xOWXYIwnv8Btq9x9vtjzHdBjdO3UJL5l2bmzXXaWDH+Y5ROhgspN2f/qo6DckP6TyiJ4lnAl7L+chWDHZd6ysyLKTfsH0uJibdTrvvuG5d7+DTlWLY6Yx9fPwIsqtN+Qknex4r/Yx1PVqMcR6+nXNr3PMq13QBPA35ZP4P5lGv9rxrg+LvS6txNqpVURPyOcrr9J5NdFz14tXdqXmZ2n+oaRVnfBC7LzA+NuixJ0tjqWdMlwLZZfoZ1lGW9hdIR9LxRljMd2AO9EouIl1NOk5w62XVRO/VU20vqKa3NgQ8x9mU4K1LW0+qp29UiYhdK78L3R1GWJGl8EbF7vfxlPcpZvN+w7EbNYZazaUQ8u8b/x1F6gkdyrJluVsZ/GxLlx8cp1xHtU68x08olKJfIfJNyrd4PKNfejcIjKZfXPIxyKu8tmfnrEZUlSRrfHpSb/4NyCcleOZpLAtYE/ptyz9MSymUjXxpBOdOOl3BIkiRJLXgJhyRJktTCSBPoiNglIi6P8t/zD/gB8vrzWMdH+a/1iyPi9aOsjyRJkrSiRnYJR/0txiso/6O+CDgX2DszL2nM815gw8w8MCJmApcDj6w/kdbTJptskrNmzRpJnSVp1M4777wbM3Pm+HNOvHqT6ecpf9V8eOfPExrTN6T8TuxWlHtoPpOZY/18mTFb0kqtX8we5U2EOwMLM/NKgIg4lnLR/CWNeRLYoP5I+fqU3xVc2r2gplmzZrFgwVg/jSxJU1dE9PvL40lVOz0OpdHpERHzm50ewP8DLsnM3TudHhHx9bE6PYzZklZm/WL2KC/h2Jzl/998Ecv/9znAIcDjKT/K/RvKD28/4BclImK/iFgQEQsWL148qvpK0nR2f6dHTYg7nR5NrTs9JGlVNMoEutffUnZfL/Jiyj8ibUb5F51DIuIhD3hT5mGZOTszZ8+cOSXPfErSys5OD0ka0CgT6EXAlo3hLShBt+n1wPeyWEj5q8ntRlgnSVJvdnpI0oBGmUCfC2wbEVtHxJrAXpT/R2+6FngBQEQ8AngccOUI6yRJ6s1OD0ka0MgS6MxcChwAnAxcCnwrMy+OiP0jYv8620eBZ0XEb4BTgAMz88ZR1UmS1JedHpI0oJH+lXdmngic2DVuXuP19cDfjbIOkqTxZebSiOh0eswAjuh0etTp8yidHkfWTo/ATg9J09RIE2hJ0srDTg9JGox/5S1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLXgz9hJmtZ2/M7JIy/jgle8eORlSNJ0MFVitj3QkiRJUgsm0JIkSVILJtCSJElSCybQkiRJUgveRChpSvjWt3ce6fL/8ZXnjHT5kqTpwwRakiRJrUz3Tg8v4ZAkSZJaMIGWJEmSWvASDkn3mzt37ipRhiRNF6OOqcbs3uyBliRJklowgZYkSZJaMIGWJEmSWvAaaGkKuvTjp450+Y9/3/NHunxJmk6M2dOPPdCSJElSCybQkiRJUgtewiH18fHXvmLkZbzva98ZeRmSNF2MOm4bs9VhAq0p75B/P36kyz/gv3Yf6fIlSdKqxQRakiQNzag7PcCOD00+r4GWJEmSWrAHWgM7/W+eN9LlP+9np490+ZI0nRizpdGxB1qSJElqwQRakiRJasFLOB6kaz/ypJEuf6sP/qbn+Gd/8dkjLfeMt50x0uVL0nQy6pgNxm1pMqzUCfRT33XUyMs479OvG3kZkjQdTGbMnqxOD0mrppFewhERu0TE5RGxMCIO6jH9XRFxfn1cFBH3RsRDR1knSVJvxmxJGszIEuiImAEcCuwKbA/sHRHbN+fJzE9n5k6ZuRPwHuD0zLx5VHWSJPVmzJakwY2yB3pnYGFmXpmZ9wDHAnuMMf/ewDEjrI8kqT9jtiQNaJQJ9ObAdY3hRXXcA0TEusAuwHf7TN8vIhZExILFixcPvaKSJGO2JA1qlAl09BiXfebdHTij36nAzDwsM2dn5uyZM2cOrYKSpPsZsyVpQKNMoBcBWzaGtwCu7zPvXngqUJImkzFbkgY0ygT6XGDbiNg6ItakBNz53TNFxIbA84D/G2FdJEljM2ZL0oBG9jvQmbk0Ig4ATgZmAEdk5sURsX+dPq/O+lLgR5l556jqIkkamzFbkgY30j9SycwTgRO7xs3rGj4SOHKU9ZAkjc+YLUmDGekfqUiSJEmrGhNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSpBRNoSZIkqQUTaEmSJKkFE2hJkiSphZEm0BGxS0RcHhELI+KgPvPMiYjzI+LiiDh9lPWRJPVnzJakwaw+qgVHxAzgUOBFwCLg3IiYn5mXNObZCPgSsEtmXhsRDx9VfSRJ/RmzJWlwo+yB3hlYmJlXZuY9wLHAHl3zvBr4XmZeC5CZN4ywPpKk/ozZkjSgUSbQmwPXNYYX1XFNjwU2jojTIuK8iHhdrwVFxH4RsSAiFixevHhE1ZWkaW1oMVuSVnUju4QDiB7jskf5TwVeAKwDnBURZ2fmFcu9KfMw4DCA2bNndy9DkrTihhazI2I/YD+ArbbaagRVlaTJNcoe6EXAlo3hLYDre8xzUmbemZk3Aj8DdhxhnSRJvQ0tZmfmYZk5OzNnz5w5c2QVlqTJMsoE+lxg24jYOiLWBPYC5nfN83/AcyNi9YhYF3g6cOkI6yRJ6s2YLUkDGtklHJm5NCIOAE4GZgBHZObFEbF/nT4vMy+NiJOAC4H7gMMz86JR1UmS1JsxW5IGN8proMnME4ETu8bN6xr+NPDpUdZDkjQ+Y7YkDcZ/IpQkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaGGkCHRG7RMTlEbEwIg7qMX1ORNwaEefXxwdHWR9JkiRpRY0sgY6IGcChwK7A9sDeEbF9j1l/npk71cdHRlUfSdLY7PSQpMGsPsJl7wwszMwrASLiWGAP4JIRlilJehAanR4vAhYB50bE/Mzsjtk/z8zdJryCkjSFjPISjs2B6xrDi+q4bs+MiAsi4ocR8YQR1keS1N/9nR6ZeQ/Q6fSQJHUZZQIdPcZl1/CvgEdl5o7AF4Hv91xQxH4RsSAiFixevHi4tZQkwRA7PYzZklZ1o0ygFwFbNoa3AK5vzpCZt2XmHfX1icAaEbFJ94Iy87DMnJ2Zs2fOnDnCKkvStDW0Tg9jtqRV3SgT6HOBbSNi64hYE9gLmN+cISIeGRFRX+9c63PTCOskSeptaJ0ekrSqG9lNhJm5NCIOAE4GZgBHZObFEbF/nT4PeAXwlohYCtwF7JWZ3T0ekqTRu7/TA/g9pdPj1c0ZIuKRwJ8yM+30kDSdjfJXODo9FCd2jZvXeH0IcMgo6yBJGp+dHpI0uJEm0JKklYedHpI0GP/KW5IkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJamHMBDoint94vXXXtJeNqlKSJEnSVDVeD/RnGq+/2zXt/UOuiyRJkjTljZdAR5/XvYYlSZKkVd54CXT2ed1rWJIkSVrlrT7O9G0iYj6lt7nzmjq8df+3SZIkSaum8RLoPRqvP9M1rXtYkiRJWuWNmUBn5unN4YhYA3gi8PvMvGGUFZMkSZKmovF+xm5eRDyhvt4QuAA4Cvh1ROw9AfWTJEmSppTxbiJ8bmZeXF+/HrgiM58EPBV490hrJkmSJE1B4yXQ9zRevwj4PkBm/nFUFZIkSZKmsvES6CURsVtEPBl4NnASQESsDqwz3sIjYpeIuDwiFkbEQWPM97SIuDciXtGm8pKk4TFmS9JgxvsVjn8GvgA8EnhHo+f5BcAPxnpjRMwADqX0XC8Czo2I+Zl5SY/5PgWc3L76kqRhMGZL0uDG+xWOK4Bdeow/mfGD587Awsy8EiAijqX8LN4lXfO9jfI34U8bsM6SpOEzZkvSgMZMoCPiC2NNz8x/GWPy5sB1jeFFwNO7lr858FLg+YwRjCNiP2A/gK222mqsKkmSHhxjtiQNaLxLOPYHLgK+BVxP+QfCQfWat/vvvw8GDszMeyP6LzozDwMOA5g9e7Z/IS5Jw2fMlqQBjZdAbwq8EngVsBT4JvDdzLxlgGUvArZsDG9BScKbZgPH1kC8CfCSiFiamd8fYPmSpOExZkvSgMb8FY7MvCkz52Xm3wL7AhsBF0fEPgMs+1xg24jYOiLWBPYC5nctf+vMnJWZs4DvAG81EEvSpDBmS9KAxuuBBiAingLsTbk7+4fAeeO9JzOXRsQBlJsNZwBHZObFEbF/nT7vQddakjRUxmxJGtx4NxF+GNgNuBQ4FnhPZi4ddOGZeSJwYte4nkE4M/cddLmSpOEzZkvSYMbrgf4AcCWwY338R732LYDMzB1GWz1JkiRpahkvgd56QmohSZIkrSTG+yOVa3qNr/9EtRfQc7okSZK0qhrzVzgi4iER8Z6IOCQi/i6Kt1Eu6/jHiamiJEmSNHWMdwnH0cAtwFnAm4B3AWsCe2Tm+aOtmiRJkjT1jJdAb5OZTwKIiMOBG4GtMvP2kddMkiRJmoLGvIQD+GvnRWbeC1xl8ixJkqTpbLwe6B0j4rb6OoB16nDnZ+weMtLaSZIkSVPMeL/CMWOiKiJJkiStDMa7hEOSJElSgwm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUwkgT6IjYJSIuj4iFEXFQj+l7RMSFEXF+RCyIiOeMsj6SpP6M2ZI0mNVHteCImAEcCrwIWAScGxHzM/OSxmynAPMzMyNiB+BbwHajqpMkqTdjtiQNbpQ90DsDCzPzysy8BzgW2KM5Q2bekZlZB9cDEknSZDBmS9KARplAbw5c1xheVMctJyJeGhGXAT8A3jDC+kiS+jNmS9KARplAR49xD+ityMzjMnM7YE/goz0XFLFfvd5uweLFi4dbS0kSGLMlaWCjTKAXAVs2hrcAru83c2b+DHh0RGzSY9phmTk7M2fPnDlz+DWVJBmzJWlAo0ygzwW2jYitI2JNYC9gfnOGiHhMRER9/RRgTeCmEdZJktSbMVuSBjSyX+HIzKURcQBwMjADOCIzL46I/ev0ecDLgddFxF+Bu4BXNW5QkSRNEGO2JA1uZAk0QGaeCJzYNW5e4/WngE+Nsg6SpMEYsyVpMP4ToSRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUwkgT6IjYJSIuj4iFEXFQj+mviYgL6+PMiNhxlPWRJPVnzJakwYwsgY6IGcChwK7A9sDeEbF912xXAc/LzB2AjwKHjao+kqT+jNmSNLhR9kDvDCzMzCsz8x7gWGCP5gyZeWZm3lIHzwa2GGF9JEn9GbMlaUCjTKA3B65rDC+q4/p5I/DDXhMiYr+IWBARCxYvXjzEKkqSKmO2JA1olAl09BiXPWeM+FtKMD6w1/TMPCwzZ2fm7JkzZw6xipKkypgtSQNafYTLXgRs2RjeAri+e6aI2AE4HNg1M28aYX0kSf0ZsyVpQKPsgT4X2DYito6INYG9gPnNGSJiK+B7wD6ZecUI6yJJGpsxW5IGNLIe6MxcGhEHACcDM4AjMvPiiNi/Tp8HfBB4GPCliABYmpmzR1UnSVJvxmxJGtwoL+EgM08ETuwaN6/x+k3Am0ZZB0nSYIzZkjQY/4lQkiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJamGkCXRE7BIRl0fEwog4qMf07SLirIi4OyLeOcq6SJLGZsyWpMGsPqoFR8QM4FDgRcAi4NyImJ+ZlzRmuxn4F2DPUdVDkjQ+Y7YkDW6UPdA7Awsz88rMvAc4FtijOUNm3pCZ5wJ/HWE9JEnjM2ZL0oBGmUBvDlzXGF5Ux7UWEftFxIKIWLB48eKhVE6StBxjtiQNaJQJdPQYlw9mQZl5WGbOzszZM2fOXMFqSZJ6MGZL0oBGmUAvArZsDG8BXD/C8iRJD54xW5IGNMoE+lxg24jYOiLWBPYC5o+wPEnSg2fMlqQBjexXODJzaUQcAJwMzACOyMyLI2L/On1eRDwSWAA8BLgvIt4BbJ+Zt42qXpKkBzJmS9LgRpZAA2TmicCJXePmNV7/kXKaUJI0yYzZkjQY/4lQkiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWpIkSWphpAl0ROwSEZdHxMKIOKjH9IiIL9TpF0bEU0ZZH0lSf8ZsSRrMyBLoiJgBHArsCmwP7B0R23fNtiuwbX3sB3x5VPWRJPVnzJakwY2yB3pnYGFmXpmZ9wDHAnt0zbMHcFQWZwMbRcSmI6yTJKk3Y7YkDSgyczQLjngFsEtmvqkO7wM8PTMPaMxzAvDJzPxFHT4FODAzF3Qtaz9KbwfA44DLH2S1NgFufJDvXVGTVbbrPD3Knm7lTmbZK1ruozJz5rAqMyxTNGbDyvs5r4xlT7dyJ7Ns13nlKbtnzF59xeozpugxrjtbH2QeMvMw4LAVrlDEgsycvaLLWZnKdp2nR9nTrdzJLHsy13nEplzMhun5OU+3dbatp0fZq9o6j/ISjkXAlo3hLYDrH8Q8kqTRM2ZL0oBGmUCfC2wbEVtHxJrAXsD8rnnmA6+rd3Y/A7g1M/8wwjpJknozZkvSgEZ2CUdmLo2IA4CTgRnAEZl5cUTsX6fPA04EXgIsBP4MvH5U9amGckpxJSvbdZ4eZU+3ciez7Mlc55GZojEbpufnPN3W2baeHmWvUus8spsIJUmSpFWR/0QoSZIktWACLUmSJLWwSibQA/wd7XYRcVZE3B0R75zAcl9T//72wog4MyJ2nMCy96jlnh8RCyLiORNRbmO+p0XEvfW3ZodigHWeExG31nU+PyI+OBHlNso+PyIujojTh1HuIGVHxLsa63tRbfOHTkC5G0bE8RFxQV3noV0bO0DZG0fEcXX7PicinjiEMo+IiBsi4qI+0yP8S+uhmayYPWDZI4nbkxWzBym7Md9Q47Yx25hdpw89ZtflTmzczsxV6kG5+eV3wDbAmsAFwPZd8zwceBrwceCdE1jus4CN6+tdgV9OYNnrs+ya9x2Ayyai3MZ8p1JuQHrFBK7zHOCESdi+NgIuAbbqbG8TVXbX/LsDp07QOr8X+FR9PRO4GVhzgsr+NPCh+no74JQhlPs3wFOAi/pMfwnwQ8rvIj9jWPvydHxMVsxuUfbQ4/ZkxexBy27MN7S4bcw2ZjfmGXrMrsua0Li9KvZAj/t3tJl5Q2aeC/x1gss9MzNvqYNnU35DdaLKviPrFgSsR48/PxhFudXbgO8CNwyhzLZlD9sg5b4a+F5mXgtle5vAspv2Bo6ZoHIT2CAignLgvxlYOkFlbw+cApCZlwGzIuIRK1JoZv6Msg79+JfWwzNZMXvQskcRtycrZg9UdjXsuG3MNmZ3DD1m12VNaNxeFRPozYHrGsOL6ripVu4bKd+EJqzsiHhpRFwG/AB4w0SUGxGbAy8F5g2hvFZlV8+sp6h+GBFPmKByHwtsHBGnRcR5EfG6IZQ7aNkARMS6wC6UA+BElHsI8HjKn2r8Bnh7Zt43QWVfALwMICJ2Bh7F8L6crki9NJjJbMvJituTFbMHKntEcduYbczumIyYPWjdBrYqJtAD/dXsZJYbEX9LCcQHTmTZmXlcZm4H7Al8dILKPRg4MDPvHUJ5bcv+FeU/7HcEvgh8f4LKXR14KvD3wIuBD0TEYyeo7I7dgTMyc6xv48Ms98XA+cBmwE7AIRHxkAkq+5OUg9/5lF6zXzOcnpQVrZcGM5ltOVlxe7Ji9qBlH8zw47Yxu3fZHcbs0RtqrBnZH6lMosn6q9mByo2IHYDDgV0z86aJLLsjM38WEY+OiE0y88YRlzsbOLacJWIT4CURsTQzv78C5Q5Udmbe1nh9YkR8aYLWeRFwY2beCdwZET8DdgSuWIFyBy27Yy+Gcypw0HJfD3yynnJeGBFXUa5tO2fUZdfP+fVQbhIBrqqPUfIvrYdnMttysuL2ZMXsQcseRdw2ZhuzgUmL2QPVrZUVuYB6Kj4oXwquBLZm2QXsT+gz71yGdxPhuOUCW1H+wetZE73OwGNYdkPKU4Dfd4Ynoq3r/EcyvJsIB1nnRzbWeWfg2olYZ8ppsVPqvOsCFwFPnKhtG9iQch3YehPY1l8G5tbXj6jb1yYTVPZG1JtfgDdTrnEbxnrPov/NKH/P8jejnDOMMqfjY7Jidovta+hxe7Jidtv2rvMPJW4bs43ZjXlGErPr8iYsbg+lwlPtQbnT8grKnaDvq+P2B/avrx9J+SZyG7Ckvn7IBJR7OHAL5bTJ+cCCCVznA4GLa7lnAc+ZiHK75j2SISXQA67zAXWdL6Dc/DOUA+Ag6wy8i3JX90XAOyZqnevwvsCxwypzwLbeDPgR5Vq6i4DXTmDZzwR+C1wGfI/6iwkrWOYxwB8oN60topy6b5YZwKG1Tr8BZg+zvafbY4DPeCQxe8CyRxK3Byh3JDF7kLK75j2S4XV8GLN7l70vxuxhlDuhcdu/8pYkSZJaWBVvIpQkSZJGxgRakiRJasEEWpIkSWrBBFqSJElqwQRakiRJasEEWlNGRNwxweWdOaTlzImIWyPi1xFxWUR8ZoD37BkR2w+jfEmaDMZsTWcm0FplRcSY/7SZmc8aYnE/z8wnA08GdouIZ48z/56AwViSKmO2Viar4l95axUSEY+m/PD5TODPwJsz87KI2B14P+Wfjm4CXpOZf4qIuZQfiJ8F3BgRV1D+SWyb+nxwZn6hLvuOzFw/IuZQ/uHsRuCJwHmUH5XPiHgJ8Nk67VfANpm5W7/6ZuZdEXE+sHkt483AfrWeC4F9gJ2AfwCeFxHvB15e3/6A9Xyw7SZJk8GYrWljmP9848PHijyAO3qMOwXYtr5+OnBqfb0xy/7y9U3Af9XXcynBdJ3G8JnAWsAmlMC9RrM8YA5wK7AF5azMWcBzgLWB64Ct63zHACf0qOOczvhar/OAR9bhhzXm+xjwtvr6SBr/7tVvPX348OFjqj6M2cbs6fywB1pTVkSsDzwL+HZEdEavVZ+3AL4ZEZtSegquarx1fmbe1Rj+QWbeDdwdETcAj6D8zWfTOZm5qJZ7PqU35A7gyszsLPsYSs9EL8+NiAuBxwGfzMw/1vFPjIiPARsB6wMnt1xPSVopGLM1nZhAaypbDViSmTv1mPZF4LOZOb9xOq/jzq557268vpfe232veaLHfP38PDN3i4jHAr+IiOMy83xKr8WemXlBROxL6fnoNtZ6StLKwpitacObCDVlZeZtwFUR8UqAKHaskzcEfl9f/9OIqnAZsE1EzKrDrxrvDZl5BfAJ4MA6agPgDxGxBvCaxqy312njrackrRSM2ZpOTKA1lawbEYsaj3+jBLA3RsQFwMXAHnXeuZTTZz+n3CwydPWU4luBkyLiF8CfKNfdjWce8DcRsTXwAeCXwI8pwb3jWOBd9WeUHk3/9ZSkqcqYbcyetjoX9EvqISLWz8w7olzodijw28z83GTXS5L0QMZsTRR7oKWxvbneoHIx5RTkf09udSRJYzBma0LYAy1JkiS1YA+0JEmS1IIJtCRJktSCCbQkSZLUggm0JEmS1IIJtCRJktTC/weIBmXeMnUZ7AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compare my Gradient Boosting Regressor with the one from sklearn by plotting side by side the errors each of them yields \n",
    "# on the test data\n",
    "\n",
    "fig, axs = plt.subplots(1,2, figsize = (12, 6))\n",
    "\n",
    "_ = sns.barplot(x = list(my_test_errors.keys()), y = list(my_test_errors.values()), ax = axs[0]);\n",
    "_ = sns.barplot(x = list(sklearn_test_errors.keys()), y = list(sklearn_test_errors.values()), ax = axs[1]);\n",
    "\n",
    "_ = axs[0].set_ylabel(\"RMSE\")\n",
    "\n",
    "for ax in axs:\n",
    "    _ = ax.set_xlabel(\"Learning Rate\")\n",
    "\n",
    "_ = axs[0].set_title(\"My Gradient Boosting Regressor\")    \n",
    "_ = axs[1].set_title(\"sklearn's Gradient Boosting Regressor\")    \n",
    "\n",
    "_ = fig.suptitle(\"Performance of Mine vs. sklearn's Gradient Boosting Regressor on the Test Set\", weight = \"bold\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}