grad_boost_clf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fbf974d8-a786-4ed4-bfe7-f277f2795107",
   "metadata": {},
   "source": [
    "# Gradient Boosting Classification from Scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e01e01b7-42db-4634-b119-d678e3934604",
   "metadata": {},
   "source": [
    "## Prepare the Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bf30bc7-9186-4d38-bb6d-186451d41ff2",
   "metadata": {},
   "source": [
    "### Loading the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "65c6160d-4fbb-49b4-9817-ba5820c1cc21",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "import pandas as pd\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "\n",
    "# Load the breast_cancer toy dataset\n",
    "cancer = datasets.load_breast_cancer(as_frame = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a306e2eb-92f4-4fbe-b13e-a71d1d91deee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension  ...  worst texture  worst perimeter  worst area  \\\n",
       "0                 0.07871  ...          17.33           184.60      2019.0   \n",
       "1                 0.05667  ...          23.41           158.80      1956.0   \n",
       "2                 0.05999  ...          25.53           152.50      1709.0   \n",
       "3                 0.09744  ...          26.50            98.87       567.7   \n",
       "4                 0.05883  ...          16.67           152.20      1575.0   \n",
       "\n",
       "   worst smoothness  worst compactness  worst concavity  worst concave points  \\\n",
       "0            0.1622             0.6656           0.7119                0.2654   \n",
       "1            0.1238             0.1866           0.2416                0.1860   \n",
       "2            0.1444             0.4245           0.4504                0.2430   \n",
       "3            0.2098             0.8663           0.6869                0.2575   \n",
       "4            0.1374             0.2050           0.4000                0.1625   \n",
       "\n",
       "   worst symmetry  worst fractal dimension  target  \n",
       "0          0.4601                  0.11890       0  \n",
       "1          0.2750                  0.08902       0  \n",
       "2          0.3613                  0.08758       0  \n",
       "3          0.6638                  0.17300       0  \n",
       "4          0.2364                  0.07678       0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Take a look at the dataframe\n",
    "data = cancer[\"frame\"]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "442e4bc7-a2fe-4f73-8ff7-6370ea1bcb35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".. _breast_cancer_dataset:\n",
      "\n",
      "Breast cancer wisconsin (diagnostic) dataset\n",
      "--------------------------------------------\n",
      "\n",
      "**Data Set Characteristics:**\n",
      "\n",
      "    :Number of Instances: 569\n",
      "\n",
      "    :Number of Attributes: 30 numeric, predictive attributes and the class\n",
      "\n",
      "    :Attribute Information:\n",
      "        - radius (mean of distances from center to points on the perimeter)\n",
      "        - texture (standard deviation of gray-scale values)\n",
      "        - perimeter\n",
      "        - area\n",
      "        - smoothness (local variation in radius lengths)\n",
      "        - compactness (perimeter^2 / area - 1.0)\n",
      "        - concavity (severity of concave portions of the contour)\n",
      "        - concave points (number of concave portions of the contour)\n",
      "        - symmetry\n",
      "        - fractal dimension (\"coastline approximation\" - 1)\n",
      "\n",
      "        The mean, standard error, and \"worst\" or largest (mean of the three\n",
      "        worst/largest values) of these features were computed for each image,\n",
      "        resulting in 30 features.  For instance, field 0 is Mean Radius, field\n",
      "        10 is Radius SE, field 20 is Worst Radius.\n",
      "\n",
      "        - class:\n",
      "                - WDBC-Malignant\n",
      "                - WDBC-Benign\n",
      "\n",
      "    :Summary Statistics:\n",
      "\n",
      "    ===================================== ====== ======\n",
      "                                           Min    Max\n",
      "    ===================================== ====== ======\n",
      "    radius (mean):                        6.981  28.11\n",
      "    texture (mean):                       9.71   39.28\n",
      "    perimeter (mean):                     43.79  188.5\n",
      "    area (mean):                          143.5  2501.0\n",
      "    smoothness (mean):                    0.053  0.163\n",
      "    compactness (mean):                   0.019  0.345\n",
      "    concavity (mean):                     0.0    0.427\n",
      "    concave points (mean):                0.0    0.201\n",
      "    symmetry (mean):                      0.106  0.304\n",
      "    fractal dimension (mean):             0.05   0.097\n",
      "    radius (standard error):              0.112  2.873\n",
      "    texture (standard error):             0.36   4.885\n",
      "    perimeter (standard error):           0.757  21.98\n",
      "    area (standard error):                6.802  542.2\n",
      "    smoothness (standard error):          0.002  0.031\n",
      "    compactness (standard error):         0.002  0.135\n",
      "    concavity (standard error):           0.0    0.396\n",
      "    concave points (standard error):      0.0    0.053\n",
      "    symmetry (standard error):            0.008  0.079\n",
      "    fractal dimension (standard error):   0.001  0.03\n",
      "    radius (worst):                       7.93   36.04\n",
      "    texture (worst):                      12.02  49.54\n",
      "    perimeter (worst):                    50.41  251.2\n",
      "    area (worst):                         185.2  4254.0\n",
      "    smoothness (worst):                   0.071  0.223\n",
      "    compactness (worst):                  0.027  1.058\n",
      "    concavity (worst):                    0.0    1.252\n",
      "    concave points (worst):               0.0    0.291\n",
      "    symmetry (worst):                     0.156  0.664\n",
      "    fractal dimension (worst):            0.055  0.208\n",
      "    ===================================== ====== ======\n",
      "\n",
      "    :Missing Attribute Values: None\n",
      "\n",
      "    :Class Distribution: 212 - Malignant, 357 - Benign\n",
      "\n",
      "    :Creator:  Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian\n",
      "\n",
      "    :Donor: Nick Street\n",
      "\n",
      "    :Date: November, 1995\n",
      "\n",
      "This is a copy of UCI ML Breast Cancer Wisconsin (Diagnostic) datasets.\n",
      "https://goo.gl/U2Uwz2\n",
      "\n",
      "Features are computed from a digitized image of a fine needle\n",
      "aspirate (FNA) of a breast mass.  They describe\n",
      "characteristics of the cell nuclei present in the image.\n",
      "\n",
      "Separating plane described above was obtained using\n",
      "Multisurface Method-Tree (MSM-T) [K. P. Bennett, \"Decision Tree\n",
      "Construction Via Linear Programming.\" Proceedings of the 4th\n",
      "Midwest Artificial Intelligence and Cognitive Science Society,\n",
      "pp. 97-101, 1992], a classification method which uses linear\n",
      "programming to construct a decision tree.  Relevant features\n",
      "were selected using an exhaustive search in the space of 1-4\n",
      "features and 1-3 separating planes.\n",
      "\n",
      "The actual linear program used to obtain the separating plane\n",
      "in the 3-dimensional space is that described in:\n",
      "[K. P. Bennett and O. L. Mangasarian: \"Robust Linear\n",
      "Programming Discrimination of Two Linearly Inseparable Sets\",\n",
      "Optimization Methods and Software 1, 1992, 23-34].\n",
      "\n",
      "This database is also available through the UW CS ftp server:\n",
      "\n",
      "ftp ftp.cs.wisc.edu\n",
      "cd math-prog/cpo-dataset/machine-learn/WDBC/\n",
      "\n",
      ".. topic:: References\n",
      "\n",
      "   - W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction \n",
      "     for breast tumor diagnosis. IS&T/SPIE 1993 International Symposium on \n",
      "     Electronic Imaging: Science and Technology, volume 1905, pages 861-870,\n",
      "     San Jose, CA, 1993.\n",
      "   - O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and \n",
      "     prognosis via linear programming. Operations Research, 43(4), pages 570-577, \n",
      "     July-August 1995.\n",
      "   - W.H. Wolberg, W.N. Street, and O.L. Mangasarian. Machine learning techniques\n",
      "     to diagnose breast cancer from fine-needle aspirates. Cancer Letters 77 (1994) \n",
      "     163-171.\n"
     ]
    }
   ],
   "source": [
    "# Print info about the dataframe\n",
    "print(cancer[\"DESCR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0276720a-2ed8-4089-aea8-40758f869e9b",
   "metadata": {},
   "source": [
    "### Selecting Features and Target Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "41b6dc54-c8ca-43e3-b9c2-e39cbe68186f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We will only use the following 5 features\n",
    "features = [\"mean radius\", \"mean texture\", \"mean perimeter\", \"mean area\", \"mean smoothness\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a38d8ace-697a-4da2-9dd2-ce0da4a04132",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate the data and target variable\n",
    "X = data[features].copy()   # We will try to use all features\n",
    "y = data[\"target\"].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "370bd855-7d19-4e0a-9732-d722dd23f1a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    357\n",
       "0    212\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check counts of the target classes\n",
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d5422be-8359-4009-b1f2-81a6b55ffe26",
   "metadata": {},
   "source": [
    "### Splitting Training and Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a06e457c-f556-4b48-8dd4-8573349a38e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Permute the dataframe to kill any possible ordering used\n",
    "np.random.seed(0)      # Fix a random seed to allow reproducibility\n",
    "old_index = data.index\n",
    "new_index = np.random.permutation(old_index)\n",
    "\n",
    "X = X.loc[new_index].reset_index(drop = True)\n",
    "y = y.loc[new_index].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "74d75c6b-82f9-4f17-88bd-81d46de09991",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use 80% of the data for training purposes and the remaining 20% for testing purposes\n",
    "train_test_cut = int(np.floor(len(X)*0.8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "410c690d-db8d-4ae7-b1ab-8aefb2895064",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the training set\n",
    "X_train = X.iloc[: train_test_cut].copy()\n",
    "y_train = y.iloc[: train_test_cut].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d2c3b06c-9816-4778-bbb0-006063cbe80b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the test set\n",
    "X_test = X.iloc[train_test_cut :].copy()\n",
    "y_test = y.iloc[train_test_cut :].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "507fe7a2-73e0-4e51-8ee6-6d81f32431c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The training set has 455 samples\n",
      "The test set has 114 samples\n"
     ]
    }
   ],
   "source": [
    "print(\"The training set has {} samples\".format(len(X_train)))\n",
    "print(\"The test set has {} samples\".format(len(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a26046e6-7d31-4d03-9750-4e8ae98fd255",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    287\n",
       "0    168\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "1    70\n",
       "0    44\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the value counts of the target in train and test sets\n",
    "y_train.value_counts()\n",
    "y_test.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a0a1127-b5d3-4983-85d4-07e243bcee64",
   "metadata": {},
   "source": [
    "## Implement a Gradient Boosting Regressor from Scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f52fe22-87cc-488d-aff3-34327568b9e4",
   "metadata": {},
   "source": [
    "### Pseudo Code of the Training Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f07e82e5-449a-495d-9fab-8e9bc7fa08f5",
   "metadata": {},
   "source": [
    "In the following we denote by $\\bf x$ the collection of training samples and by $\\bf y$ the corresponding target classes."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ea16666-2d01-4ab5-add4-9f75c11b4775",
   "metadata": {},
   "source": [
    "0. Create an empty list `fitted_gb_clf` to store the output data.\n",
    "1. Initialize the algorithm with the $\\log(odds)$ value of the variable $\\bf y$. In other words, for any sample $x_i$, set $G_0(x_i) = \\log{(M/N)}$, where $M = |\\{\\text{samples with label 1}\\}|$, $N = |\\{\\text{samples with label 0}\\}|$ and $G_k(x_i)$ is the predicted $\\log(odds)$ value of the sample $x_i$ at step $k$ (at this stage $k=0$).\n",
    "2. Append the value $\\log{(M/N)}$ to the list `fitted_gb_reg`.\n",
    "3. Transform the $\\log(odds)$ values using the logistic function $f(z) = \\dfrac{e^z}{1+e^z}$ to obtain the predicted probabilities $F_k(\\bf x)$. In other words, $F_k(x_i) = \\dfrac{e^{G_k(x_i)}}{1+e^{G_k(x_i)}}$ is the  probability that sample $x_i$ belongs to class 1.\n",
    "4. Calculate the residuals $\\bf r$, i.e., the difference between the true values and the predicted probability values. In other words, for any $i$, we set $r_i = y_i - F_k(x_i)$.\n",
    "5. Train a decision tree $T_k$ on the samples $\\bf x$ with target variable the residuals $\\bf r$.\n",
    "6. Calculate the leaves' output values using the following formula\n",
    "$$\n",
    "\\dfrac{\\sum \\text{Residual}_i}{\\sum \\text{Previous Probability}_i\\times (\\text{1 - Previous Probability}_i)}\n",
    "$$\n",
    "where the sums are taken over all the samples in the leaf.\n",
    "7. Append the fitted tree $T_k$ to the list `fitted_gb_clf`.\n",
    "8. Feed the samples $\\bf x$ to $T_k$ and let $T_k(\\bf x)$ be the outputted values.\n",
    "The new predicted $\\log(odds)$ values are $G_{k+1}(\\textbf {x}) =  G_{k}(\\textbf {x}) + \\alpha T_k(\\textbf {x})$, where $\\alpha$ is the *learning rate* (an hyperparameter fixed at the beginning). \n",
    "9. Increase $k$ by 1.\n",
    "10. If $k > \\text{n_estimators}$ (where $\\text{n_estimators}$ is a integer hyperparameter fixed at the beginning), return `fitted_gb_clf`; else go back to step 3."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b0994fe-3fd4-4a6a-937f-6e62c78552b4",
   "metadata": {},
   "source": [
    "The list `fitted_gb_clf` returned by this algorithm represents a fitted instance of the Gradient Boosting Regressor."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ae10f13-2e4c-4b88-b71b-f3e374902cdf",
   "metadata": {},
   "source": [
    "### Coding the Logistic Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a83e623b-0e9a-418a-b9ce-56e01bed80cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def logistic(x):\n",
    "    \"\"\"\n",
    "    Apply the logistic funtion to the input.\n",
    "    \n",
    "    :params x: the number to which to apply the function\n",
    "    :type x: float\n",
    "    \n",
    "    :return: the value of the logistic function applied to the input\n",
    "    \"\"\"\n",
    "    return (np.exp(x)) / (1 + np.exp(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e97a782-0e5d-4735-b9b9-304ea845bbc4",
   "metadata": {},
   "source": [
    "### Coding Step 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "18123295-ca2d-44b3-b422-5879c2d72d0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_leaf_output(residuals, probabilities):\n",
    "    \"\"\"\n",
    "    Compute the output of a leaf of a decision tree used by the Gradient Boosting Classifier.\n",
    "    The explicit formula for this value is given in the pseudo code.\n",
    "    \n",
    "    :param residuals: the residuals of the samples in the leaf\n",
    "    :type residuals: pandas.Series of shape (n_samples,)\n",
    "    :param probabilities: the predicted probabilities of the samples in the leaf\n",
    "    :type probabilities: pandas.Series of shape (n_samples,)\n",
    "    \n",
    "    :return: the output value of the leaf\n",
    "    \"\"\"\n",
    "    num = residuals.sum()\n",
    "    den = ((10**(-6) + probabilities).multiply(1 - probabilities + 10**(-6))).sum() # We add a smoothing parameter to avoid\n",
    "                                                                                    # getting division by zero\n",
    "\n",
    "    return num / den"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ec96e235-83d9-43d6-a800-1c405cb33590",
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "def all_leaves_outputs(X, tree, residuals, probabilities):\n",
    "    \"\"\"\n",
    "    Compute the output of all leaves of a decision tree used by the Gradient Boosting Classifier.\n",
    "    \n",
    "    :param X: the data on which the tree was fitted\n",
    "    :type X: pandas.DataFrame of shape (n_sample, n_features)\n",
    "    :param tree: the decision tree we want to use. It must have been already fitted to the data X\n",
    "    :type tree: sklearn.tree.DecisionTreeRegressor()\n",
    "    :param residuals: the residuals of the data X. We need them to calculate the values of the unpure leaves\n",
    "    :type residuals: pandas.Series of shape (n_samples,)\n",
    "    :param probabilities: the predicted probabilities of the data X. We need them to calculate the values of the unpure leaves\n",
    "    :type probabilities: pandas.Series of shape (n_samples,)\n",
    "    \n",
    "    :return: a dictionary with keys the indexes of each leaf in the decision tree and values the corresponding output value\n",
    "    \"\"\"\n",
    "    \n",
    "    # Get the structure of the tree showing in what nodes each sample goes\n",
    "    dec_paths = tree.decision_path(X)\n",
    "\n",
    "    # Create dictionary with keys the nodes indexes and values the indexes of the samples which passed from that node\n",
    "    nodes_samples_dict = collections.defaultdict(list)\n",
    "\n",
    "    for d, dec in enumerate(dec_paths):\n",
    "        for i in range(tree.tree_.node_count):\n",
    "            if dec.toarray()[0][i] == 1:\n",
    "                nodes_samples_dict[i].append(d)  \n",
    "    \n",
    "    # Get the indexes of the leaves\n",
    "    leaves = list(np.unique(tree.apply(X)))            \n",
    "                \n",
    "    # Create a dictionary where the keys are the leaves' indexes and the values are calculated by applying the  \n",
    "    # appropriate function to the samples in the leaf\n",
    "\n",
    "    leaves_values = dict()\n",
    "\n",
    "    for leaf in leaves:\n",
    "        samples_in_leaf = nodes_samples_dict[leaf]\n",
    "        leaves_values[leaf] = one_leaf_output(residuals.iloc[samples_in_leaf], probabilities.iloc[samples_in_leaf])\n",
    "        \n",
    "    return leaves_values    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f7bf706-b647-47ef-99d4-06eeeca1d7a3",
   "metadata": {},
   "source": [
    "### Coding the Rest of the Training Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "209bb444-09d1-4e67-9dbb-d7a7605e3435",
   "metadata": {},
   "source": [
    "0. Create an empty list `fitted_gb_clf` to store the output data.\n",
    "1. Initialize the algorithm with the $\\log(odds)$ value of the variable $\\bf y$. In other words, for any sample $x_i$, set $G_0(x_i) = \\log{(M/N)}$, where $M = |\\{\\text{samples with label 1}\\}|$, $N = |\\{\\text{samples with label 0}\\}|$ and $G_k(x_i)$ is the predicted $\\log(odds)$ value of the sample $x_i$ at step $k$ (at this stage $k=0$).\n",
    "2. Append the value $\\log{(M/N)}$ to the list `fitted_gb_reg`.\n",
    "3. Transform the $\\log(odds)$ values using the logistic function $f(z) = \\dfrac{e^z}{1+e^z}$ to obtain the predicted probabilities $F_k(\\bf x)$. In other words, $F_k(x_i) = \\dfrac{e^{G_k(x_i)}}{1+e^{G_k(x_i)}}$ is the  probability that sample $x_i$ belongs to class 1.\n",
    "4. Calculate the residuals $\\bf r$, i.e., the difference between the true values and the predicted probability values. In other words, for any $i$, we set $r_i = y_i - F_k(x_i)$.\n",
    "5. Train a decision tree $T_k$ on the samples $\\bf x$ with target variable the residuals $\\bf r$.\n",
    "6. Calculate the leaves' output values using the following formula\n",
    "$$\n",
    "\\dfrac{\\sum \\text{Residual}_i}{\\sum \\text{Previous Probability}_i\\times (\\text{1 - Previous Probability}_i)}\n",
    "$$\n",
    "where the sums are taken over all the samples in the leaf.\n",
    "7. Append the fitted tree $T_k$ to the list `fitted_gb_clf`.\n",
    "8. Feed the samples $\\bf x$ to $T_k$ and let $T_k(\\bf x)$ be the outputted values.\n",
    "The new predicted $\\log(odds)$ values are $G_{k+1}(\\textbf {x}) =  G_{k}(\\textbf {x}) + \\alpha T_k(\\textbf {x})$, where $\\alpha$ is the `learning rate` (an hyperparameter fixed at the beginning). \n",
    "9. Increase $k$ by 1.\n",
    "10. If $k >$ `n_estimators` (where `n_estimators` is a integer hyperparameter fixed at the beginning), return `fitted_gb_clf`; else go back to step 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fb21286c-0ee2-4df8-8074-e223fa3ea820",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "def grad_boost_fit(X, y, n_estimators = 100, learning_rate = 0.1): # We set the same hyperparameter default values used by\n",
    "                                                                   # sklearn's instance of Gradient Boosting Classifier\n",
    "    \"\"\"\n",
    "    Fit my Gradient Boosting Classifier.\n",
    "    \n",
    "    :param X: the samples we use to train the model\n",
    "    :type X: pandas.DataFrame of shape (n_samples, n_features)\n",
    "    :param y: the target values we are trying to predict\n",
    "    :type y: pandas.Series of shape (n_samples,)\n",
    "    :param n_estimators: the number of boosting stages to perform\n",
    "    :type n_estimators: int, default = 100\n",
    "    :param learning_rate: the learning rate to use in the algorithm. It must be in range (0, infty)\n",
    "    :type learning_rate: float, default = 0.1\n",
    "    \n",
    "    :return: gb_clf, a dictionary with the following attributes\n",
    "    \n",
    "                 fitted_gb_clf: a list containing the data which implements my Gradient Boosting Classifier.\n",
    "                                This data consists of pairs (tree, dict), where tree is the decision tree\n",
    "                                created at each boosting step and dict is a dictionary containing\n",
    "                                the leaves' output values.\n",
    "             \n",
    "                 n_estimators: the max number of boosting stages performed\n",
    "                 \n",
    "                 learning_rate: the learning rate used in the algorithm\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    # STEP 0: initialize a list which will represent a fitted instance of the Gradient Boosting Classifier\n",
    "    fitted_gb_clf = []\n",
    "    \n",
    "    # STEP 1: initialize the algorithm with the log(odds) of the target variable\n",
    "    odds = len(y[y == 1]) / len(y[y == 0])\n",
    "    log_odds_predictions = pd.Series([np.log(odds)] * len(y), index = y.index) # Using a series with same index of input data\n",
    "                                                                               # will make things more smooth later\n",
    "\n",
    "    # STEP 2: add this initial value to the list representing the fitted Gradient Boosting Classifier\n",
    "    fitted_gb_clf.append(np.log(odds))\n",
    "            \n",
    "    # Initialize a count of the number of boosting stages performed\n",
    "    n_estimators_trained = 0\n",
    "    \n",
    "    # test STEP 10 condition: keep boosting until we reach the max number of stages\n",
    "    while n_estimators_trained < n_estimators:\n",
    "        \n",
    "        # STEP 3: transform the log(odds) into probabilites\n",
    "        prob_predictions = log_odds_predictions.apply(logistic)\n",
    "        \n",
    "        # Add these probabilites to the DataFrame\n",
    "        #X.loc[:, \"predicted_probs\"] = prob_predictions.values\n",
    "        \n",
    "        # STEP 4: calculate the residuals\n",
    "        residuals = y - prob_predictions.values\n",
    "        \n",
    "        # Add these residuals to the DataFrame\n",
    "        #X.loc[:, \"residuals\"] = residuals.values\n",
    "        \n",
    "        # STEP 5: instantiate and fit a decision tree which predicts the residuals\n",
    "        tree = DecisionTreeRegressor(max_depth = 3, random_state = 1)\n",
    "        tree.fit(X, residuals)\n",
    "        \n",
    "        # STEP 6: calculate the output value of each leaf in the tree\n",
    "        leaves_values = all_leaves_outputs(X, tree, residuals, prob_predictions)\n",
    "        \n",
    "        # STEP 7: add the fitted tree (given as a dictionary containing the value of each leaf)\n",
    "        # to the list representing the fitted Gradient Boosting Classifier\n",
    "        fitted_gb_clf.append([tree, leaves_values])\n",
    "        \n",
    "        # STEP 8: eed the samples to the tree and use the outputs to update the log(odds) predictions\n",
    "        tree_outcomes = pd.Series(tree.apply(X)) # Get the leaf index each sample ends up in\n",
    "        predicted_residuals = tree_outcomes.map(leaves_values) # Calculate the output of each sample\n",
    "        log_odds_predictions += (learning_rate * predicted_residuals).values # Update the log(odds) predictions \n",
    "        \n",
    "        # STEP 9: increase the number of boosting stages performed\n",
    "        n_estimators_trained += 1\n",
    "        \n",
    "    \n",
    "    # exit STEP 10 condition: the list we created represents a fitted instance of the Gradient Boosted Classifier\n",
    "    gb_clf = {\"model\" : fitted_gb_clf,       # We return a dictionary containing the model and the hyperparameters we used\n",
    "              \"n_estimators\" : n_estimators,\n",
    "              \"learning_rate\" : learning_rate}\n",
    "                                      \n",
    "    return gb_clf                                 "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3eb325bc-79e1-4655-9d61-0b7abbf90f45",
   "metadata": {},
   "source": [
    "### Pseudo Code for Making Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dc17e75-2994-4fcf-aba5-851d069fa495",
   "metadata": {},
   "source": [
    "In the following `fitted_gb_clf` is the list representing the fitted Gradient Boosting Classifier we constructed above. Moreover,  we denote by $\\hat {\\bf x}$ the collection of new samples we are trying to classify and with $\\hat {\\bf G}, \\hat {\\bf F}, \\hat {\\bf y}$ the corresponding $\\log(\\text{odds})$, probabilities and target classes predicted by the model, respectively."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28dd5af7-a79b-462f-8f9a-141134ca88b4",
   "metadata": {},
   "source": [
    "1. Set the initial $\\log(\\text{odds})$ predictions to be the value we used to initialize the training algorithm. In other words, set $\\hat {\\bf G} = $ `fitted_gb_clf[0]`.\n",
    "2. Loop through the decision trees $T_k$ in `fitted_gb_clf` and:\n",
    "    1. Feed the samples $\\hat {\\bf x}$ to $T_k$ and get the outputted values $T_k(\\hat {\\bf x})$.\n",
    "    2. Set the new $\\log(\\text{odds})$ predicted values to be $\\hat {\\bf G} = \\hat {\\bf G} + \\alpha T_k(\\hat            {\\bf x})$, where $\\alpha$ is the `learning rate` that was established at the beginning of the training            algorithm. \n",
    "3. Calculate the predicted probabilities by applying logistic function to the log(odds) predictions. In other words, set $\\hat {\\bf F} = f(\\hat {\\bf G})$.  \n",
    "4. Convert the probabilites into classes labels using an appropriate cutoff value `prob_cutoff` (an hyperparameter in the range [0, 1]). In other words, we set\n",
    "$$\n",
    "\\hat {\\bf y} =   \\left\\{\n",
    "\\begin{array}{ll}\n",
    "      0, & \\text{if}\\,\\,\\, \\hat {\\bf F}\\leq \\text{prob_cutoff} \\\\\n",
    "      1, & \\text{if}\\,\\,\\, \\hat {\\bf F} >  \\text{prob_cutoff}.\n",
    "\\end{array} \n",
    "\\right.\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "193dfb93-f15a-4966-b1c3-2742a73c3714",
   "metadata": {},
   "source": [
    "### Coding the Making Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3cf053fb-5cc0-4f86-bf34-446007a85109",
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_boost_predict(X, gb_clf, prob_cutoff = 0.5):\n",
    "    \"\"\"\n",
    "    Use a fitted instance of my Gradient Boosting Classifier to make predictions.\n",
    "    \n",
    "    :param X: the samples we want to predict\n",
    "    :type X: pandas.DataFrame of shape (n_samples, n_features)\n",
    "    :param gb_clf: a fitted instance of my Gradient Boosting Classifier\n",
    "    :type gb_clf: list\n",
    "    :param learning_rate: the learning rate to use in the algorithm. It must be in range (0, +infty)\n",
    "    :type learning_rate: float, default = 0.1\n",
    "    :param prob_cutoff: the cutoff to use when transforming probabilities into labels. It must be in range (0, 1)\n",
    "    :type prob_cutoff: float, default = 0.5\n",
    "    \n",
    "    :return: output, a dictionary with the following keys\n",
    "    \n",
    "                 \"predictions\", the predicted classes\n",
    "                 \n",
    "                 \"n_estimators\", the number of boosting stages performed during the training phase\n",
    "                 \n",
    "                 \"learning_rate\", the learning rate used in the algorithm\n",
    "                 \n",
    "                 \"prob_cutoff\", the probability cutoff value used to determine the predicted classes\n",
    "                 \n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    # Extract the model and hyperparameters from the input data\n",
    "    fitted_gb_clf = gb_clf[\"model\"]\n",
    "    learning_rate = gb_clf[\"learning_rate\"]\n",
    "    n_estimators = gb_clf[\"n_estimators\"]\n",
    "    \n",
    "    # STEP 1: set the initial log(odds) predictions to be the value we used to initialize the training algorithm\n",
    "    log_odds_predictions = pd.Series([fitted_gb_clf[0]] * len(X))\n",
    "    \n",
    "    # Loop through all the boosting stages\n",
    "    for i in range(1, len(fitted_gb_clf)):\n",
    "        \n",
    "        # STEP 2A: feed the samples to the tree trained at the current stage and get the output values\n",
    "        current_tree = fitted_gb_clf[i][0] # Get the tree for the current boosting stage\n",
    "        current_nodes_values = fitted_gb_clf[i][1] # Get the output values of its nodes\n",
    "        tree_outcomes = pd.Series(current_tree.apply(X)) # Get the leaf index each sample ends up in\n",
    "        predicted_residuals = tree_outcomes.map(current_nodes_values) # Calculate the output of each sample\n",
    "        \n",
    "        # STEP 2B: update the model's log(odds) predictions \n",
    "        log_odds_predictions += (learning_rate * predicted_residuals).values\n",
    "        \n",
    "    # STEP 3: turn the log(odds) predictions into probabilities using the logistic function\n",
    "    prob_predictions = log_odds_predictions.apply(logistic)\n",
    "    \n",
    "    # STEP 4: predict the classes using the probabilites we computed and the given cutoff value\n",
    "    labels_predictions = prob_predictions.apply(lambda x : 1 if x > prob_cutoff else 0)\n",
    "    \n",
    "    # Equate the indexes of predicted and true classes to ease comparison\n",
    "    labels_predictions.index = X.index\n",
    "    \n",
    "    # Return a dataframe with the the predicted labels and the hyperparameters values\n",
    "    output = {\"predictions\" : labels_predictions,\n",
    "              \"n_estimators\" : n_estimators,\n",
    "              \"learning_rate\" : learning_rate,\n",
    "              \"prob_cutoff\" : prob_cutoff\n",
    "               }\n",
    "    \n",
    "    return output    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dea1f1c0-e2f6-4300-8477-7420cb5d63ac",
   "metadata": {},
   "source": [
    "## Evaluating my Gradient Boosting Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b2472996-610d-4883-bb1c-a5cb803ca16f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Initialize a dictionary to evaluate the performance of my Gradient Boosting Classifier on the test data\n",
    "my_test_accuracy_scores = dict()\n",
    "# Initialize the samples we want to predict (in this case, the test data)\n",
    "X_hat = X_test\n",
    "\n",
    "# Fit an instance of my Gradient Boosting Classifier and use it to predict the testing data at different learning rates\n",
    "# Evaluate the model using accuracy_score and collect all these values into a dictionary\n",
    "for alpha in [x/10 for x in range(1, 11)]:  # We use learning the rates 0.1, 0.2, 0.3, ..., 1\n",
    "    fitted_gb_clf = grad_boost_fit(X_train, y_train, learning_rate = alpha)\n",
    "    my_test_predictions = grad_boost_predict(X_hat, fitted_gb_clf)[\"predictions\"]\n",
    "    my_test_accuracy_scores[alpha] = (y_test == my_test_predictions).sum() / len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "24ea4df9-bbde-4441-b5cf-ab5e36b9d965",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAElCAYAAADp4+XfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAArmUlEQVR4nO3debgcZZn+8e9NIiiLgCRuYQlo2ERACIiiEnSUxQVQZBHUAMKggjD+ZHCFzDiOOI6CyhIYREQUlEUFDJtLQECEAAkQCBAgQERIgmwJm0me3x/vW5xKp/ucOkl3n5NT9+e6+jqnlq566u3qemp9WhGBmZnV10oDHYCZmQ0sJwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyKoSNJ6kv4k6XlJIelDAx3TYCfpBElzc3tdONDxdIOk8Xl5p+bu0bnb92lng6lNJE3OsYzv8HzG5fnMKvV7q6Qpkl7Kw7aQNCv/P66T8TQaUomg1IjFa56kKyWNbcPkvwKMA2YCPwAeaMM0hyxJ2wPHAqsDpwGXDWAsoySdJulBSS9KekLSzZK+0oXZP0NaX37QrglW3ZCWklLxmi9puqQj2xVLhVjPzvOeUOrd9jbpZf7rSDpR0v35s39c0u8kbdPpeTeYTVres0r9vgNsC0zJw+bl4T/I43fN8G7OrIsuAx4EdgI+AGwnadOImNPfCUl6RUT8E9g49zoxIs7q7T0VpzfUFe11c0R8bqCCkLQxcD0wApgLXAT8E9gS+CLw7RbvGx4RC5d3/hHxD+Do5Z3OcpoH/BwYCewP/FDStIi4diCC6VabSHotcCOwIfAP4ELSNu89+XVrp2MoRMRMll7m4jvy9Yj4Y/7/P5d3Xsu07kbEkHkBs4AA9szdI3J3AHvkflsAvwPm0LNhWL80jWL8o0nJ5AFgcql/8RoNrAZ8F7gfmA9MBT5ZmtaEPO6FwK+A54Hxpf6XA+cCzwE3AW8GzsjTuhPYpjStX5D2El4EngX+CLy1ybJ/GbgNWABMAtYujbMDcFVe7vmkL8mqVdqlSVuPBM4EHibt4d0I7JqHjW/SXhOaTGNcHvYU8O/Ak8BDpOT9uRzHo8Cn8vgH5PGvLE1jv9zvqhZxXp6HzwBe0zCsWft9DZgOLMr9v5eHvZA/pxuBcaX3vTG36QLgz8B/5OlMzcNHF21Qes/6wPnA3/KyXwVsUeWzLE+v4TW6ybIXn8PUUr9bcr/DSv32Am4mrVcPAacAa5WGbwlcQUooc4FLgU1Kw48mfQdeyMMnA5sAZzeJ8+wWbVIMPwK4N8dyLrByHi7SRnJObrdPlt6zdYvPfmIePgcYVer/SmDD/P/kPM743H0gcFee/0s5ls+V3rtN/pyfoed7+tnSZ30FaT1+HrgH+I+GdX1Ww2f88quh/7jcvSpwAulMxAJS8tqzFE/RxqcDV+eYxzVrj163nQO98W7nq9SIe5JOe32s1NA7Aq8n7Rm8BFxMOnII4G5glYYV8jngp3llOoK0EQ7Sl/Yk4DWkjXvkleWs/EEFsH+e1oTS9G4hbeR3LfVfTNpQzaBng3gzcEPuvq60bNeT9upOAa4p4m6y7EXcT+Tub+bhbyF9UQO4FvhxXrnWqtIuDe28EvCXPM6twM9Ie9mLcjtvn9spcrudRE4SDdMZV2qH24E/5O6nSQnmoty9AFiT9AX+B7AQeEOexi8ofZEbpv+qHFcAh1dcdxaSNtIX5P4XA7/M7f67PM5cYI2GDcndwDmlNm6aCEhf7Jmlz/6C3O5zgRF9fZak9e4setark/LrNU2WaXwp3pNIG9bFpPVsozzObnmcF/O87szdV+ThbyBt2CKvF8Xn+ndSYnpzaR4T8+fxQP5sP0HaqAYpgZ6U+y3RJg3fuydIG7fnc/chefhBufuFPPz+0ntaJYK/5eHf7uVzLz6/IhF8LS/naaT1+rk8/B15+HX07Nz9HykpnJmHnUvPNuJ04PeldhzHkongOFIyKaZ1UotEcB49249zcjsvLg0/u9QOk0nrxjatlrduiaDxdQlp43VM7r6Lni/QnNyv2Jst3nNwHyvMa0vjbpD7HZW7b4glE8H9wPDStIr+95H2dMbn7pdIG7y35u75pfeMAo4k7R38qDTvNzYs+zG5u9gzvSx3n5y7f1ua5rCq7dLQFtvnYc8Cq+V+J+Z+v2jYCE3u5fMaV1qOdVlyb3f3PM683L1d7i6W/Yukw/wnSRuNVzeZ/qjS9IrPd9dSv/IXrmi//2yYxmuAw4D/zu1SJPt35piL6ayXx/8evSeCj7NkgjyJlBiCnKwqfJZLTLOX9i0+g8bXn4GReZxJud/xuXsEPclzY9KRWgB/Kk33ttzvMGCz/P800pHcusW61bChmlB6/1Lxl2L7eO7+ae4+OXf/viHOt5Xe0yoR9LkTwNLf65WBvYHjSev0PXn4V/Pwv+bug0lH0a8oLesv87Cv5PheWRo2jlIiaPicxzXrRzrqDtIO1o9I60oR7/kN7XvN8mw7h/I1gpmkvYtbSFk5JI3OwzfLr7I3N3Rf38c8imk9HxEP5f9n5L8bNIx7UzQ/Zzcjx/VU7n48Ip6W9GzuXg1A0hjSnvfqTaYxknT6pHBb/ltMs3jPhvnvjcWIEbEoT79YlirtAj3L/khELCiWJf9tXPYq5kfEbElrlfrdUwwD1iG3Bel01BGkQ/ippCOaCyPimSbTLY4ehgPr5X6zSBfjPkv60jd6+XOXtA5wB2mvuNFI0hcU0jrwSP7/3ibjlo3Of0eRdhzKGtu61WfZX9MiYmtJq5Habz/ShcqDS/HcDRAR8yTNIx0lbtA4PJsBbE3aATpD0vHAF4ArASTdQ9qY3rkMsbZa5lENcdxVYVpzSKfu+rNOXkpKaI1G5r9fBE4ltaNI6+dxpKQxgbRz8E3SjsOLpA34Mf2Yf9no/Hcl0jpf1riu3LCM83h5BkPRjyPi3yLivyLi8mJXjLQRALg4IlS8SF/0HzdM48U+5lFM61WS1s//b5L/PtQwbqtpLeqju/BB0hfiDtKG73WlYWoYt0g40dD/wfz37S+/UVpJkuhfu1Aafz1Jq+b/Wy17Fc2Wu2lbRMQ0UnJ/G+kcOqTTEc3GfZ50ugngSElrRMSMiDiadBTRTPmzejepDeaSNoyr0LOBEunUA6R1oEg0G9O7WfnvLcBKpbZeG/hWw7itPsuX20ZS5e9wTtpTGuIs4tk0T28d0lEBpM9yieHZy5+1pGHAtyJiBGmD+508/N8aYq0aZ6tlLtp6TJN4WinuVDtEUpFIkLRy6TtLqf9a9CSBnUkxX14Mzn+nRMRWpM9rHOmI4ARJw4EHImJH0lH99qQdkS+V1o3+mpX/vkQ6givWlZVJ13XK+tpe9WqoHhG08nPgq8BHJV1Jaug3ke4uGkNPw/cpIubke+P3Bq6WdD2wTx58chtjBng8/x1D2pvdehmmMRH4DLCHpMmkPdd3A++g/+0yhXSI/Hbgz5Kmk+5GCdLeUqedSbrt7v2k6wmTehn3aNJe/luBuyUV1yFW6+U9haLdR5L2+DaitFeej2KuJd2BcpWkm4F9+5jmJNI59G2B6yXdTrp4PA7YnXToXyWul0gbhF9Ieigiju1l/FGSTiItc7GO/iX/PYV0neCrkjbKcQ0Hro6IeyWdS1o3dpZ0SZ7n23IMF5KOtP6a22EO6RoR9CTM4kjpQElrAr+hZ6ekP84F3gd8TdKbSetuX44HdiElqNslXU7aoL8b+D7pVEvZAtIe/uqkvfsn8zzLLs3J737SBn8V0pmHRcDpkjYhHTENJyXURXma/RYRcyX9ivSZ/VXS1aSj43eTvs8TlmW6zQzVI4KmIuJR0sbtMtLG9EDSIecppHPR/XUwaQOxMmkD8ABwUEQ03UNdDr8i7Zn/E/gXWtzy2JuIuJO0sfk96dzmAaSN6Ev9bZeIWAx8BPgJ6VrJXqRD+o9ExHX9jW0Z/IJ0EQ/goohouTcUETNId3oURzb7kza4t5A2cFN7ee9fSHvpT5KSznn07JkWDiC16Qakvezv9xZ43it/X57W+sCnSXvQ59JzOqxXEfES6RmNuaT17vN9vGUE6TTUp/Oy/IB0OoOI+B1pQzOdtFOzJulC5755+KOkveOrSBv5saSL5jtHug30GdIdbzsCh5JOxZwP/Fee9/+RTluMIp0+2rbKMjbxU9Ipl/mkjfsJpWFNP/+IeAzYDvghaV3fh7THfyfphonG8f9JaqOH8/ueIiW7ssmkZTyAdKR+M7BvPutwAymJ7JvndQ9wQEQ82c9lLTuEtKyLSdd8diQl8SuWY5pLUc9ZE7MVR9672xV4X/Tcg21DVN4Lf2VxTUrSO0gb3kWkGxaW69RI3dXt1JCt4CTtQEoAO5MOwf80sBFZl6wB3JlPlbxA2nMHON1JYPk5EdiKZlfSaY17gAPDh7R18SLpduuDSadiZ5FOXX5vAGMaMnxqyMys5mp1sdjMzJbmRDCAmpWmtcGpWcVPNSlNLmlYrrj5VO73vwMZ94quVEF18kDHMpQ5EXSJulD3vJRYWr1mtWHavU4jP6R2fKnsb1Hy+ZB+zKtZ6eJliblSm5faZ5GkZyXdJ+knksoPLTUrndysNPnHSBcyF5KeKh2QCp8FSRPysp3dx3hFWx3dncgqu4vUth39PYvS8oekxZLmSLpEqXpt1WmssEnLF4uHlqLmOaQHwj5EqgdUlM3+RxdiOIr0oMvDpOcMXk26d/wdNH9KeTA5n3S/9k6ke7b3kbRLRFwXzUsnL1WaXNJHc79JEfGFZQ1EQ7xcedXli4ibSM8pdMu1pCf4dwE+THqY8B1dnP/AWJ5CRUP1RS8llmPJQk8TSbVJniNVz2xV/GoySxf+mkCpEBWphsljpKczjym9dzipVsndpCcf7wIOrbAMexbTrjotWpTYZcnicC+/Wsz3kjx8r4b+ryn9vw7poaVZpER1PfDuhrYtv85uMa+WpblbtXmL6SxRvIxUIbSo8Doj9xtdXu4W028W+/g8/sGkwmzzSXe/fJVciJCe4nDXkapePlvESnpw76b8mTxEukumKB1efC6z8vTmUFp/WLL6bfGa3Mc6enSL4e/K4zxJqm11FrBOHvYKUgnkx0hPPD+V14P1mrTx0eTy7g39W5WfLtpmcpVlzuOsSSoA9wzpe/nF/J6nevm+LLH8pJ2BAJ4rjdOyRDXNC/zN6mt9z8OLKq3Pk3bW/gK8q6vbvG7ObEV40UeJ5TzO2aUP+9ekpzID+HOLaTYrY70rS5ZhvotU8rjoHpPf++3cb0ae78O5+9N9LMee5ZWxyrRoUWKXVODqwjzsmRz/SS3me3oeb15uu89RqpWf27eYz7Wk8hbP5i/BJrQoXdxiXi1Lc7dq8xbTWSIR5H7lKqVjWDoRNJv+J3LMQU8l1+2Bf839Hs7tXpQdPz5Pa3xpXvfkNvwUaa80SE8Q/4ye3xL4SX7fuNL7llp/8jI0xnNEizaYTItEQHoSvUi255XG/SOpZMMqpJ2Gn+bP4tY8/IombfxyefeG/q3KTxdtM7nKMudxisqls0lHpUW556d6+b68vPykHaYv5e4bS+O0LFFN89Lrx9H3+v4qUlJZQPqu/Yy0Pen1+9327d5Ab3gH24tqJZbPzt2/y9075+75vUy3WNHGl/oVK/VC4PW530O53975S/Zs7j4rr1zFHveNfSzHniy5V9LntOi9xO648vR6me+6pMfuy3tGi+jZw92OhoRCz4bjhIb2ndDHvPoqzb1Um7eYTrNEsFmp/440L53c7DOdQMNRDD07ChfS87sAATyWh48vtclapfcVv39QJJpT6dnorUof60+rePpYP49uMuyUYj0pfWbF7y5smscZQ9rz/h96NsQvkArrldu4sbx70b9V+emibSZX/M4MIyWtAHbKw/+N6omg/LqD/LsNeZy+SlQvEWuV9Z1UkmIRKXl8iJ7fiRjWze2erxEsbXT+W6XE8m3571P5b5VCZs08FqkuSjGt9UkryAh6ipwd1PCeZuWhe1NlWr2V2K0kImaTfhr0raTD6/1IG9LjJJ1MT/uuQd9lmFtS/0pzL4vyZ93vnzhtMDr//VhD/9dJKsc/PSKeavK+9+dXQaQCeIVW60+7FHG8nVL12uzNkkaSnvAe1jBsFdLn/HSpX6vy7o3fpb7i7+07U5QX70/J6sK1pKPZPYDNSbWVit8n76tEdTOj89+m63tEzJf0WVJyuRRA0mzSL7BN7kfcy8V3DS1tVv5bpcRyq5K5zfRWjrf8WwXlac0jHTICbBk9ZWhXIq2g/VFlWr2V2K1UTljSDpJWjYg7IuJk0l4OpI3XavS076Ok2jFFHKvSU3O9yryqlObubwnkYhlWJd0RBOkawX39eX8Ts/Lfj8SSZb43iohyZcrGUgnF+77Q8L43RSoiWGi1/sAytkGLOL7fJP7LSAluGKkQ2mosmSway6S3KgfRn+9SefzG98wjnWqB/pWsLvw6Ij5G+lnMlYATJa1SsUR1s7aelf/2tr7/NCJGkYrZHUU6qv5GP2Jebj4iWFqnSiwX5XiPkrQl6dxlryIiJJ1C+pWoqyVdStr47UA6Jz6+6swrTqu3ErtF/OtKOhO4LyK+02RWXwLem8tyz6an2uS9pHPkj5CuwbwDuFnSDaRa/zuRDuHPpknp4oj4U8N8qpTmXqrNI/2eQSvHSCruGlqPlDg/08v4VZ1MWnfOlfRrepLvHFLC7e19uwP/I+mdpPPKW5IuPm7Yy/vKijbYTdKPSKctLupl/KMl7VfqPpH0E6uHktpxI9LGdjPSr7StRM9n8XbSKbqdKsbWdhGxSNJ5pFt4z5P0e5Y+EqviRNJG+Y2k78ZZ9F2iumjrbSWdSjrK+TF9r++P51tOHyWVS4eeI6Pu6OZ5qBXlRSqtfBbpg32WdNfGh0rDz6Z0Dpu0EVri/HGTab6VdNdI8fN5e9P85+umUjrvTNor/3d67ip4nLTntVsfy7Bnk2n3Oi3SXRjFHUXz83K/r/T+79Lz+7V3tpjvx0m/VPUoae/vMdIFvY1L44wkXXCbRTqP/BDpIlpxvnkU6RRCca73S03mM4x0CusZUsLZl55zu1u3avMWMRfvW5yX+15Soi7/QPvoxs+Y6tcIRConPDVPfx7pFMSBefh4Gs4tN3yON5JOrzyVP5Oj87Aq689q+TMuLmye3KINimVpfBXzeg/p4vATpO/ENOC/87A1SDdNLCDdEXVQ6f1rNbTx6BZtPzp3n1Ruv8a2qbjMa5JKtz9Lumvoq3n44718X4rlP7rU77jcbyZpffsoaV1dQFo/fp6Hn5THXyn3Ky5OFz8t2tf6fgFpJ+lF0l1DlzW2U6dfrjVkZkOKpDVIN25E7v4K6acjr4uIKj9oUzs+NWRmQ837gK/n36xYh56bI344cCENbk4EZjbUPEw6lfP/SBeOpwHfi4gLBjSqQcynhszMas63j5qZ1VzHTg1JOot0D/mciNiiyXCRbvvbnXRHw/iIuLWv6Y4YMSJGjx7d5mjNzIa2W265ZV5ENH34rZPXCM4m3Qd9Tovhu5HuAR9Duv/4NJZ+anEpo0ePZsqUKW0K0cysHiQ1PhD7so6dGoqIa+m97PEewDmR3AisJekNnYrHzMyaG8hrBKPoeRIP0kNBo5qNKOkwSVMkTZk7d25XgjMzq4uBTASNNUigRZ2RiDgjIsZGxNiRI3ur72RmZv01kIlgNqmeS2Fdlr9ipJmZ9dNAJoJLgE8p2QF4OiL+PoDxmJnVUidvHz2PVCBqRK6vfTyp6BkRMRGYRLp1dCbp9tHGGvlmZtYFHUsEEbF/H8MD+Hyn5m9mZtX4yWIzs5pzIjAzqzlXHzWzWvjVBdt3bV77fPymrs2rHXxEYGZWcz4iGEJ2/NGOXZnP9Ude35X5mFl3OBG0wcP/+da+R2qT9Y+7o2vzMrP22+rCK7s2r2l771JpPJ8aMjOruRX+iGDbY1pVuW6vW777qa7MZ0V3zXt26tq8drr2mq7Ny2wo8xGBmVnNrfBHBGaD1bcO3Ltr8/rauRc27X/3t/7YtRg2+9p7Ww6bMGFC1+Lo5ryGCh8RmJnVnI8IbMg5+f9d2rV5HfG9D3dtXmad4iMCM7OacyIwM6s5JwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyIwM6s5JwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyIwM6s5JwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyIwM6s5JwIzs5pzIjAzq7mOJgJJu0q6R9JMSV9uMnxNSZdKmiZpuqSDOhmPmZktrWOJQNIw4BRgN2BzYH9JmzeM9nngrojYChgHfE/Syp2KyczMltbJI4LtgZkR8UBEvAScD+zRME4Aa0gSsDrwD2BhB2MyM7MGnUwEo4BHSt2zc7+yk4HNgEeBO4CjImJx44QkHSZpiqQpc+fO7VS8Zma11MlEoCb9oqF7F2Aq8EZga+BkSa9e6k0RZ0TE2IgYO3LkyHbHaWZWa51MBLOB9Urd65L2/MsOAi6OZCbwILBpB2MyM7MGnUwENwNjJG2YLwDvB1zSMM7DwPsAJL0O2AR4oIMxmZlZg+GdmnBELJR0BHAlMAw4KyKmSzo8D58IfBM4W9IdpFNJx0bEvE7FZGZmS+tYIgCIiEnApIZ+E0v/Pwp8oJMxmJlZ7/xksZlZzTkRmJnVnBOBmVnNORGYmdWcE4GZWc05EZiZ1ZwTgZlZzTkRmJnVnBOBmVnNORGYmdWcE4GZWc05EZiZ1ZwTgZlZzTkRmJnVnBOBmVnNORGYmdWcE4GZWc05EZiZ1ZwTgZlZzTkRmJnVnBOBmVnNVUoEkjaQ9C/5/1dJWqOzYZmZWbf0mQgkHQpcCJyee60L/KaDMZmZWRdVOSL4PLAj8AxARNwHvLaTQZmZWfdUSQQvRsRLRYek4UB0LiQzM+umKongGklfBV4l6f3ABcClnQ3LzMy6pUoiOBaYC9wB/CswCfh6J4MyM7PuGd7bQEkrAbdHxBbA/3UnJDMz66ZejwgiYjEwTdL6XYrHzMy6rNcjguwNwHRJNwELip4R8ZGORWVmZl1TJRH8R8ejMDOzAdNnIoiIayS9Dtgu97opIuZ0NiwzM+uWKk8W7wPcBHwc2Af4q6S9Ox2YmZl1R5VTQ18DtiuOAiSNBH5PKjthZmYruCrPEazUcCroiYrvQ9Kuku6RNFPSl1uMM07SVEnTJV1TZbpmZtY+VY4IrpB0JXBe7t4XuLyvN0kaBpwCvB+YDdws6ZKIuKs0zlrAqcCuEfGwJNcwMjPrsioXi4+R9FHgXYCAMyLi1xWmvT0wMyIeAJB0PrAHcFdpnE8AF0fEw3levghtZtZlfSYCSRsCkyLi4tz9KkmjI2JWH28dBTxS6p4NvL1hnI2BV0iaDKwB/CAizmkSw2HAYQDrr+9n28zM2qnKuf4LgMWl7kW5X1/UpF9j1dLhwLbAB4FdgG9I2nipN0WcERFjI2LsyJEjK8zazMyqqnKNYHi5DHVEvCRp5Qrvmw2sV+peF3i0yTjzImIBsEDStcBWwL0Vpm9mZm1Q5YhgrqSXy0lI2gOYV+F9NwNjJG2YE8d+wCUN4/wWeLek4ZJWJZ06urta6GZm1g5VjggOB34u6WTS6Z5HgE/19aaIWCjpCOBKYBhwVkRMl3R4Hj4xIu6WdAVwO+n005kRcecyLouZmS2DKncN3Q/sIGl1QBHxbNWJR8Qk0u8XlPtNbOj+LvDdqtM0M7P2qlJi4ihJryZVHj1R0q2SPtD50MzMrBuqXCM4OCKeAT5A+tH6g4ATOhqVmZl1TZVEUNwGujvwk4iYRvNbQ83MbAVUJRHcIukqUiK4UtIaLPlcgZmZrcCq3DV0CLA18EBEPCdpHdLpIQAkvSUipncoPjMz67Aqdw0tBm4tdT9BqkBa+BmwTftDMzOzbqhUTroPvl5gZrYCa0ciaKwfZGZmK5B2JAIzM1uBtSMRvNT3KGZmNlhVebL4IkkflNR03IjYof1hmZlZt1Q5IjiN9Eti90k6QdKmHY7JzMy6qM9EEBG/j4gDSLeIzgKulnSDpIMkvaLTAZqZWWdVukaQHyIbD3wGuA34ASkxXN2xyMzMrCuq/GbxxcCmpAfHPhwRf8+DfilpSieDMzOzzqtSYuLkiPhjswERMbbN8ZiZWZdVOTW0maS1ig5Ja0v6XOdCMjOzbqqSCA6NiKeKjoh4Eji0YxGZmVlXVUkEK0l6uZ6QpGHAyp0LyczMuqnKNYIrgV9JmkiqK3Q4cEVHozIzs66pkgiOBf4V+Cyp0uhVwJmdDMrMzLqn6u8RnJZfZmY2xFR5jmAM8G1gc+CVRf+I2KiDcZmZWZdUuVj8E9LRwEJgZ+Ac0sNlZmY2BFRJBK+KiD8AioiHImIC8N7OhmVmZt1S5WLxC7kE9X2SjgD+Bry2s2GZmVm3VDkiOBpYFfgCsC1wIPDpDsZkZmZd1OsRQX54bJ+IOAaYDxzUlajMzKxrej0iiIhFwLblJ4vNzGxoqXKN4Dbgt5IuABYUPSPi4o5FZWZmXVMlEbwGeIIl7xQKwInAzGwIqPJksa8LmJkNYVWeLP4J6QhgCRFxcEciMjOzrqpyauiy0v+vBPYCHu1MOGZm1m19PkcQEReVXj8H9gG2qDJxSbtKukfSTElf7mW87SQtkrR39dDNzKwdqjxQ1mgMsH5fI+VnEE4BdiMVrNtf0uYtxvsO6XcPzMysy6pcI3iWJa8RPEb6jYK+bA/MjIgH8nTOB/YA7moY70jgImC7KgGbmVl7VblraI1lnPYo4JFS92zg7eURJI0iXXN4L70kAkmHAYcBrL9+nwcjZmbWD32eGpK0l6Q1S91rSdqzwrSbPY3cePfRScCx+QnmliLijIgYGxFjR44cWWHWZmZWVZVrBMdHxNNFR0Q8BRxf4X2zgfVK3euy9N1GY4HzJc0C9gZOrZhkzMysTarcPtosWVR5383AGEkbkkpX7wd8ojxCRGxY/C/pbOCyiPhNhWmbmVmbVDkimCLp+5LeJGkjSScCt/T1pohYCBxBuhvobuBXETFd0uGSDl++sM3MrF2q7NkfCXwD+GXuvgr4epWJR8QkYFJDv4ktxh1fZZpmZtZeVe4aWgC0fBjMzMxWbFXuGrpa0lql7rUl+eEvM7Mhoso1ghH5TiEAIuJJ/JvFZmZDRpVEsFjSy09xSRpNk2qkZma2YqpysfhrwHWSrsnd7yE/5WtmZiu+KheLr5A0lrTxnwr8Fni+w3GZmVmXVCk69xngKNKTwVOBHYC/sORPV5qZ2QqqyjWCo0gF4R6KiJ2BtwFzOxqVmZl1TZVE8EJEvAAgaZWImAFs0tmwzMysW6pcLJ6dnyP4DXC1pCfxT1WamQ0ZVS4W75X/nSDpT8CawBUdjcrMzLqmyhHByyLimr7HMjOzFcmy/GaxmZkNIU4EZmY150RgZlZzTgRmZjXnRGBmVnNOBGZmNedEYGZWc04EZmY150RgZlZzTgRmZjXnRGBmVnNOBGZmNedEYGZWc04EZmY150RgZlZzTgRmZjXnRGBmVnNOBGZmNedEYGZWc04EZmY150RgZlZzTgRmZjXX0UQgaVdJ90iaKenLTYYfIOn2/LpB0ladjMfMzJbWsUQgaRhwCrAbsDmwv6TNG0Z7ENgpIrYEvgmc0al4zMysuU4eEWwPzIyIByLiJeB8YI/yCBFxQ0Q8mTtvBNbtYDxmZtZEJxPBKOCRUvfs3K+VQ4DLmw2QdJikKZKmzJ07t40hmplZJxOBmvSLpiNKO5MSwbHNhkfEGRExNiLGjhw5so0hmpnZ8A5OezawXql7XeDRxpEkbQmcCewWEU90MB4zM2uik0cENwNjJG0oaWVgP+CS8giS1gcuBj4ZEfd2MBYzM2uhY0cEEbFQ0hHAlcAw4KyImC7p8Dx8InAcsA5wqiSAhRExtlMxmZnZ0jp5aoiImARMaug3sfT/Z4DPdDIGMzPrnZ8sNjOrOScCM7OacyIwM6s5JwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyIwM6s5JwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyIwM6s5JwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyIwM6s5JwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyIwM6s5JwIzs5pzIjAzqzknAjOzmnMiMDOrOScCM7OacyIwM6s5JwIzs5rraCKQtKukeyTNlPTlJsMl6Yd5+O2StulkPGZmtrSOJQJJw4BTgN2AzYH9JW3eMNpuwJj8Ogw4rVPxmJlZc508ItgemBkRD0TES8D5wB4N4+wBnBPJjcBakt7QwZjMzKyBIqIzE5b2BnaNiM/k7k8Cb4+II0rjXAacEBHX5e4/AMdGxJSGaR1GOmIA2AS4ZznDGwHMW85pLK/BEAMMjjgGQwwwOOIYDDHA4IhjMMQAgyOOdsSwQUSMbDZg+HJOuDdq0q8x61QZh4g4AzijHUEBSJoSEWPbNb0VNYbBEsdgiGGwxDEYYhgscQyGGAZLHJ2OoZOnhmYD65W61wUeXYZxzMysgzqZCG4GxkjaUNLKwH7AJQ3jXAJ8Kt89tAPwdET8vYMxmZlZg46dGoqIhZKOAK4EhgFnRcR0SYfn4ROBScDuwEzgOeCgTsXToG2nmZbDYIgBBkccgyEGGBxxDIYYYHDEMRhigMERR0dj6NjFYjMzWzH4yWIzs5pzIjAzq7khnQgqlLjYVNJfJL0o6UsDFMMBubzG7ZJukLTVAMWxR45hqqQpkt7V7RhK420naVF+FqWrMUgaJ+np3A5TJR3X7hiqxFGKZaqk6ZKu6XYMko4ptcOd+TN5zQDEsaakSyVNy23R9muJFWJYW9Kv83fkJklbdCCGsyTNkXRni+GdK8kTEUPyRbpAfT+wEbAyMA3YvGGc1wLbAd8CvjRAMbwTWDv/vxvw1wGKY3V6rhltCczodgyl8f5IupFg7wFoh3HAZYNg3VwLuAtYv1hXB+LzKI3/YeCPA9QWXwW+k/8fCfwDWLnLMXwXOD7/vynwhw60xXuAbYA7WwzfHbic9PzVDu3cVgzlI4I+S1xExJyIuBn45wDGcENEPJk7byQ9SzEQccyPvLYBq9Hkwb5Ox5AdCVwEzGnz/PsTQ6dVieMTwMUR8TCkdXUAYijbHzivzTFUjSOANSSJtMPyD2Bhl2PYHPgDQETMAEZLel0bYyAiriUtWysdK8kzlBPBKOCRUvfs3G8wx3AIKeMPSByS9pI0A/gdcHC3Y5A0CtgLmNjmeVeOIXtHPg1xuaS3DFAcGwNrS5os6RZJnxqAGACQtCqwKylBt1uVOE4GNiM9bHoHcFRELO5yDNOAjwJI2h7YgM7stPWmY9u0oZwIKpWvGCwxSNqZlAiOHag4IuLXEbEpsCfwzQGI4SRSralFbZ53f2K4lVSTZSvgR8BvBiiO4cC2wAeBXYBvSNq4yzEUPgxcHxG97a12Mo5dgKnAG4GtgZMlvbrLMZxASsxTSUett9Heo5IqOrZN62StoYE2GMpXVIpB0pbAmcBuEfHEQMVRiIhrJb1J0oiIaFexrSoxjAXOT2cAGAHsLmlhRPymWzFExDOl/ydJOrXN7VApjjzOvIhYACyQdC2wFXBvF2Mo7EdnTgtVjeMgUnHKAGZKepB0nv6mbsWQ14uDIF20BR7Mr27q3Dat3Rc8BsuLlOQeADak5wLQW1qMO4HOXCzuMwZgfdKT1e8cyLYA3kzPxeJtgL8V3d3+PPL4Z9P+i8VV2uH1pXbYHni4ne3Qjzg2I52THg6sCtwJbNHtzwNYk3TeerUBXDdPAybk/1+X180RXY5hLfIFauBQ0rn6TrTHaFpfLP4gS14svqld8x2yRwRRocSFpNcDU4BXA4slHU26W+CZVtNtdwzAccA6wKl5T3hhtLnKYMU4Pkaq+/RP4Hlg38hrXxdj6KiKMewNfFbSQlI77NfOdqgaR0TcLekK4HZgMXBmRDS9rbBTMeRR9wKuinRk0nYV4/gmcLakO0gbwWOjjUdoFWPYDDhH0iLS3VyHtGv+BUnnke5aGyFpNnA88IpSDB0ryeMSE2ZmNTeULxabmVkFTgRmZjXnRGBmVnNOBGZmNedEYGZWc04ENiRImt/l+d3QpukU1U5vkzRD0v9WeM+ekjZvx/zNwInArClJvT5jExHvbOPs/hwRbwPeBnxI0o59jL8nqQiaWVsM2QfKzCS9CTiFVLr4OeDQiJgh6cPA10lPkT4BHBARj0uaQKpnMxqYJ+le0pPfG+W/J0XED/O050fE6pLGkZ5MnwdsAdwCHBgRIWl34Pt52K3ARhHxoVbxRsTzuZbNqDyPQ4HDcpwzgU+Sau18BNhJ0tdJDwLSbDmXtd2sfnxEYEPZGcCREbEt8CXg1Nz/OmCHvBd+PvDvpfdsC+wREZ/I3ZuSip5tDxwv6RVN5vM24GjSXvpGwI6SXgmcTqof9S7SRrpXktYGxgDX5l4XR8R2kQrg3Q0cEhE3AJcAx0TE1hFxfy/LaVaJjwhsSJK0OulHfy7IpTsAVsl/1wV+mWu5r8ySxcMuiYjnS92/i4gXgRclzSHVupndMLubImJ2nu9U0hHFfOCBiCimfR5p776Zd0u6HdiEVFztsdx/C0n/RapzszqpBEJ/ltOsEicCG6pWAp6KiK2bDPsR8P2IuKR0aqfQWFPnxdL/i2j+nWk2TrOSwa38OSI+lMtMXyfp1xExlVR4b8+ImCZpPKkOTaPeltOsEp8asiEpFw58UNLH4eXfey1+D3pNUgVLgE93KIQZwEaSRufufft6Q0TcC3ybnt+kWAP4ez4ddUBp1GfzsL6W06wSJwIbKlaVNLv0+iJp43mIpGnAdHp+fnAC6VTKn0kXctsun176HHCFpOuAx4GnK7x1IvAeSRsC3wD+ClxNSiyF84Fj8i2nb6L1cppV4uqjZh0iafWImJ9/yOQU4L6IOHGg4zJr5CMCs845NF88nk46HXX6wIZj1pyPCMzMas5HBGZmNedEYGZWc04EZmY150RgZlZzTgRmZjX3/wHD9nymkhwlNQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Plot the scores we obtain\n",
    "sns.barplot(x = list(my_test_accuracy_scores.keys()), y = list(my_test_accuracy_scores.values()))\n",
    "plt.xlabel(\"Learning Rate\");\n",
    "plt.ylabel(\"accuracy_score\");\n",
    "plt.title(\"Performance of my Gradient Boosting Classifier\\n on the Test Set at Different Learning Rates\", weight = \"bold\");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ba471d0-7950-4639-a9e2-fbcf357d4041",
   "metadata": {},
   "source": [
    "## Comparing my Model to sklearn's Implementation of Gradient Boosting Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "463c3309-4892-4e20-9c5a-ffff740f5591",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=0.2, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=0.3, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=0.4, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=0.5, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=0.6, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=0.7, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=0.8, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=0.9, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(learning_rate=1.0, random_state=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "# Now do the same thing for both the test and training data using the sklearn's Gradient Boosting Classifier\n",
    "\n",
    "sklearn_test_accuracy_scores = dict()\n",
    "\n",
    "for alpha in [x / 10 for x in range(1, 11)]:\n",
    "    gb_clf = GradientBoostingClassifier(learning_rate = alpha, random_state = 1)\n",
    "    gb_clf.fit(X_train, y_train)\n",
    "    \n",
    "    sklearn_test_predictions = gb_clf.predict(X_hat) \n",
    "    sklearn_test_accuracy_scores[alpha] = (y_test == sklearn_test_predictions).sum() / len(y_test)\n",
    "    #sklearn_test_accuracy_scores[alpha] = accuracy_score(y_test, sklearn_test_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "507f7e35-6a56-4f38-b376-f236c3aa5486",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAGeCAYAAAC97TYdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA+GklEQVR4nO3dd7gkVZn48e9LTgooIytxQFFEV1wdEPMYVgEDmAUTiCKrmNaAaxzTD1xXZVdQRBZZDCAqKiASlQzCkEHJcUBlCEMUEHh/f5zTMzVN971dM7fvvTP3+3mefrordJ1T1dVvvX3qVHVkJpIkSZIGs8xEV0CSJElakphAS5IkSS2YQEuSJEktmEBLkiRJLZhAS5IkSS2YQEuSJEktmEBPYRGxfkT8ISL+HhEZEa+Z6DpNdhGxV0TMrdvrF2OwvJl1WdeNQfUmnYjYqa7fSYsyXb1FxPS63TIi1qjjrqvDMye0cpPIZNkmETGr1uOgcSirs19Mr8OPjYjfRMTddfzuEXFQfT1r2PUZlqVhHbRkM4Ge5BoHgM7j1og4NiJmjMHi/wOYCVwF/DdwzRgsc6kVEVsCewCrAd8DjuoxTzOxmRcRqzamfb4x7aA6eg5l2x84/DXQSBo/ZqYv4vtXiojPRsRFEXFfTVgui4h9ImLNMa5uLwdS9qU5Y7XAQRLQrn0+I+KBiLi+rvdKY1WXUerZ74fYmG+TPuUvGxEfi4jz62d/Z0ScFRFvHWa5ffx3fdxVh3cDXgfcBvwPcAFwXJ3nrAmoXyvN/WvI5RzUtR93P2aNwbJHXEZEbBwRh0fELRFxf0TMiYhjIuJJLcpa6AeUhme5ia6ABnYUcC3wEuCVwBYRsWlm3tJ2QRGxfGb+A3hKHfXtzFzkBK6xvKVdZ3udk5kfGGD+1YEdgAMiYlngfd0zZOZVwEfHrIZ6lPHYPyNiZeAPwHOBB4FjgFuAJ1MSmP2AO/q8d7nMfGhx65CZX17cZYyB/erzDsAHKdtgwuo1HtskIpYBfgW8FngIOBq4nbIv7AD8bNh1aMrMj3aN6sStgzPzC43xP12ccpbCuH8cMK++fi2wMfBHFvzIGI8fG78Cngn8HrgCWA94MfBE4OpxKF9tZKaPSfwArgMS2L4Or1WHE9iujnsG8FvKwWou8Etgg8YyOvN/lJKEXwOc1BjfeUwHVgW+Qfmy3kNprXhnY1mz6ry/AA4D/g7s1Bj/O+DHwH3A2ZQEYv+6rEuAZzeW9VNKy9ADwN2UoPHPPdb908D5wL2Ug9OajXm2ogS+ubWMs4BVBtkuPbb1NOAA4AZK681ZwNZ12k49ttesHsuY3ph+O3BuHf+6xrgEDqrjZ9bh63q8/z21LndQfuQ0y3kPcGFd5yuBzwDL9ahP1M88u7b9NXXcjFrmMbWcvwOXA19qsY8+Gzi1brPO5/xvXdvtpDq8PSXJuBXYvHt6neeFlP3zDuBmSivi4+u05YHjgb9SEtV5wBHA+iPt713jd6ccnO6m7KsrdH0W0+vwjsCf6ja5HTgTeGGfbbBHfe8Dze1cp20ArF5fH1Tn+35djwdrue+oZd1dx10BfKCxjBUoZz3uoJwx2rWxPmt0fV9m1uFVgL3q/PcC51HjSFdd9gOOpHxnLwKe1bW85mOnUfb5Tl2+WYd/2pjvmZT97FbK9/FI4KmDfP/q9H8Fzq3rcmddnzfQ+7t5XZ9tclId3hM4pa7z6cCGjXLeULfZXcC3gJPrez7a57N/W6PcV3VN27Qrbh7U2BZn1c/zH8BfgH1YsC+uCfy8bqv7Kfvx9xv7wg8o34EHgBuBI3rs/9Mbn3HzMbMxflbjfX1jSmMbn0bZD++mR/yr8764btt5lO/vT4B1etSv5/dwhH2r+1jVWYfvAIfTtf82vnuHAjfV+hwHPGOAmPbrHttnxGVR4s3V9fOaS9nXntrnMzioR5mPq9PuAKIxfkXqMW2A+NhrW80cNJb7aPeY8Ar4GOUDaiTQlC43b2x8MV4A/BPl4P5gDSJH1Wl/Blasy+jMfx/wf5QD5u6U5DVrINi7foEPq+OuqF/Me+vwDnVZsxrLO5eSHG/dGP8IJYm+rA7PA84BzqjDpzXW7XRKcN2XBQepP/dY9069b6vDX6nTn16DVVIC9v9SDnxrDLJdurbzMpQEKSkH5h9RDmwP1+28Zd1OWbfb3jQO7o3lTG9sn2/X5y3rNrmLkiA0D6Qz6Z9A31C3z8N1+OV1nvc3ph/U2NZf7LMPfaVO/3od3qIO/6kO/5gF+8H3gROAY1rso6ex4EfVDyjJ9AF12k512kmUBOh+avLcPb0OP4MFP6gOYUHC83vKj4EVKQn6/1H2m/Pq9GMa9XnU/t41/ra63f5eh3fpsU4rU/ade+tn9iPgUuDdfbbB6XVZh46yrQ5q1OMkynfs2cBnKfvo92pZ99V5nlff96VG3Q+kHDg7y1mj6/sysw4fwoLv6cGUg/ojjenNuvyqrl8Cp9bpX6Dss53Pdm9gy1H2+e/Vx+21rDfXeZ5IOeBnXc/Od+kvlGRxxO9fXcYcyo+vg+s2mF3r2Ou7+YU+2+SkOvww5bt1Qx3+UZ3+5FpuAr+pdep8/z7a5zP9SZ1+5gif+ywW/t6/nBKzDqB85zqx+NN1+lc7y6zb82jg4jrtvXXaJcB3KcnebT32/+ks+BGYlIR977qOnc9+1iAxhYV/pFxe6/yuHuv5TMr35hFKstn5TC8Gll+E7+Hj6mfdec/eLDhWddbhEXrvv6tQjgedY9LPa93mAmuN8j39ddf2GXFZdZtmHd6P0jh0DSW+9/oMduxR5vKUuJeUhqtvUY77qzbmGS0+7t3YVgd2Pu82OYePwR8TXgEfo3xAvVuBktLqtgzwyTr8p0ZwuaWO67Sedt7znq5ld758O9XhJzTm3bCO+0gdPqMOz6rDV9No8WyMv7J+kXeqww9SujL8cx2+p/GedYEPUVrJvtMoe52udf9kHe4kEUfV4X3q8G8ay1x20O3StS22rNPu7gQsFiTAP63DnXU6aYTPa3pjPTajHIxPoByE920EuIPq/DPpn0BvUcd1flx8og53DhSdpKaTAP+1T502qdOvrcPfYOGD9c/q8H8A/wKsBCzbYh/9Y33/eygBfvnO+xvbbA4lGZ2fPPfapnUbNQ80e7PgR9KmjfX5d+A/KQly1nmWGWV/74zvJHWd9+7TY51Wq5/ZHOA1wMad/avPNriyLmuvOrxSo7zmgfigOnxy1/tXAN4EfJGy311e5/tMnX5VHX5nHX5tY9lrdH1fZlJac7Ouw3fqdjyJRpLfqMtv6/BLefR3dP4yB9znm48bqAk38Kk67g+N951fx+3KYN+/v1FaR99EadlbhkfvZyd11W2h+je2wb51eOc6fEkd/lyznvVz6cSNj/ZZ/+Ob27XPPLNofO/ruBdTvnPfAk6s04+r077OgoRxS8r+2FnXf6vTDqH8uF+Txn7Z2P7Tuz7nWY15FhrHKDGlsX3vou5vfdbzu3W+H9bh5evnlsAr234Pu/evrvGddei5/wJvZkHs2bs+Ot+j3UaJab/u2j4jLgt4Wn19IaWL5XrNeNHrM+hT7lsojU7N79FfWXAsGCQ+LvT5+xjewz7QS46jKF/Y2ygtSsdkZvNCgafVR9OTu4ZPH6WMzrL+npnX19eX1ecNu+Y9O3v327ys1mteHf5bZt4ZEXfX4VUBImITSkvTaj2WMY3SwtZxfn3uLLPzno3q8/y+aZn5cF1+Z10G2S6wYN1vzMx7O+tSn7vXfVA3U37ovKEO7wfs0uL9/dZ7en1+Y9f8a0fEapl5T3NkZl4ZEWcCz4uI51ISkEcoB0koB/f1KC3V/4/SwvEdyo+QQfw75cB5AOXH0z2UlsFvN+ZZtz4fRznN2k9n3Z5bH01PjohplL7Gy3ZNWxF4DOXUfke//b3fdp0vM++JiH+jJLRHAkTEHOCdlCSsW6e/8/p1+CHKRVqdvpTdzugaPpJy4O02rT53tt/l9fmKHvM2Ta/Py1DONjV17//d22NVFt2alM//A5T1P5Sy/p36/Lkx72XAsyjfr07ZI33/3k/58ffzOnwbZd0OXYR69tsHOtv5zwCZ+WBEXM2Cz6GXznUoA8eJiPgPynetW6ecvSldnD5AacR4GPhZRLyT0gI/E9iOBd1HToiI1ze2XVvT63PPmNIYvjQz5w2wnM72+0dEXENpnOnePqN+DwfUb//t1GVdyjZs6nUMGMmIy8rM/SLii8CHgWMBIuJySqy9ZNBCMvOwiDiCcq3TiyjXzawNfJ7SDbBTj57xkQXfGY0D78Kx5PjfzPxYZn41M3+X9acmpYUF4PDMjM6Dcsr0f7uW8cAoZXSWtXJEbFBfP7U+X981b79lPTzKcMerKQHzYkqXi7Ub06Jr3k6inl3jr63P8wNJRCwTEUG77UJj/vUjYpX6ut+6t/G9+nxaZl7c5o2NHyjd631dfX5d17pt3J08Nxxcn79BCcK/z8zOnQmuycwXUM4UbEk5/f6JiFgfICI2rY8V+yx7dmZuTkmeZlJanfaKiOYP9PMpn/X2LJxYd+us27d6rNtRlAP8spS+tKuy8EGke7/pt4/2267d/i8z1wXWoRw016McyHo5sj6/ISKelpkPZbmYq99nPr9uUW5D10meX0qJy7/rTK7PN9Xnzj7ZuTCsn+vq84PAtMZ2XAF4fde8I22Pzvd34GNF3W87P16mR8QKjfps2pi1+f3qTB/p+/e7zNyEcsr8TcDjga8tYj37rXNnO28C5UI5ev8AaurcjWeriFjoR1BtKOilc3eOL1Au5t+j85b6fHtmbk35Ubg5pYV4R0qL80OZ+VbgsZTGgRMo3aM6P9QXxXX1ebSYMugxZFN41PbrjqODfg/nH0PqBZvdRouT51LOTnXWaU0W7DeDGnFZ9SLxr2XmWpQfCl+n7L8f61qHvvtnRCwfES/MzPsz89jM/Bylrz6U/aBZj37xEUrjyIhlaWzYAr3k+wnlYo83RMSxlC/Ykyi/YDdhwRduVJl5S5R7G78JOD4iTqecUoLSXWIs/a0+b0JpqXrWIixjP0p/wO3q7auuoPxqfx7tt8tsSleE5wKnRsSllCvok9K6uqhOBF7BgmR/LOxDqdOPI+JXlEA5g9ISNrPPe35GadV6UR0+uDHtuxHxVErrxXKUBOVhSksiLGg1/BdK37xuR9YDyNWUJHxFSutg88fTXZQL5c4CPhIRczOz10Fsf0qry0ciYmNKl4+nAc+v69nZb55LaSV/SZ/1HQt/q/vVzZQuSLCglavb/1CS+xnA7Ig4mrINthqgnHsp23o1ytmAOyh9ZJt+Skne9663lNt2pAVm5tyIOIzy/f1jRBxPSThfRPnezBqgXlAuUNsY+HJEvA74ZmbeOML8e0bEw5TrIqBcRPtgRPyY8n18aW1hW4GyP/2N0m1gHqN//86Pcr/0G1jQ0j+vUU+A50TEd4HzM/MHA65j0yGUsw6vqN+tJ1C+DyP5GWXf3hb4bf3s5wLPoSSN2/d4T2c/fgdl+3bP8+m6vS+m/AiaXsffCewQEXtQYtY9jL5vDmJRYkovne/vu+udaTakbMNL6X3mZhB/o2yDFYCfRsT1mbnHKO+B0m/8GsrncHpEXES5EHAm5bNqU5/RlnUd5Xt2CmWbvaC+b1597uyf74iI1YFfZ+YfuspYkbLv/5nS4HAfC37sHl+fR4uPnbI2BPaJiCuAzy7GmQmNwF8oS7jMvJmSRBxFSULfQTnNtC/ly9XWeygthCtQWkmuAXbOzMW65VEPh1Fagv9BSTD3HHn2R8vMSygB7ARK39u3Uw4wD7bdLpn5COUU2Q8pAf/1lCD2usw8rW3dGsvNzDwxM8fyHtudHw7XUn7sbEtZpwNGqMcdLGglvYdyYWXHGZTk7a2UhOty4O31PYM4idJK+3bKmYVzgLc2zpJ06jCn1vUu4KsRsWuPel5I2R9OofQRfRul9WWvOss+lP6JK9bpbVuS2jiecoHfLpQLVn8LfLzXjJl5H2V/+zIlwXsdpe63Un64HNKvkCy3Ant3fd8WlINu95/0fI1y8FwWeBm9T/9324Wy3R6h9GF9AeWirmMGeG/HLErXsedRWuHXHnHu0h/0A5T96ZfUltb6fXwppQvPCyjJ2W+Bl2bm7QN+/06gtOq9mwV3InhvnXYK5UfGw5Q+wtu1WMf5MvPqWuerKT9izqbsz9Cn9bXWfTvgE5RE8ZWU79GD9O9e8jFKa+aGlB/23+qafh6lZXV74F2UJPLDmXkR5ft5K+W7tEst56v0uC99C61jSi+ZeQFl/c+sy9iIsg22zswHF6Vi9X17UH6UvJVye8RB3ncv5TM8hJLsvpuy//yYBV2hBq3DaMu6i7KvvICS4K5DWe+v1kX8gBJn16V083hOj2Lupxx7H6Bsu3dSYsFXKNd7DBIfoWyrOZQfsR+hXBCtIYiuY5wkSVNWRKyemXfW16tSWvTWBF6RmSdOaOUkTRp24ZAkaYHf1dPoN1DuwLIm5e4Kp0xorSRNKibQkiQtMJvSBWNNSv/3Ayj3lV6a/nVP0mKyC4ckSZLUghcRSpIkSS2YQEuSJEktmEBLkiRJLZhAS5IkSS2YQEuSJEktmEBLkiRJLZhAS5IkSS2YQEuSJEktmEBLkiRJLZhAS5IkSS2YQEuSJEktmEBLkiRJLZhAS5IkSS2YQGupEREHRcRX6+sXRcTlE12n8RQRb4+I4ya6HpIWXUTMiogft502lUXEThFxWmP4nojYeCLrNJ4iYoO6zstOdF2mEhNoLSQirouIByNira7xF0RERsT0RVzujIg4KiLuiIh5EfGniPhaRKw5JhXvkpmnZuZTx2JZdZu8YoTpMyPikRrA7omImyLiS2NR9ghlTq+fx3KdcZn5k8x85TDLlTQ11Tg4fRHe98SI+EFE3Fzj4zW1sWPTIVQTgMxcLTOvWdzlNBtlRpgnI+Leum63RsQhEbHG4pY9SpkLHZMy84a6zg8Ps1wtzARavVwL7NAZiIh/BlZe1IVFxPOBk4DTgU0zcw1ga+AhYPM+71mu1/hJ7OYawFYDXgjsEhHbT3CdJGlUw4q3EfF44AxgFeBFwGOAZwMnA/86nnUZss1r7N8YWBOYNbHV0XgwgVYvPwLe1Rh+N3BwZyAitoiIvzUDXUS8MSIu6LO8/wR+mJl7ZubfYP4v5i9m5kn1/TtFxOkR8e2IuB2YFRFPiojfR8Rt9Zf9T5q/7CPiXyLivIi4OyJ+BqzUmDYzIuY0hteJiF9GxNyIuDYiPtyYNisiDouIg+uyLo2IGXXaj4ANgCNrC8OnRtt4mXkt5aCxWaOM50fEORFxZ31+flfdjoiI2yPiqoh4X2PalhExOyLuqtv8W3XSKfV5Xq3X83qcxsyI2C0irqwt//tGRNRpy0bEN+t2vTYidu9u0ZY0PBGxRz1bdXdEXB4RL+8xz/K1RfOXEbFCj+lbRcQZ9azehRExszFt54j4c13+NRHx/sa0mRExp9bhr8APR4qDPcrdNspZxLvrOnyiz2p+DLgLeGdmXp3FvMz8YWZ+py6rczZtl4i4Afh9Hf/ziPhrjZmnRMTTG+U/vsbMuyLibOBJXfXLiHhyfb1iRPxXRNxQY+h+EbFy13b4eETcEhF/iYid67RdgbcDn6ox9sg+6zhfZt4FHMHCsX+k+L5iROwdpXX+5vp6xTptrShnbefV954aEcv0OiZF1xnJiDgpIr4S5Zh6d0QcF42zyhHxroi4Psqx9fMxyllW9ZGZPnzMfwDXAa8ALgeeBiwL3AhsCCQwvc73J2Cbxvt+BXy8x/JWBR4GZo5S7k6UFukPActRWryfTGmlWBGYRkka967zrwBcTwnQywNvAv4BfLVOnwnMqa+XAc4FvlDftzFwDfCqOn0WcD+wbV3fPYGzurfJCHWfX1Yd3gS4CXhZHX4ccAfwzrpuO9Thx9fpJwPfpfwAeBYwF3h5nXYm5eADsBqwVX09vX4ey3Vtw9MawwkcBaxBCbhzga3rtN3qZ7gepcXkhO7l+fDhYzgP4Kk1rq5Th6cDT6qvZwE/rjHwt8BBwLLNafX1usBtNW4tU2PlbcC0Ov3VlMQygJcA9wHPrtNmUuLt1ynxdeXR4mBX/f8CvKi+XrOz3B7znQXMGmVbdGLZwZTjxcp1/HsoLdYrAnsDFzTecyhwWJ3/GTXedse+J9fXe1OS2sfV5R0J7Nm1Hb5MOY5sW7fTmnX6QdRjygj1b5a1JnAc8OXG9JHi+5frNnoC5Rh3BvCVOm1PYL9ar+UpLfhRp11H45hE1/GAcsb3auAp9bM9CdirTtsMuIdypnQF4L8ox86+xzgfvR+2QKufTiv0vwKXUQJU0/8B7wCIiMcBrwJ+2mM5a1KC+187IyLiP+uv6nsj4nONeW/OzO9k5kOZ+ffMvCozj8/MBzJzLvAtyoEAYCtKUNk7M/+Rmb8AzumzLltQDipfzswHs/SN+wHwtsY8p2Xm0Vn6kP2IPl1LRrBOXae7gCuAPwKd1uBXA1dm5o/quh1C2aavjYj1KYFsj8y8PzMvAA6gJNtQAtuTI2KtzLwnM89qWa+9srT43AD8gRLAAd4C/HdmzsnMO4C9Wi5X0qJ7mJIYbhYRy2fmdZl5dWP6Y4FjKEnQztm7b+s7gKNr3HokM48HZlOSQDLzt7mg1fdkSmL3osb7HwG+WOPr3+u4QePgP2rdH5uZd2TmeX3mW4uFY//rapy8Ox59wfOszLy3U5fMPDAz787MByjJ/eYRsXqUC+XeCHyhzn8J5Xj0KBERwPuAj2Xm7Zl5N/D/WDj2/4OS8P4jM4+mJJdtr585LyLmAbdSGiu+X8sfLb6/vZZ9Sz3GfYmFY/8TgQ1r3U7NLBnwgH6YmVfU7XkYC2L/m4AjM/O0zHyQ0rDUZrmqTKDVz4+AHSmtmgf3mP5jSgK4GiUZOzUz/9JjvjsogfqJnRGZ+aks/aB/RWmR7bix+caIeEJEHFpPEd5Vy+ychloHuKkroFzfZ102ZEGCO68Gus8Aazfm+Wvj9X3AStGuO8PNmblGZj6W0uL7dxYE9XV61O16SgvSOkAnsHdPA9iF0opwWZSuH69pUSd49Hqt1qhTc3svtO0lDU9mXgV8lJIY3lLj3DqNWbYCnkn5AdwvudkQeHNXXHshNdZGxDYRcVY9/T+Pklg3Lw6fm5n3dy1z0Dj4xrq86yPi5Ih4Xp863sbCsf+IGvs/Rmn9bJofg6J0MdsrIq6usf+6OmktSkvtciwcs/rF/mmU/tfnNrbRMXX8/Dpm5kON4WacHNSz63qtBHwPODUiVmL0+N59bLi+jgP4BnAVcFyULjifblmngWJ/Zt5H+ZzUkgm0esrM6ykXE24LHN5j+k2U7gWvp/xi/lGf5dxLaY19wyDFdg3vWcc9syam76CcjoRyCnHd2sLQsUGf5d4IXFsT3M7jMZm57QB16lWvkWfOvJPSGv/aOupmysGuaQNKq/7NwOMi4jE9ppGZV2bmDpRTfF8HfhERq7atUw9/oXTf6Fh/MZcnqYXM/GlmvpAF3eO+3ph8HCX+nRgRa/d6PyWu/agrrq2amXvVfrS/pJyeX7smd0ezIH7CYsSQzDwnM7ejxKVfU1o4ezkR2D4iBsk1mvXZEdiO0p1wdUoXBSj1n0vpdtGMWf1i/62UxoynN7bR6lku+BtE29j/D0oL80aUriUjxncefWzYoI6jtr5/PDM3phxL/j0W9JNfnPi/UOyv/cEfvxjLm7JMoDWSXSj9eO/tM/1g4FPAP1Nak/v5FPCeiPh0RDwBICLWowSZkTyGcjptXkSsC3yyMe1MShD9cEQsFxFvALbss5yzgbuiXDCzcm3deEZEbDFK+R1/o/SbHkhtlX8bcGkddTTwlIjYsdb1rZR+aEdl5o2Ufm97RsRKEfFMynb/SV3WOyJiWmY+Asyry3uYchB5pE29uhwGfCQi1o1yYeYei7gcSS1FxFMj4mU10b2fkuQt1E0jM/+T8kP8xOi6rWjVOQv4qhrTVopyUdx6lNbdFanJZkRsA4zJLS4jYoUo95xfvSaMd3XXveFblG58P4pyUXjUZPJZoxTzGOABSsvoKpRuFwDU7iWHUy40XyUiNqNc6P4oNW7+APh249izbkS8asDVbRv7lwV2pnye14wW34FDgM9FxLT6GX+B8rkSEa+JiCfXRqLONu5s51b16vILyn7z/CgXpn6JhX9YaUAm0Oqr9p+bPcIsv6L8ev7VCEk2mXka8DLgxcAVjdNoJwHfGWH5X6Lc8uhOysU081vCa9+tN1C6mNwBvJUeLeV13ocpv+CfRWlVv5XSSrD6CGU37UkJcvOi/9Xm60S9DzTlNNzjKP3byMzbgNcAH6ccED4FvCYzb63v3YHSwnIzZZt+sfZnhHK7v0vrcv8beFvtS3cf8DXg9FqvrQZcl44fUFq5LgLOpyT5D9H/QChp7KxIue7gVsqp9idQupUtJDO/QmnhPSHKtSbNaTdSWmk/Q0mUb6Q0MixTuwx8mPJD+Q5Ki+4RY1j/dwLX1e4Vu1Gvh+lR/1sp3VHup1wTcjdwASVB/rcRln8wJY7eRLnYufvaj90pXRL+SrnQ74cjLGsPSleIs2p9T2DwPs7/S+nrPS8ifj3CfBfWGH0HJZl/fWbeXqeNFN+/Sum3fhFwMXBeHQflYvQTKI1IZwLfzXrXKgY7JvWUmZdSLtY/lNIafTdwC+UHi1roXNEpLZKIuBp4f2aeMNF10aKrLVT7ZWZ3VxNJ0lKqnjGdB2yS5RasGpAt0FpkEfFGSl+s3090XdRO7cqybe1Ssi7wRUbuhiNJWgpExGtr95dVKf3kL2bBhZoakAm0FklEnES52viDtZ+ZlixB6SJzB6ULx58p/e8kSUu37ShdSm6mdBV5W8tb5Am7cEiSJEmt2AItSQIgIg6M8pfGl/SZHhHxP1H+kviiiHj2eNdRkiYDE2hJUsdBlDu/9LMN5ZTvJsCulG5ckjTltPmntUlhrbXWyunTp090NSRpkZx77rm3Zua00eccf5l5SkRMH2GW7YCDa3/JsyJijYh4Yp9/IQWM2ZKWbP1i9tAS6Ig4kHLv21sy8xk9pgflvrbbUv5mcqfMPG+05U6fPp3Zs0e6NbEkTV4R0e9vh5cE67LwXyjPqeMWSqAjYldKCzUbbLCBMVvSEqtfzB5mF46D8FSgJC1Nev1j2aOuRM/M/TNzRmbOmDZtUja2S9JiGVoCnZmnALePMMv8U4GZeRawRkQ8cVj1kSQttjnA+o3h9Si3wpKkKWUiLyLsdyrwUSJi14iYHRGz586dOy6VkyQ9yhHAu+rdOLYC7hyp/7MkLa0m8iLCgU4FQjkdCOwPMGPGDG9cLUlDEBGHADOBtSJiDuUfKpcHyMz9gKMp161cRbl2ZeeJqakkTayJTKA9FShJk0hm7jDK9AQ+OE7VkaRJayK7cHgqUJIkSUucYd7GzlOBkiRJWuoMLYH2VKAkSZKWRv6VtyRJktSCCbQkSZLUggm0JEmS1IIJtCRJktSCCbQkSZLUwkT+kYokzXfYz7cc6vLf8uazh7p8SZpKpnrMtgVakiRJasEEWpIkSWrBLhxLmBd85wVDXf7pHzp9qMuXpKlk2DEbjNvSRLAFWpIkSWrBBFqSJElqwS4ci+iGL//zUJe/wRcuHuryJRWb/+LYoZdx4ZteNfQylgTP+eTBQy/j3G+8q+f4qRizT37xS4a6/JeccvJQl78ovvaONw11+Z/98S+GunyNbrLEbFugJUmSpBZMoCVJkqQWluguHBN5OnAqmoqnAyVJGs2fv/b7oS7/aZ99Wd9ps2bNGmrZw17+ksoWaEmSJKmFJboFWhqmYV+MAv0vSJmo1ozxaGmwNUOStKQzgZYkSWNmn48fOfQydv/ma4dehjQSu3BIkiRJLdgCrUlv2K0ZtmRIkqQ2bIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJEmSWjCBliRJklowgZYkSZJaMIGWJAEQEVtHxOURcVVEfLrH9NUj4siIuDAiLo2InSeinpI00UygJUlExLLAvsA2wGbADhGxWddsHwT+lJmbAzOBb0bECuNaUUmaBIaaQNuaIUlLjC2BqzLzmsx8EDgU2K5rngQeExEBrAbcDjw0vtWUpIk3tATa1gxJWqKsC9zYGJ5TxzXtAzwNuBm4GPhIZj7SvaCI2DUiZkfE7Llz5w6rvpI0YYbZAm1rhiQtOaLHuOwafhVwAbAO8Cxgn4h47KPelLl/Zs7IzBnTpk0b63pK0oQbZgJta4YkLTnmAOs3htejxOamnYHDs7gKuBbYdJzqJ0mTxjATaFszJGnJcQ6wSURsVLvSvQ04omueG4CXA0TE2sBTgWvGtZaSNAkMM4G2NUOSlhCZ+RCwO3As8GfgsMy8NCJ2i4jd6mxfAZ4fERcDJwJ7ZOatE1NjSZo4yw1x2fNbM4CbKK0ZO3bN02nNONXWDEmaWJl5NHB017j9Gq9vBl453vWSpMlmaAl0Zj4UEZ3WjGWBAzutGXX6fpTWjINqa0Zga4YkSZImuWG2QNuaIUmSpKWO/0QoSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLUwUAIdERtGxCvq65Uj4jHDrZYkSZI0OY2aQEfE+4BfAN+vo9YDfj3EOkmSJkBEbB0Rl0fEVRHx6T7zzIyICyLi0og4ebzrKEmTwSAt0B8EXgDcBZCZVwJPGGThBmNJWjJExLLAvsA2wGbADhGxWdc8awDfBV6XmU8H3jze9ZSkyWC5AeZ5IDMfjAgAImI5IEd7UyMY/yswBzgnIo7IzD815lmDEoy3zswbImKgxFySNOa2BK7KzGsAIuJQYDvgT415dgQOz8wbADLzlnGvpSRNAoO0QJ8cEZ8BVo6IfwV+Dhw5wPvmB+PMfBDoBOMmg7EkTQ7rAjc2hufUcU1PAdaMiJMi4tyIeFevBUXErhExOyJmz507d0jVlaSJM0gCvQcwF7gYeD9wNPC5Ad43ZsFYkjR00WNc99nG5YDnAK8GXgV8PiKe8qg3Ze6fmTMyc8a0adPGvqaSNMFG7MIREcsAF2XmM4AftFx2m2D8cmBl4MyIOCszr+iqx67ArgAbbLBBy2pIkgYwB1i/MbwecHOPeW7NzHuBeyPiFGBz4AokaQoZsQU6Mx8BLoyIRclaBw3Gx2TmvZl5K9AJxt31sDVDkobrHGCTiNgoIlYA3gYc0TXPb4AXRcRyEbEK8Fzgz+NcT0macINcRPhE4NKIOBu4tzMyM183yvvmB2PgJkow3rFrnt8A+9QLE1egBONvD1h3SdIYycyHImJ34FhgWeDAzLw0Inar0/fLzD9HxDHARcAjwAGZecnE1VqSJsYgCfSXFmXBBmNJWrJk5tGU61ya4/brGv4G8I3xrJckTTajJtCZeXJErA1sUUedPejdMgzGkiRJWtoM8k+EbwHOptww/y3AHyPiTcOumCRJkjQZDdKF47PAFp1W54iYBpxA+XtvSZIkaUoZ5D7Qy3R12bhtwPdJkiRJS51BWqCPiYhjgUPq8FuB3w2vSpIkSdLkNchFhJ+MiDcAL6T8Ocr+mfmroddMkiRJmoRGTaDrfZyPzszD6/DKETE9M68bduUkSZKkyWaQvsw/p9yjuePhOk6SJEmacgZJoJfLzAc7A/X1CsOrkiRJkjR5DZJAz42I+X/bHRHbAbcOr0qSJEnS5DXIXTh2A34SEftQLiK8EXjXUGslSZIkTVKD3IXjamCriFgNiMy8e/jVkiRJkianQf7K+yMR8VjgXuDbEXFeRLxy+FWTJEmSJp9B+kC/JzPvAl4JPAHYGdhrqLWSJEmSJqlBEuioz9sCP8zMCxvjJEmSpCllkAT63Ig4jpJAHxsRj2Hh+0JLkiRJU8Ygd+HYBXgWcE1m3hcRj6d04wAgIp6emZcOqX6SJEnSpDLIXTgeAc5rDN8G3NaY5UfAs8e+apIkSdLkM0gXjtHYH1qSJElTxlgk0DkGy5AkSZKWCGORQEuSJElTxlgk0A+OwTIkSZKkJcIg/0T4y4h4dUT0nDcztxr7akmSJEmT0yAt0N8DdgSujIi9ImLTIddJkiRJmrRGTaAz84TMfDvlVnXXAcdHxBkRsXNELD/sCkqSJEmTyUB9oOufp+wEvBc4H/hvSkJ9/NBqJkmSJE1Co/6RSkQcDmxK+cOU12bmX+qkn0XE7GFWTpIkSZpsBvkr730y8/e9JmTmjDGujyRJkjSpDdKF42kRsUZnICLWjIgPDK9KkiRJ0uQ1SAL9vsyc1xnIzDuA9w2tRpIkSdIkNkgCvUxERGcgIpYFVhhelSRJkqTJa5A+0McCh0XEfkACuwHHDLVWkiRJ0iQ1SAK9B/B+4N+AAI4DDhhmpSRJkqTJatQEOjMfofwb4feGXx1JkiRpchvkPtCbAHsCmwErdcZn5sZDrJckSZI0KQ1yEeEPKa3PDwEvBQ6m/KmKJEmSNOUMkkCvnJknApGZ12fmLOBlw62WJEmSNDkNchHh/RGxDHBlROwO3AQ8YbjVkiRJkianQVqgPwqsAnwYeA7wDuDdQ6yTJEmSNGmN2AJd/zTlLZn5SeAeYOdxqZUkSZI0SY3YAp2ZDwPPaf4ToSRJkjSVDdIH+nzgNxHxc+DezsjMPHxotZIkSZImqUES6McBt7HwnTcSMIGWJEnSlDPIPxHa71mSJEmqBvknwh9SWpwXkpnvGUqNJEmSpElskC4cRzVerwS8Hrh5ONWRJEmSJrdBunD8sjkcEYcAJwytRpIkSdIkNsgfqXTbBNhgrCsiSZIkLQkG6QN9Nwv3gf4rsMfQaiRJkiRNYoN04XjMeFREkiRJWhKM2oUjIl4fEas3hteIiO2HWitJkiRpkhqkD/QXM/POzkBmzgO+OLQaSZIkSZPYIAl0r3kGuf2dJEmStNQZJIGeHRHfiognRcTGEfFt4NxhV0ySJEmajAZJoD8EPAj8DDgM+DvwwWFWSpIkSZqsBrkLx73Ap8ehLpIkSdKkN8hdOI6PiDUaw2tGxLFDrZUkadxFxNYRcXlEXBURfRtOImKLiHg4It40nvWTpMlikC4ca9U7bwCQmXcATxhk4QZjSVoyRMSywL7ANsBmwA4RsVmf+b4O2JAiacoaJIF+JCLm/3V3RExn4X8m7MlgLElLlC2BqzLzmsx8EDgU2K7HfB8CfgncMp6Vk6TJZJDb0X0WOC0iTq7DLwZ2HeB984MxQER0gvGfuubrBOMtBqqxJGkY1gVubAzPAZ7bnCEi1gVeD7yMEWJ2ROxKPU5ssMEG/WaTpCXWqC3QmXkMMAO4nHInjo9T7sQxml7BeN3mDI1gvN9IC4qIXSNidkTMnjt37gBFS5Jaih7jus827g3skZkPj7SgzNw/M2dk5oxp06aNVf0kadIYtQU6It4LfARYD7gA2Ao4k9ICMeJbe4zrG4wjes1e35S5P7A/wIwZM0btPiJJam0OsH5jeD3g5q55ZgCH1ni9FrBtRDyUmb8elxpK0iQxSBeOj1BO1Z2VmS+NiE2BLw3wPoOxJC05zgE2iYiNgJuAtwE7NmfIzI06ryPiIOAo47WkqWiQBPr+zLw/IoiIFTPzsoh46gDvMxhL0hIiMx+KiN0pF3QvCxyYmZdGxG51+ohd7SRpKhkkgZ5T7wP9a+D4iLiDR7ckP4rBWJKWLJl5NHB017iesTozdxqPOknSZDTIPxG+vr6cFRF/AFYHjhlk4QZjSZIkLW0GaYGeLzNPHn0uSZIkaek1yB+pSJIkSapMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFkygJUmSpBZMoCVJkqQWTKAlSZKkFoaaQEfE1hFxeURcFRGf7jH97RFxUX2cERGbD7M+kqT+jNmSNJihJdARsSywL7ANsBmwQ0Rs1jXbtcBLMvOZwFeA/YdVH0lSf8ZsSRrcMFugtwSuysxrMvNB4FBgu+YMmXlGZt5RB88C1htifSRJ/RmzJWlAw0yg1wVubAzPqeP62QX4Xa8JEbFrRMyOiNlz584dwypKkipjtiQNaJgJdPQYlz1njHgpJRjv0Wt6Zu6fmTMyc8a0adPGsIqSpMqYLUkDWm6Iy54DrN8YXg+4uXumiHgmcACwTWbeNsT6SJL6M2ZL0oCG2QJ9DrBJRGwUESsAbwOOaM4QERsAhwPvzMwrhlgXSdLIjNmSNKChtUBn5kMRsTtwLLAscGBmXhoRu9Xp+wFfAB4PfDciAB7KzBnDqpMkqTdjtiQNbphdOMjMo4Gju8bt13j9XuC9w6yDJGkwxmxJGoz/RChJkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktTDUBDoito6IyyPiqoj4dI/pERH/U6dfFBHPHmZ9JEn9GbMlaTBDS6AjYllgX2AbYDNgh4jYrGu2bYBN6mNX4HvDqo8kqT9jtiQNbpgt0FsCV2XmNZn5IHAosF3XPNsBB2dxFrBGRDxxiHWSJPVmzJakAUVmDmfBEW8Cts7M99bhdwLPzczdG/McBeyVmafV4ROBPTJzdteydqW0dgA8Fbh8Eau1FnDrIr53cU1U2a7z1Ch7qpU7kWUvbrkbZua0sarMWJmkMRuW3M95SSx7qpU7kWW7zktO2T1j9nKLV58RRY9x3dn6IPOQmfsD+y92hSJmZ+aMxV3OklS26zw1yp5q5U5k2RO5zkM26WI2TM3Peaqts9t6apS9tK3zMLtwzAHWbwyvB9y8CPNIkobPmC1JAxpmAn0OsElEbBQRKwBvA47omucI4F31yu6tgDsz8y9DrJMkqTdjtiQNaGhdODLzoYjYHTgWWBY4MDMvjYjd6vT9gKOBbYGrgPuAnYdVn2pMTikuYWW7zlOj7KlW7kSWPZHrPDSTNGbD1Pycp9o6u62nRtlL1ToP7SJCSZIkaWnkPxFKkiRJLZhAS5IkSS0slQn0AH9Hu2lEnBkRD0TEJ8ax3LfXv7+9KCLOiIjNx7Hs7Wq5F0TE7Ih44XiU25hvi4h4uN5rdkwMsM4zI+LOus4XRMQXxqPcRtkXRMSlEXHyWJQ7SNkR8cnG+l5St/njxqHc1SPiyIi4sK7zmPWNHaDsNSPiV3X/PjsinjEGZR4YEbdExCV9pkf4l9ZjZqJi9oBlDyVuT1TMHqTsxnxjGreN2cbsOn3MY3Zd7vjG7cxcqh6Ui1+uBjYGVgAuBDbrmucJwBbA14BPjGO5zwfWrK+3Af44jmWvxoI+788ELhuPchvz/Z5yAdKbxnGdZwJHTcD+tQbwJ2CDzv42XmV3zf9a4PfjtM6fAb5eX08DbgdWGKeyvwF8sb7eFDhxDMp9MfBs4JI+07cFfke5L/JWY/VdnoqPiYrZLcoe87g9UTF70LIb841Z3DZmG7Mb84x5zK7LGte4vTS2QI/6d7SZeUtmngP8Y5zLPSMz76iDZ1HuoTpeZd+TdQ8CVqXHnx8Mo9zqQ8AvgVvGoMy2ZY+1QcrdETg8M2+Asr+NY9lNOwCHjFO5CTwmIoJy4L8deGicyt4MOBEgMy8DpkfE2otTaGaeQlmHfvxL67EzUTF70LKHEbcnKmYPVHY11nHbmG3M7hjzmF2XNa5xe2lMoNcFbmwMz6njJlu5u1B+CY1b2RHx+oi4DPgt8J7xKDci1gVeD+w3BuW1Krt6Xj1F9buIePo4lfsUYM2IOCkizo2Id41BuYOWDUBErAJsTTkAjke5+wBPo/ypxsXARzLzkXEq+0LgDQARsSWwIWP343Rx6qXBTOS2nKi4PVExe6CyhxS3jdnG7I6JiNmD1m1gS2MCPdBfzU5kuRHxUkog3mM8y87MX2XmpsD2wFfGqdy9gT0y8+ExKK9t2edR/sN+c+A7wK/HqdzlgOcArwZeBXw+Ip4yTmV3vBY4PTNH+jU+luW+CrgAWAd4FrBPRDx2nMrei3Lwu4DSanY+Y9OSsrj10mAmcltOVNyeqJg9aNl7M/Zx25jdu+wOY/bwjWmsGdofqUygifqr2YHKjYhnAgcA22TmbeNZdkdmnhIRT4qItTLz1iGXOwM4tJwlYi1g24h4KDN/vRjlDlR2Zt7VeH10RHx3nNZ5DnBrZt4L3BsRpwCbA1csRrmDlt3xNsbmVOCg5e4M7FVPOV8VEddS+radPeyy6+e8M5SLRIBr62OY/EvrsTOR23Ki4vZExexByx5G3DZmG7OBCYvZA9WtlcXpQD0ZH5QfBdcAG7GgA/vT+8w7i7G7iHDUcoENKP/g9fzxXmfgySy4IOXZwE2d4fHY1nX+gxi7iwgHWed/aqzzlsAN47HOlNNiJ9Z5VwEuAZ4xXvs2sDqlH9iq47itvwfMqq/XrvvXWuNU9hrUi1+A91H6uI3Fek+n/8Uor2bhi1HOHosyp+JjomJ2i/1rzOP2RMXsttu7zj8mcduYbcxuzDOUmF2XN25xe0wqPNkelCstr6BcCfrZOm43YLf6+p8ov0TuAubV148dh3IPAO6gnDa5AJg9juu8B3BpLfdM4IXjUW7XvAcxRgn0gOu8e13nCykX/4zJAXCQdQY+Sbmq+xLgo+O1znV4J+DQsSpzwG29DnAcpS/dJcA7xrHs5wFXApcBh1PvmLCYZR4C/IVy0docyqn7ZpkB7FvrdDEwYyy391R7DPAZDyVmD1j2UOL2AOUOJWYPUnbXvAcxdg0fxuzeZe+EMXssyh3XuO1feUuSJEktLI0XEUqSJElDYwItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACrUkjIu4Z5/LOGKPlzIyIOyPi/Ii4LCL+a4D3bB8Rm41F+ZI0EYzZmspMoLXUiogR/2kzM58/hsWdmpn/AvwL8JqIeMEo828PGIwlqTJma0myNP6Vt5YiEfEkyo3PpwH3Ae/LzMsi4rXA5yj/dHQb8PbM/FtEzKLcIH46cGtEXEH5J7GN6/Pemfk/ddn3ZOZqETGT8g9ntwLPAM6l3FQ+I2Jb4Ft12nnAxpn5mn71zcy/R8QFwLq1jPcBu9Z6XgW8E3gW8DrgJRHxOeCN9e2PWs9F3W6SNBGM2ZoyxvKfb3z4WJwHcE+PcScCm9TXzwV+X1+vyYK/fH0v8M36ehYlmK7cGD4DWBFYixK4l2+WB8wE7gTWo5yVORN4IbAScCOwUZ3vEOCoHnWc2Rlf63Uu8E91+PGN+b4KfKi+PojGv3v1W08fPnz4mKwPY7Yxeyo/bIHWpBURqwHPB34eEZ3RK9bn9YCfRcQTKS0F1zbeekRm/r0x/NvMfAB4ICJuAdam/M1n09mZOaeWewGlNeQe4JrM7Cz7EErLRC8vioiLgKcCe2XmX+v4Z0TEV4E1gNWAY1uupyQtEYzZmkpMoDWZLQPMy8xn9Zj2HeBbmXlE43Rex71d8z7QeP0wvff7XvNEj/n6OTUzXxMRTwFOi4hfZeYFlFaL7TPzwojYidLy0W2k9ZSkJYUxW1OGFxFq0srMu4BrI+LNAFFsXievDtxUX797SFW4DNg4IqbX4beO9obMvALYE9ijjnoM8JeIWB54e2PWu+u00dZTkpYIxmxNJSbQmkxWiYg5jce/UwLYLhFxIXApsF2ddxbl9NmplItFxlw9pfgB4JiIOA34G6Xf3Wj2A14cERsBnwf+CBxPCe4dhwKfrLdRehL911OSJitjtjF7yup06JfUQ0Sslpn3ROnoti9wZWZ+e6LrJUl6NGO2xost0NLI3lcvULmUcgry+xNbHUnSCIzZGhe2QEuSJEkt2AItSZIktWACLUmSJLVgAi1JkiS1YAItSZIktWACLUmSJLXw/wF6srOpLNBqSgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compare my Gradient Boosting Classifier with the one from sklearn by plotting side by side the scores each of them yields  \n",
    "# on the test data\n",
    "\n",
    "fig, axs = plt.subplots(1,2, figsize = (12, 6))\n",
    "\n",
    "_ = sns.barplot(x = list(my_test_accuracy_scores.keys()), y = list(my_test_accuracy_scores.values()), ax = axs[0]);\n",
    "_ = sns.barplot(x = list(sklearn_test_accuracy_scores.keys()), y = list(sklearn_test_accuracy_scores.values()), ax = axs[1]);\n",
    "\n",
    "_ = axs[0].set_ylabel(\"accuracy_score\")\n",
    "\n",
    "for ax in axs:\n",
    "    _ = ax.set_xlabel(\"Learning Rate\")\n",
    "\n",
    "_ = axs[0].set_title(\"My Gradient Boosting\")    \n",
    "_ = axs[1].set_title(\"sklearn's Gradient Boosting\")    \n",
    "\n",
    "_ = fig.suptitle(\"Performance of Mine vs. sklearn's Gradient Boosting Classifier on the Test Set\", weight = \"bold\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}