Copia_di_Copia_di_ML7upfin.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyP49Q0sPCPgQ9tJcW+gv3Ii",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/francescopatane96/Bioactivity-prediction-with-ML/blob/main/Copia_di_Copia_di_ML7upfin.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install rdkit"
      ],
      "metadata": {
        "id": "sTiafVkFRvo8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install lazypredict"
      ],
      "metadata": {
        "id": "npXUo4YLmFLJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/volkamerlab/teachopencadd.git"
      ],
      "metadata": {
        "id": "L10Sz7LoR5f0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f131bf55-4c75-4994-c8a9-7dcdb520ed08"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/volkamerlab/teachopencadd.git\n",
            "  Cloning https://github.com/volkamerlab/teachopencadd.git to /tmp/pip-req-build-m7gu1i4y\n",
            "  Running command git clone -q https://github.com/volkamerlab/teachopencadd.git /tmp/pip-req-build-m7gu1i4y\n",
            "Building wheels for collected packages: teachopencadd\n",
            "  Building wheel for teachopencadd (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for teachopencadd: filename=teachopencadd-2022.9.0+2.g8f2c87a-py3-none-any.whl size=108231919 sha256=6ad674f9867e73e539660d5fd38c19b6032cd9392a7c5545038dd2c79d8ba620\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-3gkyv1r2/wheels/ee/da/82/b2c07c8ddbf7e09a56a283a394ebff26d956480b85a9ef011f\n",
            "Successfully built teachopencadd\n",
            "Installing collected packages: teachopencadd\n",
            "Successfully installed teachopencadd-2022.9.0+2.g8f2c87a\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VqMlkn50RdJD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aa5db107-ae4f-4b31-d8c7-4a49d11f5463"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:143: FutureWarning: The sklearn.utils.testing module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.utils. Anything that cannot be imported from sklearn.utils is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n"
          ]
        }
      ],
      "source": [
        "from pathlib import Path\n",
        "import seaborn as sns\n",
        "from warnings import filterwarnings\n",
        "import time\n",
        "import lazypredict\n",
        "from lazypredict.Supervised import LazyRegressor\n",
        "from lazypredict.Supervised import LazyClassifier\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn import svm, metrics, clone\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.model_selection import KFold, train_test_split\n",
        "from sklearn.metrics import auc, accuracy_score, recall_score\n",
        "from sklearn.metrics import roc_curve, roc_auc_score\n",
        "import matplotlib.pyplot as plt\n",
        "from rdkit import Chem\n",
        "from rdkit.Chem import MACCSkeys\n",
        "from rdkit.Chem.AllChem import GetMorganFingerprintAsBitVect\n",
        "\n",
        "from teachopencadd.utils import seed_everything\n",
        "\n",
        "# Silence some expected warnings\n",
        "filterwarnings(\"ignore\")\n",
        "# Fix seed for reproducible results\n",
        "SEED = 22\n",
        "seed_everything(SEED)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Read data (Lipinski)\n",
        "chembl_df = pd.read_csv(\n",
        "    \"IDH_compounds_lipinski.csv\",\n",
        "    index_col=0,\n",
        ")\n",
        "\n",
        "# Look at head\n",
        "print(\"Shape of dataframe : \", chembl_df.shape)\n",
        "chembl_df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        },
        "id": "QWPd9haHSfne",
        "outputId": "d3794555-43a8-4c40-a5e2-3f56671e2732"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of dataframe :  (1545, 11)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id  IC50 units  \\\n",
              "0      CHEMBL4279047  0.04    nM   \n",
              "1      CHEMBL4278845  0.25    nM   \n",
              "2      CHEMBL4283785  0.30    nM   \n",
              "3      CHEMBL4280132  0.70    nM   \n",
              "4      CHEMBL3909586  1.00    nM   \n",
              "\n",
              "                                              smiles  pIC50  \\\n",
              "0  C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...  10.40   \n",
              "1  C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...   9.60   \n",
              "2  CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...   9.52   \n",
              "3  FC1(F)CCC(Nc2nc(NC3CCC(F)(F)C3)nc(-c3cccc(Cl)n...   9.15   \n",
              "4  C[C@H](Nc1nccc(N2C(=O)OC[C@@H]2[C@H](C)F)n1)c1...   9.00   \n",
              "\n",
              "                                               ROMol  molecular_weight  n_hba  \\\n",
              "0  <img data-content=\"rdkit/molecule\" src=\"data:i...            414.08      6   \n",
              "1  <img data-content=\"rdkit/molecule\" src=\"data:i...            414.08      6   \n",
              "2  <img data-content=\"rdkit/molecule\" src=\"data:i...            414.08      6   \n",
              "3  <img data-content=\"rdkit/molecule\" src=\"data:i...            430.13      6   \n",
              "4  <img data-content=\"rdkit/molecule\" src=\"data:i...            446.17      7   \n",
              "\n",
              "   n_hbd  logp  ro5_fulfilled  \n",
              "0      2  4.31           True  \n",
              "1      2  4.31           True  \n",
              "2      2  4.31           True  \n",
              "3      2  4.79           True  \n",
              "4      1  4.46           True  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-51e34630-1f26-4b02-a104-0751342c09c4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>IC50</th>\n",
              "      <th>units</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "      <th>ROMol</th>\n",
              "      <th>molecular_weight</th>\n",
              "      <th>n_hba</th>\n",
              "      <th>n_hbd</th>\n",
              "      <th>logp</th>\n",
              "      <th>ro5_fulfilled</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL4279047</td>\n",
              "      <td>0.04</td>\n",
              "      <td>nM</td>\n",
              "      <td>C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>10.40</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>414.08</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>4.31</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL4278845</td>\n",
              "      <td>0.25</td>\n",
              "      <td>nM</td>\n",
              "      <td>C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>9.60</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>414.08</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>4.31</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL4283785</td>\n",
              "      <td>0.30</td>\n",
              "      <td>nM</td>\n",
              "      <td>CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...</td>\n",
              "      <td>9.52</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>414.08</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>4.31</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CHEMBL4280132</td>\n",
              "      <td>0.70</td>\n",
              "      <td>nM</td>\n",
              "      <td>FC1(F)CCC(Nc2nc(NC3CCC(F)(F)C3)nc(-c3cccc(Cl)n...</td>\n",
              "      <td>9.15</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>430.13</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>4.79</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CHEMBL3909586</td>\n",
              "      <td>1.00</td>\n",
              "      <td>nM</td>\n",
              "      <td>C[C@H](Nc1nccc(N2C(=O)OC[C@@H]2[C@H](C)F)n1)c1...</td>\n",
              "      <td>9.00</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>446.17</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>4.46</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-51e34630-1f26-4b02-a104-0751342c09c4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-51e34630-1f26-4b02-a104-0751342c09c4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-51e34630-1f26-4b02-a104-0751342c09c4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Feature for proving and Proving our data \\\\ NaN finder\n",
        "def check_missing_values(dataframe):\n",
        "    \n",
        "    if dataframe.isnull().sum().sum() > 0:\n",
        "        m_total = dataframe.isnull().sum().sort_values(ascending=False) \n",
        "        total = m_total[m_total > 0]\n",
        "\n",
        "        m_percent = dataframe.isnull().mean().sort_values(ascending=False) \n",
        "        percent = m_percent[m_percent > 0] \n",
        "\n",
        "        missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n",
        "    \n",
        "        print(f'Total and Percentage of NaN:\\n {missing_data}')\n",
        "    else: \n",
        "        print('No NaN found.')\n",
        "        \n",
        "        \n",
        "check_missing_values(dataframe=chembl_df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YFTwK2XPnXK3",
        "outputId": "4056eafd-0eef-4089-81d3-303a6cdcf1bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "No NaN found.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# remove NaN\n",
        "chembl_df = chembl_df.dropna()"
      ],
      "metadata": {
        "id": "B0m1hX4InpUt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chembl_df.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H_DaNn2An0wO",
        "outputId": "62d8f61f-d88e-49d7-8580-b537a472bef6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1545, 11)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Keep only the columns we want\n",
        "chembl_df = chembl_df[[\"molecule_chembl_id\", \"smiles\", \"pIC50\"]]\n",
        "chembl_df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ui_bJaycSn6u",
        "outputId": "2a033272-78fa-4997-fb5b-108b5724e498"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id                                             smiles  pIC50\n",
              "0      CHEMBL4279047  C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...  10.40\n",
              "1      CHEMBL4278845  C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...   9.60\n",
              "2      CHEMBL4283785  CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...   9.52\n",
              "3      CHEMBL4280132  FC1(F)CCC(Nc2nc(NC3CCC(F)(F)C3)nc(-c3cccc(Cl)n...   9.15\n",
              "4      CHEMBL3909586  C[C@H](Nc1nccc(N2C(=O)OC[C@@H]2[C@H](C)F)n1)c1...   9.00"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b902aeee-b081-463a-b67d-20da51f5ce45\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL4279047</td>\n",
              "      <td>C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>10.40</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL4278845</td>\n",
              "      <td>C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>9.60</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL4283785</td>\n",
              "      <td>CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...</td>\n",
              "      <td>9.52</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CHEMBL4280132</td>\n",
              "      <td>FC1(F)CCC(Nc2nc(NC3CCC(F)(F)C3)nc(-c3cccc(Cl)n...</td>\n",
              "      <td>9.15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CHEMBL3909586</td>\n",
              "      <td>C[C@H](Nc1nccc(N2C(=O)OC[C@@H]2[C@H](C)F)n1)c1...</td>\n",
              "      <td>9.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b902aeee-b081-463a-b67d-20da51f5ce45')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b902aeee-b081-463a-b67d-20da51f5ce45 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b902aeee-b081-463a-b67d-20da51f5ce45');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Add column for activity\n",
        "chembl_df[\"active\"] = np.zeros(len(chembl_df))\n",
        "\n",
        "# Mark every molecule as active with an pIC50 of >= 6.3, 0 otherwise\n",
        "chembl_df.loc[chembl_df[chembl_df.pIC50 >= 6.3].index, \"active\"] = 1.0\n",
        "\n",
        "# NBVAL_CHECK_OUTPUT\n",
        "print(\"Number of active compounds:\", int(chembl_df.active.sum()))\n",
        "print(\"Number of inactive compounds:\", len(chembl_df) - int(chembl_df.active.sum()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8NqHNrxjS1Zj",
        "outputId": "83dfdd4f-6d2b-471a-a5a9-b0fdcab2225c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of active compounds: 978\n",
            "Number of inactive compounds: 567\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "chembl_df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "oW3bY18rS8cD",
        "outputId": "c89f584f-756f-4c08-9a8d-e1f11b31fe17"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id                                             smiles  \\\n",
              "0      CHEMBL4279047  C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...   \n",
              "1      CHEMBL4278845  C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...   \n",
              "2      CHEMBL4283785  CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...   \n",
              "3      CHEMBL4280132  FC1(F)CCC(Nc2nc(NC3CCC(F)(F)C3)nc(-c3cccc(Cl)n...   \n",
              "4      CHEMBL3909586  C[C@H](Nc1nccc(N2C(=O)OC[C@@H]2[C@H](C)F)n1)c1...   \n",
              "\n",
              "   pIC50  active  \n",
              "0  10.40    1.00  \n",
              "1   9.60    1.00  \n",
              "2   9.52    1.00  \n",
              "3   9.15    1.00  \n",
              "4   9.00    1.00  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-49e41373-b99f-45dc-9d7d-bfbaca0e86af\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "      <th>active</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL4279047</td>\n",
              "      <td>C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>10.40</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL4278845</td>\n",
              "      <td>C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>9.60</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL4283785</td>\n",
              "      <td>CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...</td>\n",
              "      <td>9.52</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CHEMBL4280132</td>\n",
              "      <td>FC1(F)CCC(Nc2nc(NC3CCC(F)(F)C3)nc(-c3cccc(Cl)n...</td>\n",
              "      <td>9.15</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CHEMBL3909586</td>\n",
              "      <td>C[C@H](Nc1nccc(N2C(=O)OC[C@@H]2[C@H](C)F)n1)c1...</td>\n",
              "      <td>9.00</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-49e41373-b99f-45dc-9d7d-bfbaca0e86af')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-49e41373-b99f-45dc-9d7d-bfbaca0e86af button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-49e41373-b99f-45dc-9d7d-bfbaca0e86af');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def smiles_to_fp(smiles, method=\"maccs\", n_bits=2048):\n",
        "    \"\"\"\n",
        "    Encode a molecule from a SMILES string into a fingerprint.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    smiles : str\n",
        "        The SMILES string defining the molecule.\n",
        "\n",
        "    method : str\n",
        "        The type of fingerprint to use. Default is MACCS keys.\n",
        "\n",
        "    n_bits : int\n",
        "        The length of the fingerprint.\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    array\n",
        "        The fingerprint array.\n",
        "\n",
        "    \"\"\"\n",
        "\n",
        "    # convert smiles to RDKit mol object\n",
        "    mol = Chem.MolFromSmiles(smiles)\n",
        "\n",
        "    if method == \"maccs\":\n",
        "        return np.array(MACCSkeys.GenMACCSKeys(mol))\n",
        "    if method == \"morgan2\":\n",
        "        return np.array(GetMorganFingerprintAsBitVect(mol, 2, nBits=n_bits))\n",
        "    if method == \"morgan3\":\n",
        "        return np.array(GetMorganFingerprintAsBitVect(mol, 3, nBits=n_bits))\n",
        "    else:\n",
        "        # NBVAL_CHECK_OUTPUT\n",
        "        print(f\"Warning: Wrong method specified: {method}. Default will be used instead.\")\n",
        "        return np.array(MACCSkeys.GenMACCSKeys(mol))"
      ],
      "metadata": {
        "id": "O4iESr2UTAAz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Add column for fingerprint\n",
        "chembl_df[\"fp\"] = chembl_df[\"smiles\"].apply(smiles_to_fp)\n",
        "chembl_df.head(3)\n",
        "# NBVAL_CHECK_OUTPUT"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "XBm4Na25TCcd",
        "outputId": "facd8c90-f8db-49f1-c7c1-d4c5a5779d20"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id                                             smiles  \\\n",
              "0      CHEMBL4279047  C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...   \n",
              "1      CHEMBL4278845  C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...   \n",
              "2      CHEMBL4283785  CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...   \n",
              "\n",
              "   pIC50  active                                                 fp  \n",
              "0  10.40    1.00  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
              "1   9.60    1.00  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
              "2   9.52    1.00  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-809d44f1-e6e5-42c9-ab6d-7ffd154f7b95\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "      <th>active</th>\n",
              "      <th>fp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL4279047</td>\n",
              "      <td>C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>10.40</td>\n",
              "      <td>1.00</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL4278845</td>\n",
              "      <td>C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>9.60</td>\n",
              "      <td>1.00</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL4283785</td>\n",
              "      <td>CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...</td>\n",
              "      <td>9.52</td>\n",
              "      <td>1.00</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-809d44f1-e6e5-42c9-ab6d-7ffd154f7b95')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-809d44f1-e6e5-42c9-ab6d-7ffd154f7b95 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-809d44f1-e6e5-42c9-ab6d-7ffd154f7b95');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_roc_curves_for_models(models, test_x, test_y, save_png=False):\n",
        "    \"\"\"\n",
        "    Helper function to plot customized roc curve.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    models: dict\n",
        "        Dictionary of pretrained machine learning models.\n",
        "    test_x: list\n",
        "        Molecular fingerprints for test set.\n",
        "    test_y: list\n",
        "        Associated activity labels for test set.\n",
        "    save_png: bool\n",
        "        Save image to disk (default = False)\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    fig:\n",
        "        Figure.\n",
        "    \"\"\"\n",
        "\n",
        "    fig, ax = plt.subplots()\n",
        "\n",
        "    # Below for loop iterates through your models list\n",
        "    for model in models:\n",
        "        # Select the model\n",
        "        ml_model = model[\"model\"]\n",
        "        # Prediction probability on test set\n",
        "        test_prob = ml_model.predict_proba(test_x)[:, 1]\n",
        "        # Prediction class on test set\n",
        "        test_pred = ml_model.predict(test_x)\n",
        "        # Compute False postive rate and True positive rate\n",
        "        fpr, tpr, thresholds = metrics.roc_curve(test_y, test_prob)\n",
        "        # Calculate Area under the curve to display on the plot\n",
        "        auc = roc_auc_score(test_y, test_prob)\n",
        "        # Plot the computed values\n",
        "        ax.plot(fpr, tpr, label=(f\"{model['label']} AUC area = {auc:.2f}\"))\n",
        "\n",
        "    # Custom settings for the plot\n",
        "    ax.plot([0, 1], [0, 1], \"r--\")\n",
        "    ax.set_xlabel(\"False Positive Rate\")\n",
        "    ax.set_ylabel(\"True Positive Rate\")\n",
        "    ax.set_title(\"Receiver Operating Characteristic\")\n",
        "    ax.legend(loc=\"lower right\")\n",
        "    # Save plot\n",
        "    if save_png:\n",
        "        fig.savefig(f\"{DATA}/roc_auc\", dpi=300, bbox_inches=\"tight\", transparent=True)\n",
        "    return fig"
      ],
      "metadata": {
        "id": "COhODkg0TiVk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def model_performance(ml_model, test_x, test_y, verbose=True):\n",
        "    \"\"\"\n",
        "    Helper function to calculate model performance\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    ml_model: sklearn model object\n",
        "        The machine learning model to train.\n",
        "    test_x: list\n",
        "        Molecular fingerprints for test set.\n",
        "    test_y: list\n",
        "        Associated activity labels for test set.\n",
        "    verbose: bool\n",
        "        Print performance measure (default = True)\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    tuple:\n",
        "        Accuracy, sensitivity, specificity, auc on test set.\n",
        "    \"\"\"\n",
        "\n",
        "    # Prediction probability on test set\n",
        "    test_prob = ml_model.predict_proba(test_x)[:, 1]\n",
        "\n",
        "    # Prediction class on test set\n",
        "    test_pred = ml_model.predict(test_x)\n",
        "\n",
        "    # Performance of model on test set\n",
        "    accuracy = accuracy_score(test_y, test_pred)\n",
        "    sens = recall_score(test_y, test_pred)\n",
        "    spec = recall_score(test_y, test_pred, pos_label=0)\n",
        "    auc = roc_auc_score(test_y, test_prob)\n",
        "\n",
        "    if verbose:\n",
        "        # Print performance results\n",
        "        # NBVAL_CHECK_OUTPUT        print(f\"Accuracy: {accuracy:.2}\")\n",
        "        print(f\"Sensitivity: {sens:.2f}\")\n",
        "        print(f\"Specificity: {spec:.2f}\")\n",
        "        print(f\"AUC: {auc:.2f}\")\n",
        "\n",
        "    return accuracy, sens, spec, auc"
      ],
      "metadata": {
        "id": "lyGpuP7BTlyF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def model_training_and_validation(ml_model, name, splits, verbose=True):\n",
        "    \"\"\"\n",
        "    Fit a machine learning model on a random train-test split of the data\n",
        "    and return the performance measures.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    ml_model: sklearn model object\n",
        "        The machine learning model to train.\n",
        "    name: str\n",
        "        Name of machine learning algorithm: RF, SVM, ANN\n",
        "    splits: list\n",
        "        List of desciptor and label data: train_x, test_x, train_y, test_y.\n",
        "    verbose: bool\n",
        "        Print performance info (default = True)\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    tuple:\n",
        "        Accuracy, sensitivity, specificity, auc on test set.\n",
        "\n",
        "    \"\"\"\n",
        "    train_x, test_x, train_y, test_y = splits\n",
        "\n",
        "    # Fit the model\n",
        "    ml_model.fit(train_x, train_y)\n",
        "\n",
        "    # Calculate model performance results\n",
        "    accuracy, sens, spec, auc = model_performance(ml_model, test_x, test_y, verbose)\n",
        "\n",
        "    return accuracy, sens, spec, auc"
      ],
      "metadata": {
        "id": "nGak5nNLTpOh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fingerprint_to_model = chembl_df.fp.tolist()\n",
        "label_to_model = chembl_df.active.tolist()\n",
        "\n",
        "# Split data randomly in train and test set\n",
        "# note that we use test/train_x for the respective fingerprint splits\n",
        "# and test/train_y for the respective label splits\n",
        "(\n",
        "    static_train_x,\n",
        "    static_test_x,\n",
        "    static_train_y,\n",
        "    static_test_y,\n",
        ") = train_test_split(fingerprint_to_model, label_to_model, test_size=0.2, random_state=SEED)\n",
        "splits = [static_train_x, static_test_x, static_train_y, static_test_y]\n",
        "# NBVAL_CHECK_OUTPUT\n",
        "print(\"Training data size:\", len(static_train_x))\n",
        "print(\"Test data size:\", len(static_test_x))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m9wCL7BPTrqU",
        "outputId": "166f3cd6-ef4c-46ff-996a-52b01b710524"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training data size: 1236\n",
            "Test data size: 309\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set model parameter for random forest\n",
        "param = {\n",
        "    \"n_estimators\": 200,  # number of trees to grows\n",
        "    \"criterion\": \"entropy\",  # cost function to be optimized for a split\n",
        "}\n",
        "model_RF = RandomForestClassifier(**param)"
      ],
      "metadata": {
        "id": "tPyPvcrpT1wc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit model on single split\n",
        "performance_measures = model_training_and_validation(model_RF, \"RF\", splits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5k-n0yqqT4P_",
        "outputId": "1182c351-927e-4830-b351-dd496f3a2a0b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sensitivity: 0.79\n",
            "Specificity: 0.74\n",
            "AUC: 0.86\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "r2_RF = model_RF.score(static_test_x, static_test_y)\n",
        "r2_RF"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P3rxw9fSoy5H",
        "outputId": "35f65334-3726-4d28-f366-da43ffaa2851"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7734627831715211"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize the list that stores all models. First one is RF.\n",
        "models = [{\"label\": \"Model_RF\", \"model\": model_RF}]\n",
        "# Plot roc curve\n",
        "plot_roc_curves_for_models(models, static_test_x, static_test_y);"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "pNiArRiUT72r",
        "outputId": "b430b6fd-cf15-47e2-fcc0-46a3109f1e22"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUZfbA8e8hlISOAZQWOgKhCRFEpQsCosgPC1gWdkEUVpS1l11FdG3YUFHBsmIDFcVFRVERxFUUAUF6kRoglNAhQMr5/fHewBBSJiSTyWTO53nmydx737n33EkyZ+597z2vqCrGGGPCV7FgB2CMMSa4LBEYY0yYs0RgjDFhzhKBMcaEOUsExhgT5iwRGGNMmLNEYHJFRJaLSOdgx1FYiMgDIvJGkLb9tog8Foxt5zcRuV5EvjnD19rfZB5ZIghhIrJRRJJE5JCIJHgfDGUDuU1VjVXVOYHcRjoRKSUiT4jIZm8/14rI3SIiBbH9TOLpLCLxvvNU9XFVHRqg7YmI3CYiy0TksIjEi8jHItI8ENs7UyIyWkTey8s6VPV9Ve3hx7ZOS34F+TdZVFkiCH2Xq2pZoBVwHnB/kOPJNREpnsWij4FuQG+gHHAjMAwYF4AYREQK2//DOOB24DbgLKAR8BlwWX5vKJvfQcAFc9vGo6r2CNEHsBG4xGf6aeBLn+kLgJ+BfcASoLPPsrOA/wDbgL3AZz7L+gCLvdf9DLTIuE2gOpAEnOWz7DxgN1DCm/4bsNJb/0ygtk9bBf4OrAU2ZLJv3YCjQK0M89sBqUADb3oO8AQwHzgA/DdDTNm9B3OAfwM/efvSAPirF/NBYD1ws9e2jNcmDTjkPaoDo4H3vDZ1vP0aBGz23osHfbYXBUzy3o+VwD1AfBa/24befrbN5vf/NjAe+NKL91egvs/yccAW731ZCHTwWTYamAq85y0fCrQF5nnv1XbgZaCkz2tigW+BPcAO4AGgJ3AcSPbekyVe2wrAm956tgKPARHessHee/48kOgtGwz8z1su3rKdXmxLgWa4LwHJ3vYOAZ9n/D8AIry4/vTek4Vk+BuyRyZ/S8EOwB55+OWd+g9Q0/uHGedN1/D+yXrjjvy6e9NVvOVfAh8ClYASQCdv/nneP2A7759qkLedUpls83vgJp94xgKvec/7AuuAJkBx4J/Azz5t1ftQOQuIymTfngR+yGK/N3HyA3qO90HTDPdh/QknP5hzeg/m4D6wY70YS+C+bdf3Pow6AUeA1l77zmT44CbzRPA67kO/JXAMaOK7T957XhP4I+P6fNZ7C7Aph9//297+tPXifx+Y4rP8BiDaW3YnkABE+sSdDFzpvTdRQBtc4izu7ctKYJTXvhzuQ/1OINKbbpfxPfDZ9jRggvc7qYpL1Om/s8FACjDS21YUpyaCS3Ef4BW930MToJrPPj+Wzf/B3bj/g3O917YEooP9v1rYH0EPwB55+OW5f4BDuG8+CswCKnrL7gXezdB+Ju6DvRrum22lTNb5KvBohnmrOZkofP/phgLfe88F9+2zozf9FTDEZx3FcB+qtb1pBbpms29v+H6oZVj2C943bdyH+ZM+y5rivjFGZPce+Lx2TA7v8WfA7d7zzviXCGr6LJ8PDPCerwcu9Vk2NOP6fJY9CPySQ2xvA2/4TPcGVmXTfi/Q0ifuuTmsfxQwzXs+EPg9i3Yn3gNv+mxcAozymTcQmO09HwxszrCOwZxMBF2BNbikVCyTfc4uEawG+gbi/60oPwrbOVGTe1eqajnch1RjoLI3vzZwtYjsS38AF+OSQC1gj6ruzWR9tYE7M7yuFu40SEafAO1FpBrQEZdcfvRZzzifdezBJYsaPq/fks1+7fZizUw1b3lm69mE+2Zfmezfg0xjEJFeIvKLiOzx2vfm5HvqrwSf50eA9A786hm2l93+J5L1/vuzLUTkLhFZKSL7vX2pwKn7knHfG4nIF96FBweAx33a18KdbvFHbdzvYLvP+z4Bd2SQ6bZ9qer3uNNS44GdIjJRRMr7ue3cxGk8lgiKCFX9Afdt6Rlv1hbct+GKPo8yqvqkt+wsEamYyaq2AP/O8LrSqjo5k23uBb4BrgWuw32DV5/13JxhPVGq+rPvKrLZpe+AdiJSy3emiLTD/bN/7zPbt00M7pTH7hzeg9NiEJFSuOT2DHC2qlYEZuASWE7x+mM77pRQZnFnNAuoKSJxZ7IhEemA64O4BnfkVxHYz8l9gdP351VgFdBQVcvjzrWnt98C1MticxnXswV3RFDZ530vr6qx2bzm1BWqvqiqbXBHeI1wp3xyfJ237fo5tDEZWCIoWl4AuotIS1wn4OUicqmIRIhIpHf5Y01V3Y47dfOKiFQSkRIi0tFbx+vALSLSzruSpoyIXCYi5bLY5gfAX4CrvOfpXgPuF5FYABGpICJX+7sjqvod7sPwExGJ9fbhAm+/XlXVtT7NbxCRpiJSGhgDTFXV1Ozegyw2WxIoBewCUkSkF+B7SeMOIFpEKvi7Hxl8hHtPKolIDeDWrBp6+/cKMNmLuaQX/wARuc+PbZXDnYffBRQXkYeAnL5Vl8N1zh4SkcbAcJ9lXwDVRGSUd1lvOS8pg3tf6qRfdeX9fX0DPCsi5UWkmIjUF5FOfsSNiJzv/f2VAA7jLhpI89lWVgkJ3CnFR0Wkoff320JEov3ZbjizRFCEqOou4B3gIVXdguuwfQD3YbAF960q/Xd+I+6b8ypc5/Aobx0LgJtwh+Z7cR2+g7PZ7HTcFS4JqrrEJ5ZpwFPAFO80wzKgVy53qT8wG/ga1xfyHu5KlJEZ2r2LOxpKwHVk3ubFkNN7cApVPei99iPcvl/n7V/68lXAZGC9d8ojs9Nl2RkDxAMbcEc8U3HfnLNyGydPkezDnfLoB3zux7Zm4t63NbjTZUfJ/lQUwF24fT6I+0LwYfoC773pDlyOe5/XAl28xR97PxNFZJH3/C+4xLoC915Oxb9TXeAS1uve6zbhTpON9Za9CTT13v/PMnntc7jf3ze4pPYmrjPaZENOHskbE3pEZA6uozIod/fmhYgMx3Uk+/VN2ZhAsSMCYwqIiFQTkYu8UyXn4i7FnBbsuIyxO/qMKTglcVfP1MWd6pmC6wcwJqjs1JAxxoQ5OzVkjDFhLuRODVWuXFnr1KkT7DCMMSakLFy4cLeqVslsWcglgjp16rBgwYJgh2GMMSFFRDZltcxODRljTJizRGCMMWHOEoExxoQ5SwTGGBPmLBEYY0yYC1giEJG3RGSniCzLYrmIyIsisk5E/hCR1oGKxRhjTNYCeUTwNm4806z0wlWtbIgbi/TVAMZijDEmCwG7j0BV54pInWya9AXe8QYy+UVEKopINa+WuTHGhLwZS7ezavuBPK+n+NEjlN63h/O7tKFlrczGk8rj+vN9jf6rwan10eO9eaclAhEZhjtqICYmpkCCM8b4Z+fBo3y1NIG9R44HO5RC54Xv3PhJIjk0zEb7jUt44uuXOFiqDIs//bbIJQK/qepEYCJAXFycVckzJsiOpaTy/cqdTF0Yz5w1u0hNs3/LrDzerznXtTuDL7D79sHdd8OUN6BBA3jjDZpdWDf/AyS4iWArp47ZWtObZ4wJkK37kvjs963kpepwwoGjfPHHdvYdSebs8qUY1rEe/VvXpH6VMvkYadEhZ3I4kJoKF14Iq1fDPffA6NEQFbiB1oKZCKYDt4rIFKAdsN/6B4zJP7NX72TFtlPPT7/8/TqSklPztN6SxYtxaew5XNWmJhc3qExEsTyc9zCnSkyEs86CiAj497+hVi2Iiwv4ZgOWCERkMtAZqCwi8cDDQAkAVX0NmAH0xo2JewT4a6BiMaYoO5qcypT5m0lKTjtl/lNfr8q0fb0qZZg5quMZb6+YiH345zdVeP99uP12ePJJuOkm6NevwDYfyKuGBuawXIG/B2r7xoSLeesTGf35ikyX3X3puQztcOp55RLFilHMPsgLjy1b4JZbYMYMuOACuOiiAg8hJDqLjTFZS0115/un3tKeZjUqnLIsskREMEIy/po8GW6+2fUJvPAC3HqrOy1UwCwRGBOivl+1gw27j7Am4SAApYpH2Ad/qKlUCdq1g4kToW5grgjyhyUCY0LULe8u4niq6xcoWbwY0WVLBjkik6OUFHj+eTh+HB58EHr2hEsvzduNBvnAEoExISo5LY2bO9ZjRJcGlCpezI4GCrslS2DIEFi4EK65xnUQiwQ9CYBVHzUmpJUqXowKUSUsCRRmx47Bv/7lLgPdsgU+/himTCkUCSCdHREYU0jNWrmDhANHs1yeh3vCTEFauxaeegquuw6eew6io4Md0WksERhTCB04msyQSQtybHd2hcgCiMbk2qFD8N//wvXXQ7NmsGoV1KsX7KiyZInAmELo8LEUAO7s3ohrz6+VaZtixYTKZUsVZFjGH99+C8OGwaZN0Lo1NGlSqJMAWCIwJihUle9W7jytYqeqMn/DXr5a5qqt1K5chqrl7Vt/SNi7F+66C956Cxo1gh9+cEkgBFgiMKYArdx+gOXbDrBlzxHGzVqbaZtypYrTt1V1+reuSZvalQo4QnNGUlPdHcFr1sD998NDD0Fk6CRwSwTGBEBamvLdyh0cPp5yyvx/fLjklOmn+7fgwgandh5WLlvKrgIKFbt3nywS9/jjEBPjTgeFGEsExuTRmh0HWZlhFKo/4vfz5v82ZNq+W+OqjL4illLFi9lpn1ClCu++C6NGuSJxw4bBlVcGO6ozZonAmFxYtHkv8XuTANh18Bif/b6VpVv3Z9l+/HWtia1e/pR5tc4qbdU7Q9mmTa4+0MyZbsyAjmdeybWwsERgjJ/S0pQBE345UdYBoGm18jzUpykdGp5el790yeKcY5d3Fi3vvQfDh7sjgpdeghEjoFjo35dricCENVVl2dYDbNl7JMe2aaocT01jUPva3Ni+DpElilGzUukCiNIUGlWquE7hCROgdu1gR5NvLBGYsLTzwFGm/b6VqQvjWbvzUK5eW69KWRpULRugyEyhkpwMzz7rfv7rX65AXI8ehao8RH6wRGCKjLQ05Zf1iRw6lpJlm4NHU/jij238sGYXaQptalfiif9rznkxFRFy/ueOKAb1KlsSCAu//+6KxP3+OwwYUKiKxOU3SwQmpKkqv2/Zx55Dx1kSv4+Xvl+X42uqVYhkeOf69G9dk3pV7EPdZHD0KIwZA08/DZUrwyefwP/9X7CjCihLBCZkpKSmMX/DHo6mpKIKy7Ye4JNF8Wzec+r5/aevakHTauUzXUfxCKFh1XJ21Y7J2rp18Mwz8Je/uNNClYr+TX2WCEzI+HHtbv769m+nzLuwfjS3d2tIo7PLAVCmVIR9yze5d+gQTJsGN97oisStXh3UEcMKmiUCEzKOHE8F4MWB51H7rNJULV+KahWighyVCXkzZ7obwrZscWMGNGkSVkkAbGAaEwLW7TzE/9buZlWCu3u38TnlaFmroiUBkzeJiTBokBsusnRp+PHHkCkSl9/siMAUSnsPH2fl9gOkpCl/eWv+KcvKlLI/W5NH6UXi1q1zYwf/858hVSQuv9l/lCmUHpi2lK+WJZyYvqxFNQZfWIeKUSWoUdGOBMwZ2rXLjRAWEeFGDatdG1q1CnZUQWeJwBQ6qWnK3iPHaVC1LP++shnFI4QWNStSIsLOZJozpApvvw133OGKxN18M/TtG+yoCg1LBCboVicc5LeNe1ix/QArth1gVcIBjiancWH9aNrVK3zju5oQs3Gj6wz+9lvo0AG6dAl2RIWOJQITFHsPH2f97kPsT0rmb2+7sXnLRxanafXyXNe2Nk2rl6djw8pBjtKEvHffdUXiROCVV9yRQBEoEpffLBGYoBjx/iLmrU88Mf14v+YMbFsLKYK375sgOvtsVyb6tdfcoDEmU5YITFAcOpZCy1oVubN7I0pEFCOuTiVLAibvkpNdaYjUVDdcZI8e7mGyZYnAFJjUNGVVwgHS0uDI8RRqR5ehY6MqwQ7LFBWLFsHf/gZLlsB1150sEmdyZInAFJh35m3kkc9XnJhunEU9IGNyJSkJHnnE1QeqUsWVigjhYSODIaCJQER6AuOACOANVX0yw/IYYBJQ0Wtzn6rOCGRM5swdS0nlz52Hz/j1f+5ydf8n3tiGYiK0rFUxv0Iz4Wz9enjuORg8GMaODYsicfktYIlARCKA8UB3IB74TUSmq+oKn2b/BD5S1VdFpCkwA6gTqJhM3jz6xQre+2VzntZRqngxLmlyNsWs+qfJiwMH4NNP3Yd/bCysXVukRgwraIE8ImgLrFPV9QAiMgXoC/gmAgXSzw9UALYFMB6TR/uTUqharhRj+jY743XUrBRlScDkzYwZcMstsHUrtGvn6gNZEsiTQCaCGsAWn+l4oF2GNqOBb0RkJFAGuCSzFYnIMGAYQIxdAlbgfl2fyMcL4/luxQ5qR5emZ7Nzgh2SCUe7d8M//uEGkG/aFH76KWyLxOW3YHcWDwTeVtVnRaQ98K6INFPVNN9GqjoRmAgQFxenQYgz7Ow7cpw9h4/zw5pdPPL5CsqWKs4VLasz6MI6wQ7NhKP0InHr17vLQh94AEqVCnZURUYgE8FWoJbPdE1vnq8hQE8AVZ0nIpFAZWBnAOMyOUhOTePip2afMvbvl7ddTO3oMkGMyoSlHTvclUAREe6qoNq1oUWLYEdV5ATyXuvfgIYiUldESgIDgOkZ2mwGugGISBMgEtgVwJiMH1JSlUPHUri8ZXXGDWjFpL+1tSRgCpYqvPkmnHsuTJzo5l1+uSWBAAnYEYGqpojIrcBM3KWhb6nqchEZAyxQ1enAncDrIvIPXMfxYFW1Uz+FRGz18vRtVSPYYZhws3493HQTfP89dOoEl2TadWjyUUD7CLx7AmZkmPeQz/MVwEWBjMH472hyKjOXJzB1YTwAxe3qHlPQJk2CESPcqaDXXnMJwYrEBVywO4tNkKkqizbvZerCeL5Ysp2Dx1KoUTGK27o24Oq4WjmvwJj8VL06dO0Kr74KNWsGO5qwYYkgTG3fn8Sni7YydWE8G3YfJqpEBL2an8NVrWtyQb1ou9bfFIzjx91AMWlpMHo0dO/uHqZAWSIIA3sOHyc17WTXy/6kZC557gcA2tU9ixGd69OreTXK2ljApiD99psrErdsGdx4oxWJCyL7zy/i3v91Ew9OW5bpsif+rzkD29oNeqaAHTni7gV4/nmoVg2mT3dXBJmgsURQxO3YfxSAR688tSxEqYhiXNaiWjBCMuFuwwZ46SXXEfzUU1ChQrAjCnuWCIqQlNQ0Dh9PPWXesZQ0RODGC6wWiwmi/ftdkbi//tUViVu3DmrZxQiFhSWCIuSaCfNYtHnfafNLRtjldyaIvvzSjRW8fTu0bw+NG1sSKGQsERQhW/cl0apWRS5vWf2U+XUrlw5SRCas7doFo0bBBx9As2buiKBx42BHZTJhiaCIaXxOOYZcXDfYYZhwl5oKF1/s+gMeeQTuuw9Klgx2VCYLlgiMMfknIQGqVnV3Bj/7LNSp444GTKHm98ljEbHzC8aYzKWlwYQJ0KiR+wnQp48lgRCRYyIQkQtFZAWwyptuKSKvBDwyk620NCU1w8PK9ZmgWLcOunVzo4adfz5cemmwIzK55M+poeeBS/FKSKvqEhHpGNCoTLbW7DjIFS//j6PJaacts9IQpkD95z+uSFzJkvD66zBkiN0dHIL86iNQ1S1y6i83Nau2JvC27UviaHIaA9vWolqFqBPzBehtN4mZghQT444Axo+HGlayPFT5kwi2iMiFgIpICeB2YGVgwzL+uDquFq1jKgU7DBNOjh2DJ55wfQJjxrhTQt26BTsqk0f+JIJbgHG4wei3At8AIwIZlMnc5S/9jxXbD5DmdQYUs0NwU5B+/dWd+lm+HAYNsiJxRYg/ieBcVb3ed4aIXAT8FJiQTFZWJxykZc0KXFi/MuUiixNbvXywQzLh4PBh+Ne/4IUX3OmfL76Ayy4LdlQmH/mTCF4CWvsxz+RBSmoaPV6YS/zepCzbHE9No339aO669NwCjMyEvU2b4JVX3FVBTz4J5e0LSFGTZSIQkfbAhUAVEbnDZ1F53BjEJh8lJaeyftdh2teLpmWtipm2KSZwjY0aZgrCvn0wdSoMHQpNm7pLRG3EsCIruyOCkkBZr005n/kHgKsCGVQ469akKkM71At2GCac/fe/MHw47NzpykQ0bmxJoIjLMhGo6g/ADyLytqpuKsCYjDHBsHMn3HYbfPghtGjhBoyxInFhwZ8+giMiMhaIBSLTZ6pq14BFFUae+Golk3/djN0UbIIqNRUuugg2b4bHHoN77oESJYIdlSkg/iSC94EPgT64S0kHAbsCGVQ4Wbx5H1ElI+jVrBolIoReze2GMFOAtm2Dc85xReLGjXNF4po2DXZUpoD5kwiiVfVNEbnd53TRb4EOrChKSU3jhzW7+GRRPPM37CFN4UBSMnF1KjH6ithgh2fCSXqRuHvvdVcCjRgBvXsHOyoTJP4kgmTv53YRuQzYBpwVuJCKpoNHk+n5wo9s3ZfEWWVK0rVxVaJKuIuvujWpGuToTFhZs8aNFzx3LlxyCfTqFeyITJD5kwgeE5EKwJ24+wfKA6MCGlURk5qmfLQgnq37khjZtQEjuzakZHEbPtIEwZtvwq23QmQkvPUWDB5sdwebnBOBqn7hPd0PdIETdxYbP/xv7W4e+3IFqxIO0jqmIkMvrmdJwARPnTruCGD8eKhm/VHGye6GsgjgGlyNoa9VdZmI9AEeAKKA8womxNC0budBHp+xiu9X7aTWWVGMv641vZufg9i3L1OQjh2DRx91zx97zIrEmUxld0TwJlALmA+8KCLbgDjgPlX9rCCCCyW7Dx3jhjd+5eDRFAASDhyldIkIHujdmEEX1qFUcbsZ2xSwn392ReJWrYK//c2KxJksZZcI4oAWqpomIpFAAlBfVRMLJrTQsmXPEVYlHOSiBtGcUz6Ks8uXYsjFdYkuWyrYoZlwc+gQPPggvPQS1KoFX39to4aZbGWXCI6rahqAqh4VkfW5TQIi0hNXwjoCeENVn8ykzTXAaECBJap6XW62UdgM7VCPLufaVUAmiDZvdpeG/v3v8PjjUK5czq8xYS27RNBYRP7wngtQ35sWQFW1RXYr9voYxgPdgXjgNxGZrqorfNo0BO4HLlLVvSJin6DGnIm9e+Hjj2HYMHdD2Pr1UL16sKMyISK7RNAkj+tuC6xT1fUAIjIF6Aus8GlzEzBeVfcCqOrOPG6zwN3/6R/M37An0/GDjSkQ06a5G8J27YJOneDccy0JmFzJruhcXgvN1QC2+EzHA+0ytGkEICI/4U4fjVbVrzOuSESGAcMAYmJi8hhW/vp2xQ5KlyxOq5iKXFg/mvOyKCFtTL5LSICRI1256Fat4MsvXRIwJpf8Grw+wNtvCHQGagJzRaS5qu7zbaSqE4GJAHFxcYWuPluHhpX5d7/mwQ7DhJPUVOjQAbZscf0Ad91lReLMGQtkItiKu/w0XU1vnq944FdVTQY2iMgaXGIo9LWMjhxP4aulCScuFzWmQMTHu9M+ERHw4otQt66VijZ55tctriISJSK5Peb8DWgoInVFpCQwAJieoc1nuKMBRKQy7lTR+lxup8D9d/FWzn/sO+78eAlnl4+kVzO7Q9MEWFqauxy0cWN49VU3r1cvSwImX+R4RCAilwPP4EYsqysirYAxqnpFdq9T1RQRuRWYiTv//5aqLheRMcACVZ3uLeshIiuAVODuULhP4Zf1iYgIH93cnvPrVLK7hU1grVrlhoz86Sd3P0CfPsGOyBQx/pwaGo27AmgOgKouFpG6/qxcVWcAMzLMe8jnuQJ3eI+QUrpkBG3rWhFWE2BvvOGKxJUuDZMmwY032t3BJt/5VYZaVfdn+NZb6DpsjSmS6teHyy+Hl1+Gs88OdjSmiPInESwXkeuACO8GsNuAnwMbVuGzfX8Sd328hKTjqWzec4Ri9q3MBMLRozBmjHv++OPQpYt7GBNA/nQWj8SNV3wM+ABXjjrsxiNYse0AP61LJFWhSbXyXN+udrBDMkXNTz+5+wGeeMLdHKZ24G0Khj9HBI1V9UHgwUAHEwoe7RtLi5p205jJRwcPwgMPuDECateGmTOhR49gR2XCiD9HBM+KyEoReVREmgU8ImPCTXy86xQeORKWLrUkYApcjolAVbvgRibbBUwQkaUi8s+AR2ZMUZaYePJ+gCZNXJG4ceOgbNngxmXCkl83lKlqgqq+CNwCLAYeyuElxpjMqLraQE2bwm23werVbr4NG2mCKMdEICJNRGS0iCzFDV7/M65chDEmN7Zvh/794eqr3YAxCxZYkThTKPjTWfwW8CFwqapuC3A8xhRN6UXitm6Fp5+Gf/wDige75qMxTo5/iaraviACMaZI2rIFatRwReLGj3dF4ho1CnZUxpwiy1NDIvKR93OpiPzh81jqM3KZMSYzqamuOqhvkbhLL7UkYAql7I4Ibvd+WoUrY3Jj5UoYMgTmzXMVQi+/PNgRGZOt7EYo2+49HaGq9/ouE5GngHtPf1XR89w3q1m94yA7Dx4LdigmFEyc6O4HKFcO3n0Xrr/eisSZQs+fy0e7ZzKvV34HUli9PHsdCzbuJel4KufXqUTts8oEOyRTmDVsCP36wYoVcMMNlgRMSMjyiEBEhgMjgHoZ+gTKAT8FOrDC5Lp2MdzZwy7zM5lISoLRo90H/pNPWpE4E5Ky6yP4APgKeAK4z2f+QVXdE9CojAkFc+e6AWPWroVbbnE3i9kRgAlB2Z0aUlXdCPwdOOjzQERsRBYTvg4cgBEjoFMnd3XQrFnuyiBLAiZE5XRE0AdYiBuIxvevXIF6AYzLmMJr2zZ4+2244w43dkAZ6zcyoS27q4b6eD/9GpbSmCJt92746CN3JNC4MWzYYCOGmSLDn1pDF4lIGe/5DSLynIjEBD40YwoBVfjwQ1ckbtQoWLPGzbckYIoQf4qdvAq0FJGWwJ3AG8C7QKdABhZMP63bzfTFrqxSmg0SFb62bYPhw2H6dIiLc30BdmewKYL8SQQpqqoi0hd4WVXfFJEhgQ4smCb9vJHvV+2kctlS1KgYZSOShaPUVOjY0WWJyQMAAB6ASURBVBWJe+YZuP12KxJniix//rIPisj9wI1ABxEpBpQIbFjB16BqWb4e1THYYZiCtmkT1KzpisS98grUqwcNGgQ7KmMCyp87i6/FDVz/N1VNwI1FMDagURlT0FJT4bnn3Ghh6UXievSwJGDCgj9DVSYA7wMVRKQPcFRV3wl4ZMYUlGXL4MIL4c47oVs3uPLKYEdkTIHy56qha4D5wNXANcCvInJVoAMzpkC89hq0bu3GDP7gA9cxXNMG4DPhxZ8+ggeB81V1J4CIVAG+A6YGMrCCtvvQMd74cQPHU9JYlXCQ0iUjgh2SCaT0chBNmrihI194AapUCXZUxgSFP4mgWHoS8CTi56D3oWTO6l289sOflCkZQTERLqxvg4kXSUeOwEMPuc7gp55yZSI6FdkroY3xiz+J4GsRmQlM9qavBWYELqTgUHU3DHw9qiO1ziod5GhMQMyZ44rE/fmnu0PYisQZA/jXWXw3MAFo4T0mZhyoxphCbf9+uPnmk+Whv//ejR9sScAYIPvxCBoCzwD1gaXAXaq6taACMybfbN8O770Hd90FjzwCpe2Izxhf2R0RvAV8AfTHVSB9KbcrF5GeIrJaRNaJyH3ZtOsvIioicbndhjGZ2rULXvL+ZBs3ho0bYexYSwLGZCK7RFBOVV9X1dWq+gxQJzcrFpEIYDxuWMumwEARaZpJu3LA7cCvuVm/MZlSdZeBNmni7gtILxJnVwQZk6XsEkGkiJwnIq1FpDUQlWE6J22Bdaq6XlWPA1OAvpm0exR4Cjia6+iN8bVlC1x+uRswvkED+P13KxJnjB+yu2poO/Ccz3SCz7QCXXNYdw1gi890PNDOt4GXUGqp6pcicndWKxKRYcAwgJgYq4BtMpGSAp07Q0ICPP88jBzpLhE1xuQou4FpAjoCt1e87jlgcE5tVXUiMBEgLi7OCkObkzZuhFq1XGXQCRNckbh6NnieMbkRyBvDtgK1fKZrevPSlQOaAXNEZCNwATA9WB3Gll1CTEqKKw/dpImrEgpwySWWBIw5A4EssP4b0FBE6uISwADguvSFqrofqJw+LSJzcJeoLghgTJk6cjyF93/ZRFSJCCqWLvIVtkPfH3/AkCGwYAH07Qv9+wc7ImNCWsCOCFQ1BbgVmAmsBD5S1eUiMkZErgjUdnMrJTWNv7+/iKVb9/PSwPMoF2mJoFB75RVo08aNG/DhhzBtGlSvHuyojAlpOR4RiIgA1wP1VHWMN17xOao6P6fXquoMMpSjUNWHsmjb2a+I89m//ruM2at38Xi/5lzS1MahLbTSy0E0awYDBrgO4cqVc36dMSZH/hwRvAK0BwZ60wdx9weEvANHk5k8fws3XlCb69rZ1UiF0uHD8I9/wD33uOmOHeHddy0JGJOP/OkjaKeqrUXkdwBV3SsiJQMcV0At37af+Rv2cOR4KgD1qpQJckQmU7NmwU03wYYN7nJQKxJnTED4kwiSvbuEFU6MR5AW0KgC7LEvVjJvfeKJ6WoVooIYjTnNvn2uLtCbb0LDhjB3LnToEOyojCmy/EkELwLTgKoi8m/gKuCfAY0qwFLS0mhb5ywm/qUNEcXEOogLmx07YMoUuPdeePhhiLJEbUwg5ZgIVPV9EVkIdAMEuFJVVwY8sgArHiFULB3SZ7iKlvQP/9tvh3PPdTeKWT+AMQXCnzGLY4AjwOfAdOCwN8+YvFN1JaKbNnUdwmvXuvmWBIwpMP6cGvoS1z8gQCRQF1gNxAYwLhMONm+GW26Br76C9u1P9gkYYwqUP6eGmvtOe4XiRgQsIhMe0ovE7dwJL77oho60InHGBEWuS0yo6iIRaZdzS2MysX491K7tisS9/jrUrw916gQ7KmPCmj93Ft/hM1kMaA1sC1hEpmhKSYFnn3VXAT39NNx2G3TrFuyojDH4d0RQzud5Cq7P4JPAhGOKpMWLXZG4RYugXz+4+upgR2SM8ZFtIvBuJCunqncVUDymqHn5ZVciIjoapk61SqHGFEJZXj4qIsVVNRW4qADjMUWFeiM8tGjhho5cscKSgDGFVHZHBPNx/QGLRWQ68DFwOH2hqn4a4NhMKDp0CB58EEqUcAPHdOzoHsaYQsuf6qORQCJujOI+wOXeT2NO9c03rkz0Sy9BcvLJowJjTKGW3RFBVe+KoWWcvKEsnf2Hm5P27oU77oC333blIebOhYsvDnZUxhg/ZZcIIoCynJoA0lkiMCft3Ok6gu+/Hx56CCIjgx2RMSYXsksE21V1TIFFYkJLQgJMnuyuCEovEhcdHeyojDFnILs+AhsBxJxOFSZNckXi7r//ZJE4SwLGhKzsEoHd9mlOtXEj9OwJgwe7RLB4sRWJM6YIyPLUkKruKchATCGXkgJdusDu3TB+vKsaWsyfi86MMYVdrovOmTCzbh3UreuKxL31FtSr54rGGWOKDPtKZzKXnAyPPw6xse4IANwRgSUBY4qcsDwiOHg0hbPL2yWOWVq0yBWJW7zYFYi79tpgR2SMCaCwOyLYceAoqxIO0rbuWcEOpXB68UVo29ZdHvrpp/DRR3D22cGOyhgTQGGXCGav2glAtyZVgxxJIZNeDuK88+Avf3FF4vr1C25MxpgCEXanhmat2kmNilGce3a5nBuHg4MH3f0ApUq5gWM6dHAPY0zYCKsjgqPJqfxv7W66Nq6KiN0vx9dfuyJxr7zijgisSJwxYSmsEsEv6xNJSk6100KJiTBoEPTqBWXKwE8/wXPPgSVHY8JSWCWC71ftJKpEBBfUC/NyCImJMG0a/Otf8Pvv0L59sCMyxgRRQBOBiPQUkdUisk5E7stk+R0iskJE/hCRWSISsIvUVZVZK3dyccPKRJaICNRmCq/t291AMarQqBFs2gRjxri+AWNMWAtYIvDGOx4P9AKaAgNFpGmGZr8DcaraApgKPB2oeNbsOMTWfUl0axxmp4VU3R3BTZq4I4B169z8SpWCG5cxptAI5BFBW2Cdqq5X1ePAFKCvbwNVna2qR7zJX4CagQpm3p+7AejYqEqgNlH4bNgAPXq4m8NatoQlS6xInDHmNIG8fLQGsMVnOh5ol037IcBXmS0QkWHAMICYmJgzCuZYShoAFUuXOKPXh5yUFOja1fUHvPoqDBtmReKMMZkqFPcRiMgNQBzQKbPlqjoRmAgQFxdn1zhmZ+1aVxiueHH4z3+gfn2oVSvYURljCrFAfkXcCvh+AtX05p1CRC4BHgSuUNVjAYynaEtOhscec/cFvPyym9e5syUBY0yOAnlE8BvQUETq4hLAAOA63wYich4wAeipqjsDGEvRtmCB6wf44w8YMAAGDgx2RMaYEBKwIwJVTQFuBWYCK4GPVHW5iIwRkSu8ZmOBssDHIrJYRKYHKp4ia9w4aNfODRjz3/+6cYSrhtmVUcaYPAloH4GqzgBmZJj3kM/zSwK5/SJN1d0JHBfnjgaefhoqVgx2VMaYEFQoOotNLhw4APfeC5GR8PzzcNFF7mGMMWfIricMJTNmuBHDJk50VwVZkThjTD6wRBAKdu+GG26Ayy6DChXg559h7FgrEmeMyReWCELB3r3w+efw8MNuGMl22d2XZ4wxuWN9BIXV1q3w/vtw992uLMSmTdYZbIwJCDsiKGxU4fXXoWlTGD0a/vzTzbckYIwJEEsEhcmff0K3bq4uUOvW7gaxBg2CHZUxpoizU0OFRUqKSwJ79sCECTB0qBWJM8YUCEsEwbZ6tSsMV7w4TJrkntcMWDVuY4w5jX3lDJbjx+GRR6B5cxg/3s3r1MmSgDGmwNkRQTDMn+/KQixbBtddB9dfH+yIjDFhzI4ICtoLL7jB4tPvDXj/fahcOdhRGWPCmCWCgpJeDqJtW7jpJli+HPr0CW5MxhiDnRoKvP374Z57ICrKHQ1ceKF7GGNMIWFHBIH0+efuxrA33oBSpaxInDGmULJEEAi7drlO4CuugOho+OUXeOopKxJnjCmULBEEwv79rmT0I4+4YSTPPz/YERljTJasjyC/bNkC770H993nykJs2uRKRhtjTCFnRwR5lZYGr73mBox57LGTReIsCRhjQoQlgrxYuxa6doXhw91loUuXWpE4Y0zIsVNDZyolBbp3h3374M034a9/tc7gMJCcnEx8fDxHjx4NdijGZCoyMpKaNWtSokQJv19jiSC3Vq50A8UULw7vvuuKxFWvHuyoTAGJj4+nXLly1KlTB7HEbwoZVSUxMZH4+Hjq1q3r9+vs1JC/jh1zQ0W2aAEvv+zmdehgSSDMHD16lOjoaEsCplASEaKjo3N9xGpHBP745RdXJG7FCrjxRvcwYcuSgCnMzuTv044IcvLss64kxMGD7t6Ad95xN4kZY0wRYYkgK2lp7mf79nDLLa5kdK9ewY3JGGMCwBJBRvv2udNAt9/upi+8EF55BcqXD25cxnhEhBtuuOHEdEpKClWqVKFPLqvZ1qlTh927d+epTUREBK1ataJZs2Zcfvnl7Nu3D4CNGzcSFRVFq1atTjyOHz+e6TpGjRpFjRo1SEv/8gWMHj2aZ555JstYEhISGDBgAPXr16dNmzb07t2bNWvW+LXfwTZp0iQaNmxIw4YNmTRpUqZtFi9ezAUXXECrVq2Ii4tj/vz5J5bNmTOHVq1aERsbS6dOnfIlJusj8PXZZzBiBOzc6SqGqtoloSZLj3y+nBXbDuTrOptWL8/Dl8dm26ZMmTIsW7aMpKQkoqKi+Pbbb6lRo0a+xuGvqKgoFi9eDMCgQYMYP348Dz74IAD169c/sSwraWlpTJs2jVq1avHDDz/QpUuXHLepqvTr149BgwYxZcoUAJYsWcKOHTto1KhRrvchNTWViIiIXL/uTOzZs4dHHnmEBQsWICK0adOGK664gkqVKp3S7p577uHhhx+mV69ezJgxg3vuuYc5c+awb98+RowYwddff01MTAw7d+7Ml7jsiADcB/8110C/fnD22W4EsccftyRgCq3evXvz5ZdfAjB58mQGDhx4YtmePXu48soradGiBRdccAF//PEHAImJifTo0YPY2FiGDh2K+lTDfe+992jbti2tWrXi5ptvJjU1NdcxtW/fnq1bt+bqNXPmzCE2Npbhw4czefJkv14ze/ZsSpQowS233HJiXsuWLenQocNpba+88kratGlDbGwsEydOPDG/bNmy3HnnnbRs2ZJ58+Zluf/Dhw8nLi6O2NhYHn744VztW2ZmzpxJ9+7dOeuss6hUqRLdu3fn66+/Pq2diHDggPuSsX//fqp7Vyd+8MEH/N///R8xMTEAVK1aNc8xAS67htKjTZs2eiZem7NOa9/7hR4+lnz6wrVrVStWVP33v1WPHz+j9ZvwsGLFimCHoGXKlNElS5Zo//79NSkpSVu2bKmzZ8/Wyy67TFVVb731Vh09erSqqs6aNUtbtmypqqojR47URx55RFVVv/jiCwV0165dumLFCu3Tp48e9/72hw8frpMmTVJV1dq1a+uuXbuyjUVVNSUlRa+66ir96quvVFV1w4YNGhkZqS1bttSWLVvqiBEjMn390KFD9Z133tH9+/dr9erVT8Tw8MMP69ixY09pmx7LuHHjdNSoUX69V4mJiaqqeuTIEY2NjdXdu3erqiqgH374oapqtvuf/vqUlBTt1KmTLlmy5LRtPP300yf20/cxcuTI09qOHTtWH3300RPTY8aMOW0/02OqVauW1qxZU6tXr64bN25UVdXbb79dR4wYoZ06ddLWrVufiDOz12cELNAsPlfD99TQ5s3uhrAHHnBlITZvhnLlgh2VMX5p0aIFGzduZPLkyfTu3fuUZf/73//45JNPAOjatSuJiYkcOHCAuXPn8umnnwJw2WWXnTgdMWvWLBYuXMj5XpXcpKQkv79pJiUl0apVK7Zu3UqTJk3o3r37iWU5nRo6fvw4M2bM4LnnnqNcuXK0a9eOmTNn0qdPnywvgcztpZEvvvgi06ZNA2DLli2sXbuW6OhoIiIi6N+/P5D9/n/00UdMnDiRlJQUtm/fzooVK2jRosUp27j77ru5++67cxVXTl599VWef/55+vfvz0cffcSQIUP47rvvSElJYeHChcyaNYukpCTat2/PBRdccEanxHwFNBGISE9gHBABvKGqT2ZYXgp4B2gDJALXqurGQMZEWprr/L33Xvf82mtdIrAkYELMFVdcwV133cWcOXNITEw84/WoKoMGDeKJJ57I9WvT+wiOHDnCpZdeyvjx47ntttv8eu3MmTPZt28fzZs3B+DIkSNERUXRp08foqOj2b59+yntDx48SMWKFYmNjWXq1Kk5rn/OnDl89913zJs3j9KlS9O5c+cTN1pFRkae6BfIav83bNjAM888w2+//UalSpUYPHhwpjdqjR07lvfff/+0+R07duTFF188ZV6NGjWYM2fOien4+Hg6d+582msnTZrEuHHjALj66qsZOnQoADVr1iQ6OpoyZcpQpkwZOnbsyJIlS/KcCAJ2Cgf34f8nUA8oCSwBmmZoMwJ4zXs+APgwp/Xm5dRQl6GvacrFF6uCavfuqhs2nNG6TPgqLKeGVFW3bNmi48aNU1U95dTQyJEjdcyYMSfmt2rV6sT89NMSM2bMOHFqaPny5dqgQQPdsWOHqrrTIemnIvw9NaSqumjRIo2JidHk5GTdsGGDxsbGZrsfAwcO1A8++ODE9KFDh7RKlSp6+PBhXbJkiTZr1kwPHDigqqqffPKJdunSRVVV09LStG3btjphwoQTr12yZInOnTv3lPV/9tln2qdPH1VVXblypZYqVUpnz559WtxZ7f/ixYu1RYsWmpqaqgkJCVq1alX9z3/+k+0+5SQxMVHr1Kmje/bs0T179midOnVOnH7y1bhx4xOxfvfdd9q6dWtVdX9/Xbt21eTkZD18+LDGxsbq0qVLT3t9bk8NBTIRtAdm+kzfD9yfoc1MoL33vDiwG5Ds1numiWDCd6t0S/mqmlaxoup//qOalnZG6zHhrTAlAl++iSAxMVH79u2rzZs313bt2p04r717927t3r27Nm3aVIcOHaoxMTEnPuSnTJmiLVu21ObNm2vr1q113rx5qpq7RKCq2qdPH33nnXdyTASHDx/WSpUq6f79+0+Z369fP50yZYqqqr722mvaokULbdmypXbv3l3//PPPE+22bt2qV199tdarV0+bNm2qvXv31jVr1pyyrqNHj2rPnj21cePG2rdvX+3UqVOmiSC7/R80aJA2bNhQu3btqv369ctzIlBVffPNN7V+/fpav359feutt07MHzJkiP7222+qqvrjjz9q69attUWLFtq2bVtdsGDBiXZPP/20NmnSRGNjY/X555/PdBu5TQTiluc/EbkK6KmqQ73pG4F2qnqrT5tlXpt4b/pPr83uDOsaBgwDiImJabNp06Zcx/PN8gSWfvglfx96KZExNc90t0yYW7lyJU2aNAl2GMZkK7O/UxFZqKpxmbUPic5iVZ0ITASIi4s7o8zVI/YceowZkq9xGWNMURDIRLAVqOUzXdObl1mbeBEpDlTAdRobYwqJxMREunXrdtr8WbNmEW11t4qEQCaC34CGIlIX94E/ALguQ5vpwCBgHnAV8L0G6lyVMflEVcOqAml0dHSOdwibwuNMPkIDdmexqqYAt+I6hFcCH6nqchEZIyJXeM3eBKJFZB1wB3BfoOIxJj9ERkaSmJh4Rv9sxgSaqhuYJjIyMlevC1hncaDExcXpggULgh2GCVM2VKUp7LIaqjLkO4uNKSxKlCiRqyEAjQkFVnTOGGPCnCUCY4wJc5YIjDEmzIVcZ7GI7AJyf2uxUxlXxiKc2D6HB9vn8JCXfa6tqlUyWxByiSAvRGRBVr3mRZXtc3iwfQ4PgdpnOzVkjDFhzhKBMcaEuXBLBBNzblLk2D6HB9vn8BCQfQ6rPgJjjDGnC7cjAmOMMRlYIjDGmDBXJBOBiPQUkdUisk5ETqtoKiKlRORDb/mvIlKn4KPMX37s8x0iskJE/hCRWSJSOxhx5qec9tmnXX8RUREJ+UsN/dlnEbnG+10vF5EPCjrG/ObH33aMiMwWkd+9v+/ewYgzv4jIWyKy0xvBMbPlIiIveu/HHyLSOs8bzWoMy1B9ABHAn0A9oCSwBGiaoc0I4DXv+QDgw2DHXQD73AUo7T0fHg777LUrB8wFfgHigh13AfyeGwK/A5W86arBjrsA9nkiMNx73hTYGOy487jPHYHWwLIslvcGvgIEuAD4Na/bLIpHBG2Bdaq6XlWPA1OAvhna9AUmec+nAt0ktEcayXGfVXW2qh7xJn/BjRgXyvz5PQM8CjwFFIW60f7s803AeFXdC6CqOws4xvzmzz4rUN57XgHYVoDx5TtVnQvsyaZJX+AddX4BKopItbxssygmghrAFp/peG9epm3UDaCzHwjlMff82WdfQ3DfKEJZjvvsHTLXUtUvCzKwAPLn99wIaCQiP4nILyLSs8CiCwx/9nk0cIOIxAMzgJEFE1rQ5Pb/PUc2HkGYEZEbgDigU7BjCSQRKQY8BwwOcigFrTju9FBn3FHfXBFprqr7ghpVYA0E3lbVZ0WkPfCuiDRT1bRgBxYqiuIRwVagls90TW9epm1EpDjucDKxQKILDH/2GRG5BHgQuEJVjxVQbIGS0z6XA5oBc0RkI+5c6vQQ7zD25/ccD0xX1WRV3QCswSWGUOXPPg8BPgJQ1XlAJK44W1Hl1/97bhTFRPAb0FBE6opISVxn8PQMbaYDg7znVwHfq9cLE6Jy3GcROQ+YgEsCoX7eGHLYZ1Xdr6qVVbWOqtbB9YtcoaqhPM6pP3/bn+GOBhCRyrhTResLMsh85s8+bwa6AYhIE1wi2FWgURas6cBfvKuHLgD2q+r2vKywyJ0aUtUUEbkVmIm74uAtVV0uImOABao6HXgTd/i4DtcpMyB4Eeedn/s8FigLfOz1i29W1SuCFnQe+bnPRYqf+zwT6CEiK4BU4G5VDdmjXT/3+U7gdRH5B67jeHAof7ETkcm4ZF7Z6/d4GCgBoKqv4fpBegPrgCPAX/O8zRB+v4wxxuSDonhqyBhjTC5YIjDGmDBnicAYY8KcJQJjjAlzlgiMMSbMWSIwhZKIpIrIYp9HnWzaHsqH7b0tIhu8bS3y7lDN7TreEJGm3vMHMiz7Oa8xeutJf1+WicjnIlIxh/atQr0apwk8u3zUFEoickhVy+Z322zW8TbwhapOFZEewDOq2iIP68tzTDmtV0QmAWtU9d/ZtB+Mq7p6a37HYooOOyIwIUFEynrjKCwSkaUiclqlURGpJiJzfb4xd/Dm9xCRed5rPxaRnD6g5wINvNfe4a1rmYiM8uaVEZEvRWSJN/9ab/4cEYkTkSeBKC+O971lh7yfU0TkMp+Y3xaRq0QkQkTGishvXo35m/14W+bhFRsTkbbePv4uIj+LyLnenbhjgGu9WK71Yn9LROZ7bTOr2GrCTbBrb9vDHpk9cHfFLvYe03B3wZf3llXG3VWZfkR7yPt5J/Cg9zwCV2+oMu6DvYw3/17goUy29zZwlff8auBXoA2wFCiDuyt7OXAe0B943ee1Fbyfc/DGPEiPyadNeoz9gEne85K4KpJRwDDgn978UsACoG4mcR7y2b+PgZ7edHmguPf8EuAT7/lg4GWf1z8O3OA9r4irRVQm2L9vewT3UeRKTJgiI0lVW6VPiEgJ4HER6Qik4b4Jnw0k+LzmN+Atr+1nqrpYRDrhBiv5ySutURL3TTozY0Xkn7g6NUNw9WumqephL4ZPgQ7A18CzIvIU7nTSj7nYr6+AcSJSCugJzFXVJO90VAsRucprVwFXLG5DhtdHichib/9XAt/6tJ8kIg1xZRZKZLH9HsAVInKXNx0JxHjrMmHKEoEJFdcDVYA2qposrqJopG8DVZ3rJYrLgLdF5DlgL/Ctqg70Yxt3q+rU9AkR6ZZZI1VdI26sg97AYyIyS1XH+LMTqnpUROYAlwLX4gZaATfa1EhVnZnDKpJUtZWIlMbV3/k78CJuAJ7ZqtrP61ifk8XrBeivqqv9ideEB+sjMKGiArDTSwJdgNPGXBY3DvMOVX0deAM33N8vwEUikn7Ov4yINPJzmz8CV4pIaREpgzut86OIVAeOqOp7uGJ+mY0Zm+wdmWTmQ1yhsPSjC3Af6sPTXyMijbxtZkrdaHO3AXfKyVLq6aWIB/s0PYg7RZZuJjBSvMMjcVVpTZizRGBCxftAnIgsBf4CrMqkTWdgiYj8jvu2PU5Vd+E+GCeLyB+400KN/dmgqi7C9R3Mx/UZvKGqvwPNgfneKZqHgccyeflE4I/0zuIMvsENDPSduuEXwSWuFcAicYOWTyCHI3Yvlj9wA7M8DTzh7bvv62YDTdM7i3FHDiW82JZ70ybM2eWjxhgT5uyIwBhjwpwlAmOMCXOWCIwxJsxZIjDGmDBnicAYY8KcJQJjjAlzlgiMMSbM/T8olKWUDJwlHAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Specify model\n",
        "model_SVM = svm.SVC(kernel=\"rbf\", C=1, gamma=0.1, probability=True)\n",
        "\n",
        "# Fit model on single split\n",
        "performance_measures = model_training_and_validation(model_SVM, \"SVM\", splits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EfQ5UuKQUCj9",
        "outputId": "c886afc4-c7a9-4201-ea9b-e08f50a146e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sensitivity: 0.83\n",
            "Specificity: 0.69\n",
            "AUC: 0.83\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "r2_SVM = model_SVM.score(static_test_x, static_test_y)\n",
        "r2_SVM"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zPYt6tUAq4EX",
        "outputId": "a4f941eb-05e9-433c-8268-e02abfd5b02b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7799352750809061"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Append SVM model\n",
        "models.append({\"label\": \"Model_SVM\", \"model\": model_SVM})\n",
        "# Plot roc curve\n",
        "plot_roc_curves_for_models(models, static_test_x, static_test_y);"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "yF9PJI5NUAQU",
        "outputId": "101efd7c-8bb1-40d3-fbe5-07c06d775cb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxU9frA8c8XVETcUUsFxDUVBVLcKvfMJc38eUtt05u2aIt12251b6nZapuVLd6srEyz3cq00swyy6U0931DRRE3UBAYnt8f3wEHZBmEYYB53q/XvOCc851znjMw5znr8zUiglJKKd/l5+0AlFJKeZcmAqWU8nGaCJRSysdpIlBKKR+niUAppXycJgKllPJxmghUoRhjNhhjeng7jtLCGPOIMeZtLy37PWPMZG8su7gZY643xnx/nu/V/8ki0kRQhhljdhtjko0xScaYOOeGoaonlykiESKyxJPLyGSMCTDGPG2M2etcz23GmAeMMaYklp9LPD2MMbGu40TkKREZ46HlGWPM3caY9caYU8aYWGPMJ8aYtp5Y3vkyxkwwxnxYlHmIyCwRucKNZZ2T/Eryf7K80kRQ9g0SkapANHAx8LCX4yk0Y0yFPCZ9AvQGBgDVgBuBW4GpHojBGGNK2/dhKjAeuBuoDbQAvgSuLO4F5fM38DhvLls5iYi+yugL2A1c7jL8HPCty3Bn4DfgOLAW6OEyrTbwLnAAOAZ86TJtILDG+b7fgMicywQaAMlAbZdpFwNHgIrO4ZuBTc75LwQaubQV4A5gG7Arl3XrDaQAoTnGdwIcQDPn8BLgaWAFcBL4KkdM+X0GS4AngWXOdWkG/NMZcyKwE7jN2TbI2SYDSHK+GgATgA+dbcKd6zUS2Ov8LB51WV4gMNP5eWwCHgRi8/jbNneuZ8d8/v7vAdOAb53x/gE0dZk+Fdjn/FxWA11dpk0APgU+dE4fA3QEljs/q4PAa0All/dEAD8AR4FDwCNAPyAVSHN+JmudbWsAM5zz2Q9MBvyd00Y5P/OXgATntFHAr87pxjntsDO2dUAb7E5AmnN5ScDXOb8HgL8zrh3Oz2Q1Of6H9JXL/5K3A9BXEf542b8AIc4vzFTncEPnl2wA9sivj3O4rnP6t8DHQC2gItDdOf5i5xewk/NLNdK5nIBclrkYuMUlninAm87fBwPbgVZABeA/wG8ubcW5UakNBOaybs8AP+ex3ns4u4Fe4tzQtMFurD/j7Ia5oM9gCXaDHeGMsSJ2b7upc2PUHTgNtHO270GODTe5J4L/YTf6UcAZoJXrOjk/8xDg75zzc5nv7cCeAv7+7znXp6Mz/lnAHJfpNwDBzmn3AXFAZZe404CrnZ9NINAemzgrONdlE3CPs3017Eb9PqCyc7hTzs/AZdlfAG85/yb1sIk68282CkgH7nIuK5DsiaAvdgNe0/l3aAXUd1nnyfl8Dx7Afg8ucr43Cgj29ne1tL+8HoC+ivDHs1+AJOyejwCLgJrOaQ8BH+RovxC7Ya+P3bOtlcs83wCeyDFuC2cTheuXbgyw2Pm7we59dnMOfweMdpmHH3aj2sg5LECvfNbtbdeNWo5pv+Pc08ZuzJ9xmdYau8fon99n4PLeSQV8xl8C452/98C9RBDiMn0FMNz5+06gr8u0MTnn5zLtUeD3AmJ7D3jbZXgAsDmf9seAKJe4lxYw/3uAL5y/jwD+yqNd1mfgHL4AmwADXcaNAH5y/j4K2JtjHqM4mwh6AVuxSckvl3XOLxFsAQZ74vtWnl+l7ZyoKryrRaQadiPVEqjjHN8IuMYYczzzBVyGTQKhwFEROZbL/BoB9+V4Xyj2NEhOnwFdjDH1gW7Y5PKLy3ymuszjKDZZNHR5/7581uuIM9bc1HdOz20+e7B79nXI/zPINQZjTH9jzO/GmKPO9gM4+5m6K87l99NA5gX8BjmWl9/6J5D3+ruzLIwx9xtjNhljTjjXpQbZ1yXnurcwxnzjvPHgJPCUS/tQ7OkWdzTC/g0Ounzub2GPDHJdtisRWYw9LTUNOGyMmW6Mqe7msgsTp3LSRFBOiMjP2L2l552j9mH3hmu6vIJE5BnntNrGmJq5zGof8GSO91URkdm5LPMY8D0wDLgOuwcvLvO5Lcd8AkXkN9dZ5LNKPwKdjDGhriONMZ2wX/bFLqNd24RhT3kcKeAzOCcGY0wANrk9D1wgIjWB+dgEVlC87jiIPSWUW9w5LQJCjDEx57MgY0xX7DWIa7FHfjWBE5xdFzh3fd4ANgPNRaQ69lx7Zvt9QJM8FpdzPvuwRwR1XD736iISkc97ss9Q5BURaY89wmuBPeVT4Pucy25aQBuVgyaC8uVloI8xJgp7EXCQMaavMcbfGFPZeftjiIgcxJ66ed0YU8sYU9EY0805j/8BtxtjOjnvpAkyxlxpjKmWxzI/Am4C/uH8PdObwMPGmAgAY0wNY8w17q6IiPyI3Rh+ZoyJcK5DZ+d6vSEi21ya32CMaW2MqQJMAj4VEUd+n0Eei60EBADxQLoxpj/gekvjISDYGFPD3fXIYS72M6lljGkI3JlXQ+f6vQ7MdsZcyRn/cGPMv91YVjXsefh4oIIx5jGgoL3qatiLs0nGmJbAWJdp3wD1jTH3OG/rreZMymA/l/DMu66c/1/fAy8YY6obY/yMMU2NMd3diBtjTAfn/19F4BT2poEMl2XllZDAnlJ8whjT3Pn/G2mMCXZnub5ME0E5IiLxwPvAYyKyD3vB9hHsxmAfdq8q829+I3bPeTP24vA9znmsAm7BHpofw17wHZXPYudh73CJE5G1LrF8ATwLzHGeZlgP9C/kKg0FfgIWYK+FfIi9E+WuHO0+wB4NxWEvZN7tjKGgzyAbEUl0vncudt2vc65f5vTNwGxgp/OUR26ny/IzCYgFdmGPeD7F7jnn5W7OniI5jj3lMQT42o1lLcR+bluxp8tSyP9UFMD92HVOxO4QfJw5wfnZ9AEGYT/nbUBP5+RPnD8TjDF/On+/CZtYN2I/y09x71QX2IT1P+f79mBPk01xTpsBtHZ+/l/m8t4XsX+/77FJbQb2YrTKhzl7JK9U2WOMWYK9UOmVp3uLwhgzFnsh2a09ZaU8RY8IlCohxpj6xphLnadKLsLeivmFt+NSSp/oU6rkVMLePdMYe6pnDvY6gFJepaeGlFLKx+mpIaWU8nFl7tRQnTp1JDw83NthKKVUmbJ69eojIlI3t2llLhGEh4ezatUqb4ehlFJlijFmT17T9NSQUkr5OE0ESinl4zQRKKWUj9NEoJRSPk4TgVJK+TiPJQJjzDvGmMPGmPV5TDfGmFeMMduNMX8bY9p5KhallFJ58+QRwXvY/kzz0h9btbI5ti/SNzwYi1JKqTx47DkCEVlqjAnPp8lg4H1nRya/G2NqGmPqO2uZK6VUmffnd++Sun9dkefjdyaNiomnCbriJlq0K/5itd58oKwh2eujxzrHnZMIjDG3Yo8aCAsLK5HglFLuOZyYwnfr4jh2OtXboXiHCBckbSTs+B/4iSPbpC57pwOQISa3d7pnVzrmm2QIgBXhLaCcJQK3ich0YDpATEyMVslTysvOpDtYvOkwn66OZcnWeBwZvvm1vN7/R0b6L6SF3/482/wR8Ridrrmv8DM/fhweeAA+eBuaNYO336ZTd890XeHNRLCf7H22hjjHKaU8ZP/xZL78az9FqTocdzKFb/4+yPHTaVxQPYBbuzVhaLsQmtYNKsZIS9jJA/D3x5DhKLhtJkcqZuk7SJ2LkM5TIWIwBJzbi2knv/O4FOtwwCWXwJYt8OCDMGECBHquozVvJoJ5wJ3GmDlAJ+CEXh9Qqvj8tOUwGw+czDbutcXbSU4rxMYuF5Uq+NE34kL+0T6Ey5rVwd+vCKc9isvhTbBl/vm/f9lUSDlxXm81Pf4Nbf7v/JftKiEBatcGf3948kkIDYWYmOKZdz48lgiMMbOBHkAdY0ws8DhQEUBE3gTmAwOwfeKeBv7pqViUKs9S0hzMWbGX5LSMbOOfXbA51/ZN6gax8J5u5708P2OKd+OfetpuxE8U1KVyPn6cUPQ4aoTBXavBFGbdDPgXw2ZUBGbNgvHj4Zln4JZbYMiQos/XTZ68a2hEAdMFuMNTy1fKVyzfmcCErzfmOu2BvhcxpmvjbOMq+vnh5+29eBHY9wf89SFs+BJSE4s+zyY94Lq55/9+vwrg51/0OApr3z64/XaYPx86d4ZLLy3xEMrExWKlVN4cDnu+/9Pbu9CmYfZz1JUrFvOG7dBG2PVz4c6l55RyHNZ/Bkd3QsUgiLgaoq+DBu0KuTeeg38AnM/5eG+aPRtuu81eE3j5ZbjzTntaqIRpIlCqjFq8+RC7jpxma5zdmw6o4F/8G/5Mf30IK9+GA38Vz/zCu0K3B6HVIAioWjzzLItq1YJOnWD6dGjcuOD2HqKJQKky6vYP/iTVYa8LVKrgR3DVSkWf6emjdm/ddY8/NQkWPwHBzaDfM9B6MFQqwsbbrwJUqlL0WMui9HR46SVITYVHH4V+/aBv36IdCRUDTQRKlVFpGRnc1q0J43o2I6CC3/kfDZw6AlsX2Iu2K/8HR7bm3q7Xf+1pHHV+1q6F0aNh9Wq49lp7ncQYrycB0ESgVNmSsAN2LAbgBr8NdIyvS4119c5vXpIBu5baJJCRfna8fwDcsw78K54d51cBKlcvQuA+7MwZmDzZ3g1UuzZ88gkMHVoqEkAmTQRKeYMIbPwKko/m2WTzwUROpKRlG9dpwxNZvz9REdjlfJ2voHrQeSxEDoNqDey4SkFQsXIRZqqy2bYNnn0WrrsOXnwRgoO9HdE5NBEoVZIO/AUH18Kx3fDrS/k2bZnH+DUZTRid+gAADw9oyT/ahebR0g2BtYrnPniVXVISfPUVXH89tGkDmzdDkybejipP+h+gVH4y73fP67x5Yc27K/vw0BkQftk5zQ6dTGHgq79ye/emDI5qkG1ag8DafOdfET8/Q52qAcUTlyo+P/wAt94Ke/ZAu3bQqlWpTgKgiUCp3KWeguXTYM1HcKwo515y0fpqpO9T/LzzJIfPVIUtZ0//iAgrdh3ju/UHOU0t6jZoRJ0GDfKZmSo1jh2D+++Hd96BFi3g559tEigDNBEolZsNX8JPT0KjS6H7QxB+KZiiP6y07XASa09UYd+KU0xdtDPXNtUCKjA4ugFD24XQvlGtIi9TlQCHwz4RvHUrPPwwPPYYVC4711k0ESjlaucSSDwEe3+zw0PfhuqF3yPPyBB+3HSIU6np2cbf+/GObMPPDY3kkmbZLx7WqRrguQfDVPE6cuRskbinnoKwMHs6qIzRRKBUwg7Y/6ctfTD//rPj/QPcenBq66FENh3MXuXz79gTzPg191NKvVvWY8JVEQRU8KNe9bKz16hciMAHH8A999jbQm+9Fa4uu89YaCJQvulMImz/0T5B+9no7NN6Pwatr7Z31OS4d/7PvceIPZYMQHziGb78az/r9uddvnjade2IaJB9HqG1q5SO0s3q/OzZY+sDLVxo+wzodv6VXEsLTQSq/Du8GQ7nqM75++sQu/LscMP2MGS6fYiqZliuD/tkZAjD3/o9q6wDQOv61XlsYGu6Nj+3Ln+VShW4sIbu8ZcrH34IY8faI4JXX4Vx48peobtcaCJQ5d8nIyE+99r8MnY52+KT2Zleh4yDzlo9sXG5ts0QIdWRwcgujbixSziVK/oRUstHa+b4qrp17UXht96CRo28HU2x0USgyo7k47Dnt+zlENxxOgFa9IPLJ2aNSjh1hm+2p/HhrHi2HU4Cct/456ZJ3ao0q+fDFTN9SVoavPCC/fnf/9oCcVdcUarKQxQHTQSq9Nu9DFbNgE3fgOPMec1iT0ZdtsTXJDElnW/+PsDPW+PJEGjfqBZP/19bLg6riaHgL7e/HzSpo0nAJ/z1ly0S99dfMHx4qSoSV9w0EajSzZEO719l795pPxIihkDA2YuvgrA5LpETp9PYciiR2Sty7+5wx/oGpK1fDUD9GpUZ26MpQ9uF0KSubtRVDikpMGkSPPcc1KkDn30G/1dMfRKXUpoIlHeciIVDGwpu50izp4IuuYv0S//Fil1HSUl2IALr95/ksz9j2Xv0tLOxAcJ47h+RtK6fe6XMCv6G5vWq6V07Km/bt8Pzz8NNN9nTQrXK/0N9mgiUd3x2y9mHttxRuQa/bDvCP99bmW30JU2DGd+7OS0uqAZAUIC/7uWrwktKgi++gBtvtEXitmzxao9hJU0TgfKOtNMQ2hn6PVVwW+MPF7bl9PrDALwy4mIa1a5CveoB1K8R6OFAVbm3cKF9IGzfPoiJsfWBfCgJgCYC5U2Va9j79wuw/XAScTuOsTnOPr3b8sJqWUcASp23hAT417/g/fehZUv45ZcyUySuuGkiUKXSsVOpbDp4kvQM4aZ3VmSbFhSg/7aqiDKLxG3fbvsO/s9/ylSRuOKm3yhVKj3yxTq+W3/23v4rI+sz6pJwagZWpGFNPR2kzlN8vO0hzN/f9hrWqBFER3s7Kq/TRKBKHUeGcOx0Ks3qVeXJq9tQwd8QGVKTiv5l/1F+5SUi8N579lTQM8/YWkGDB3s7qlJDE4Hyui1xiazcfZSNB0+y8cBJNsedJCUtg0uaBtOpSenr31WVMbt324vBP/wAXbtCz57ejqjU0USgPCPxkO2XNw+OM0kk+tfmz82HuPm9VQBUr1yB1g2qc13HRrRuUJ1uzeuUULCq3PrgA1skzhh4/XV7JFAOisQVN00Eqvikn4G49YDAjD4gGXk29Qd+ia/DXdttEnhqSFtGdAzFlMPH95UXXXCBLRP95pu20xiVK00Eyn1pyeeWc3a1+EnYsejscJ0W0O+ZXJs+9tUGdlZqxvt9O1DR34+Y8FqaBFTRpaXZ0hAOh+0u8oor7EvlSxOBct+Ch2H1uwW3u/5T+zOkAwTWzBrtyBA2x50kIwOW4U+jakF0a1HXQ8Eqn/Pnn3DzzbB2LVx33dkicapAmgiU+86chGr1YdDUvNvUbgJ1muc66f3lu5n49dkjipZ51ANSqlCSk2HiRFsfqG5dWyqiDHcb6Q0eTQTGmH7AVOwp4bdF5Jkc08OAmUBNZ5t/i8h8T8akzp8jQ0j3C2RH1S55N0oFDpzMddKO+CQApt/YHj9jiAqtmWs7pQpl50548UUYNQqmTPGJInHFzWOJwBjjD0wD+gCxwEpjzDwRcT3J/B9groi8YYxpDcwHwj0VkyqaDQdOEHQsmQGv/HLe8wio4MflrS7AT6t/qqI4eRI+/9xu/CMiYNu2ctVjWEnz5BFBR2C7iOwEMMbMAQYDrolAgMzzAzWAAx6MRxVRmkOo4Gd484aC6wPlJaRWoCYBVTTz58Ptt8P+/dCpk60PpEmgSDyZCBoCrr2ExAKdcrSZAHxvjLkLCAIuz21GxphbgVsBwvQWsBL3x84EPlkdS88TKVxQwdCvzYXeDkn5oiNH4N57bQfyrVvDsmU+WySuuHn7YvEI4D0RecEY0wX4wBjTRiT7DegiMh2YDhATEyNeiNPnHD+dytFTqfy8NZ6JX2+kakAFbqgVSF0CvB2a8kWZReJ27rS3hT7yCATo/2Jx8WQi2A+EugyHOMe5Gg30AxCR5caYykAd4LAH41IFSHNkcNmzP5F05mwn8d/efRmNfvoYDupTmaoEHTpk7wTy97d3BTVqBJGR3o6q3PHkt3ol0NwY09gYUwkYDszL0WYv0BvAGNMKqAzEezAm5YZ0h5B0Jp1BUQ2YOiyKOcNCaOQXD6mnvB2a8hUiMGMGXHQRTJ9uxw0apEnAQzx2RCAi6caYO4GF2FtD3xGRDcaYScAqEZkH3Af8zxhzL/bC8SgR0VM/Je3kQchIyxo0aRk0Nfu5Pm05nX9dAAnbzratF+GFAJVP2bkTbrkFFi+G7t3h8lwvHapiZMradjcmJkZWrVrl7TDKvpQTcCYRNnwB3/8n73Zhl0DE1RDg7BHswki4sE3JxKh8z8yZMG6cPRU0ZYpNCFokrlgYY1aLSExu07x9sVh5Q/JxeOEiSE/JGvV1g7tZsT+NlHQHNQMr0T68Npf2GkS1Bi28GKjyOQ0aQK9e8MYbEBLi7Wh8hiYCX3TmJKSnsOmCQXxzvBHrE4NYsa8d/dteyD/ahdC5SbDe669KRmqq7SgmIwMmTIA+fexLlShNBD7g6KlUHBlnTwGeSjhNOPBObH32hg3kH31CmNa2PlW1L2BVklautEXi1q+HG2/UInFepN/8cm7WH3t49Iv1APjjoDqnaGCO8m0ADG0XQueh+dQNUsoTTp+2zwK89BLUrw/z5tk7gpTXaCIo5w6dsNcBnri6DT1X30lI/NKsadHhWgJaecGuXfDqq/ZC8LPPQo0a3o7I52kiKEfSHRmcSnVkG3cmPQNj4MbOjeCvY/b2z/ajoEIlKre5yjuBKt9z4oQtEvfPf9oicdu3Q2howe9TJUITQTly7VvL+XPv8XPG1/RPhZSTIA6o1Rg63eqF6JTP+vZb21fwwYPQpQu0bKlJoJTRRFCO7D+eTHRoTQZFNcga13z/l3TbNAEye4LIo9MYpYpdfDzccw989BG0aWOPCFq29HZUKheaCMqZlhdWY/RljcGRBhnp8PMJO+GKJ+3PZvqUpioBDgdcdpm9HjBxIvz731CpkrejUnnQRFAenTwAr7aHtNN22K8iXHKnd2NSviEuDurVs08Gv/AChIfbowFVqrn97LYxpoonA1HFKOmQTQJRI+DyCXDNe14OSJV7GRnw1lvQooX9CTBwoCaBMqLAIwJjzCXA20BVIMwYEwXcJiLjPB2cyltGhpCzStQ5ZaNaD4aL+pdUSMpXbd9ubwVdssSWh+jb19sRqUJy59TQS0BfnCWkRWStMaabR6NS+dp6KJGrXvuVlLSMc6ZpaQhVot591xaJq1QJ/vc/GD1anw4ug9y6RiAi+0z2P64jr7bK8w4cTyYlLYMRHUOpXyMwa7yfpDO0xlb4aZYdYbRqo/KwsDB7BDBtGjRs6O1o1HlyJxHsc54eEmNMRWA8sMmzYSl3XBMTSruwWmdHvDsAfl0GgbWh01hodKn3glPl05kz8PTT9prApEnQu7d9qTLNnURwOzAV2xn9fuB7QK8PeMGgV39l48GTZDgvBvjlPAQ/uhMuutJeHK6gt+qpYvbHH/bUz4YNMHKkFokrR9xJBBeJyPWuI4wxlwLLPBOSysuWuESiQmpwSdM6VKtcgYgG1c9tFBSsSUAVr1On4L//hZdftqd/vvkGrrzS21GpYuROIngVaOfGOFUE6Y4Mrnh5KbHHkvNsk+rIoEvTYO7ve1EJRqZ83p498PrrcPvttu+A6rnsgKgyLc9EYIzpAlwC1DXG/MtlUnVsH8SqGCWnOdgZf4ouTYKJCq2ZbVolxynqnt7Ohae30vnEN/DWRkjYDhk5rtmnJ+sFYlU8jh+HTz+FMWOgdWt7i6j2GFZu5XdEUAn77EAFoJrL+JPAPzwZlC/r3aoeY7o2OTvi5ynw0+Szw4G1oX4kXHwD+Oc4BWQMRA4rmUBV+fXVVzB2LBw+bMtEtGypSaCcyzMRiMjPwM/GmPdEZE8JxqRcHVoHVS+Aq16FC9tCtfp6gU55xuHDcPfd8PHHEBlpO4zRInE+wZ1rBKeNMVOACKBy5kgR6eWxqHzI099tYvYfe895SjibwFrQQp/WVB7kcMCll8LevTB5Mjz4IFSs6O2oVAlxJxHMAj4GBmJvJR0JxHsyKF+yZu9xAiv5079NfSr6G/q3re/tkJQvOXAALrzQFombOtUWiWvd2ttRqRLmTiIIFpEZxpjxLqeLVno6sPIo3ZHBz1vj+ezPWFbsOkqGwMnkNGLCazHhqoizDQ/8BXOuh/QU26FMcDPvBa3Kp8wicQ89ZO8EGjcOBgzwdlTKS9xJBGnOnweNMVcCB4DangupfEpMSaPfy7+w/3gytYMq0atlPQIr2puvereql71x/FY4uR/aXgOVa0B4Vy9ErMqtrVttkbilS+Hyy6G/Fib0de4kgsnGmBrAfdjnB6oD93g0qnLGkSHMXRXL/uPJ3NWrGXf1ak6lCm7c5tnjYQhu6vkAle+YMQPuvBMqV4Z33oFRo/TmA1VwIhCRb5y/ngB6QtaTxcoNv247wuRvN7I5LpF2YTUZc1kT95KAUp4QHm6PAKZNg/p6PUpZ+T1Q5g9ci60xtEBE1htjBgKPAIHAxSUTYtm0/XAiT83fzOLNhwmtHci069oxoO2FGN37UiXpzBl44gn7++TJWiRO5Sq/I4IZQCiwAnjFGHMAiAH+LSJflkRwZcmRpDPc8PYfJKakAxB3MoUqFf15ZEBLRl4STkCFAh7GznDAzKvg+F5ITSqBiFW599tvtkjc5s1w881aJE7lKb9EEANEikiGMaYyEAc0FZGEkgmtbNl39DSb4xK5tFkwF1YP5ILqAYy+rDHBVQPcm0HqKdjzKzS4GMIvs8XjaoV7NGZVTiUlwaOPwquvQmgoLFigvYapfOWXCFJFJANARFKMMTsLmwSMMf2wJaz9gbdF5Jlc2lwLTAAEWCsi1xVmGaXNmK5N6HlRvYIb5qXNP7SjeVU0e/faW0PvuAOeegqqVSv4Pcqn5ZcIWhpj/nb+boCmzmEDiIhE5jdj5zWGaUAfIBZYaYyZJyIbXdo0Bx4GLhWRY8aYImxBlfJhx47BJ5/ArbfaB8J27oQGDbwdlSoj8ksErYo4747AdhHZCWCMmQMMBja6tLkFmCYixwBE5HARl1niHv78b1bsOppr/8FKlYgvvrAPhMXHQ/fucNFFmgRUoeRXdK6oheYaAvtchmOBTjnatAAwxizDnj6aICILcs7IGHMrcCtAWFhYEcMqXj9sPESVShWIDqvJJU2DuThHCWmlPCYuDu66y5aLjo6Gb7+1SUCpQnKr83oPL7850AMIAZYaY9qKyHHXRiIyHZgOEBMTk299Nm/o2rwOTw5p6+0wlC9xOKBrV9i3z14HuP9+LRKnzpsnE8F+7O2nmUKc41zFAn+ISBqwyxizFZsYSn0to9Op6Xy3Li7rdlGlSkRsrD3t4+8Pr7wCjRtrqWhVZG494mqMCTTGFPaYcyXQ3BjT2BhTCRgOzMvR5kvs0XpX30sAACAASURBVADGmDrYU0U7C7mcEvfVmv10mPwj932ylguqV6Z/G31CU3lYRoa9HbRlS3jjDTuuf39NAqpYFHhEYIwZBDyP7bGssTEmGpgkIlfl9z4RSTfG3AksxJ7/f0dENhhjJgGrRGSec9oVxpiNgAN4oCw8p/D7zgSMMcy9rQsdwmsV7Wnh39+AtXNAHAW3Vb5p82bbZeSyZfZ5gIEDvR2RKmfcOTU0AXsH0BIAEVljjGnszsxFZD4wP8e4x1x+F+BfzleZUqWSPx0bF0MR1k3fwPE9ENIRajaCZvr4v3Lx9tu2SFyVKjBzJtx4oz4drIqdW2WoReREjr3eUnfBtkyrFwHXz/V2FKo0atoUBg2C116DCy7wdjSqnHInEWwwxlwH+DsfALsb+M2zYZU+B08kc/8na0lOdbD36Gn8dK9MeUJKCkyaZH9/6ino2dO+lPIgdy4W34Xtr/gM8BG2HLXP9Uew8cBJlm1PwCHQqn51ru/UyNshqfJm2TL7PMDTT9uHw0QPvFXJcOeIoKWIPAo86ulgyoInBkcQGaIPjalilJgIjzxi+who1AgWLoQrrvB2VMqHuHNE8IIxZpMx5gljTBuPR6SUr4mNtReF77oL1q3TJKBKXIGJQER6YnsmiwfeMsasM8b8x+ORKVWeJSScfR6gVStbJG7qVKha1btxKZ/k1gNlIhInIq8AtwNrgMcKeIvKT9x6+HAovD/Y9kGgfIeIrQ3UujXcfTds2WLHa7eRyosKTATGmFbGmAnGmHXYzut/w5aLUOdr18+w/Uc4k2ifH4i42tsRqZJw8CAMHQrXXGM7jFm1SovEqVLBnYvF7wAfA31F5ICH4/EtN34BlWt4OwpVEjKLxO3fD889B/feCxW8XfNRKavA/0QR6VISgZR7jjT4ejycOgLHdns7GlVS9u2Dhg1tkbhp02yRuBYtvB2VUtnkeWrIGDPX+XOdMeZvl9c6l57LlLtO7IM1syB+E1QMhFaDoJJ2IVhuORy2Oqhrkbi+fTUJqFIpvyOC8c6fWuGqOPV8FKKGezsK5UmbNsHo0bB8ua0QOmiQtyNSKl/59VB20PnrOBF5yHWaMeZZ4KFz31X+vPj9FrYcSuRw4hlvh6LKgunT7fMA1arBBx/A9ddrkThV6rlz+2ifXMb1L+5ASqvXftrOqt3HSE510CG8Fo1qB3k7JFWaNW8OQ4bAxo1www2aBFSZkOcRgTFmLDAOaJLjmkA1YJmnAytNrusUxn1X6G1+KhfJyTBhgt3gP/OMFolTZVJ+1wg+Ar4Dngb+7TI+UUSOejQqpcqCpUtthzHbtsHtt9uHxfQIQJVB+Z0aEhHZDdwBJLq8MMYUQ48sSpVRJ0/CuHHQvbu9O2jRIntnkCYBVUYVdEQwEFiN7YjG9b9cgCYejEup0uvAAXjvPfjXv2zfAUF63UiVbfndNTTQ+dOtbimVKteOHIG5c+2RQMuWsGuX9himyg13ag1daowJcv5+gzHmRWNMmOdDU6oUEIGPP7ZF4u65B7ZuteM1CahyxJ1iJ28AUcaYKOA+4G3gA6C7JwPzpmXbjzBvjS2rlFHUTqLWfQo7l9gCc6psOXAAxo6FefMgJsZeC9Ang1U55E4iSBcRMcYMBl4TkRnGmNGeDsybZv62m8WbD1OnagANawYWrUeypc/DsV0QWBtqhUO9VsUWp/IghwO6dbNF4p5/HsaP1yJxqtxy5z870RjzMHAj0NUY4wdU9GxY3tesXlUW3NPt/N58eDOsmA6SAYkHoPkVMOyD4g1QecaePRASYovEvf46NGkCzZp5OyqlPMqdJ4uHYTuuv1lE4rB9EUzxaFRl3bq5sGoGbP4W/AMgtKO3I1IFcTjgxRdtb2GZReKuuEKTgPIJ7pShjjPGzAI6GGMGAitE5H3Ph1aGiYBfRXhgm7cjUe5Yv94WiVuxAgYOhKu1oyDlW9y5a+haYAVwDXAt8Icx5h+eDkypEvHmm9Cune0z+KOP7IXhEO2AT/kWd64RPAp0EJHDAMaYusCPwKeeDKykHUk6w9u/7CI1PYPNcYlUqeTv7ZCUJ2WWg2jVynYd+fLLULeut6NSyivcSQR+mUnAKQE3O70vS5ZsiefNn3cQVMkfP2O4pKl2Jl4unT4Njz1mLwY/+6wtE9G93N4JrZRb3EkEC4wxC4HZzuFhwHzPheQdIvaBgQX3dCO0dhUvR6M8YskSWyRuxw77hLAWiVMKcGPPXkQeAN4CIp2v6Tk7qlGqVDtxAm677Wx56MWLbf/BmgSUAvLvj6A58DzQFFgH3C8i+0sqMKWKzcGD8OGHcP/9MHEiVNEjPqVc5XdE8A7wDTAUW4H01cLO3BjTzxizxRiz3Rjz73zaDTXGiDEmprDLUCpX8fHwqvNftmVL2L0bpkzRJKBULvJLBNVE5H8iskVEngfCCzNjY4w/MA3brWVrYIQxpnUu7aoB44E/CjN/pXIlYm8DbdUK7rvvbJE4vSNIqTzllwgqG2MuNsa0M8a0AwJzDBekI7BdRHaKSCowBxicS7sngGeBlEJHr5Srfftg0CDbYXyzZvDXX1okTik35HfX0EHgRZfhOJdhAXoVMO+GwD6X4Vigk2sDZ0IJFZFvjTEP5DUjY8ytwK0AYWGlvAJ2hgNOxXs7Ct+Tng49ekBcHLz0Etx1l71FVClVoPw6pvFoD9zO4nUvAqMKaisi04HpADExMUUtDO0Z6amwdAr89aEtNBfc3NsR+YbduyE01FYGfestWySuiXaep1RhePLBsP1AqMtwiHNcpmpAG2CJMWY30BmY560LxkXOLofWwdLnoHZjuOY9uP3XYohK5Sk93ZaHbtXKVgkFuPxyTQJKnQdPFlhfCTQ3xjTGJoDhwHWZE0XkBFAnc9gYswR7i+oqD8aUq9Op6cz6fQ+BFf2pWeU8K2yfSrA/L7sXmvcpvuDUuf7+2xaJW7UKBg+GoUO9HZFSZZrHEoGIpBtj7gQWAv7AOyKywRgzCVglIvM8tezCSHdkcMesP1m3/wTTb4yhWuVCJIIzibDhS1jzEez9DYw/VLvQc8Equ/c/fjzUqmW7kLzmGn0wTKkiKjARGGMMcD3QREQmOfsrvlBEVhT0XhGZT45yFCLyWB5te7gVcTH771fr+WlLPE8NacvlrfPphzYpHjZ8AY4zdjhuPWyaB2mn7fWA3o9D5DCo0bBkAvc1meUg2rSB4cPtBeE6dQp+n1KqQO4cEbwOZGDvEpoEJAKfAR08GFeJOJmSxuwV+7ixcyOu65TH3UhpKfDHG7D0BUh16Xc4oIbd8EdfDyExulfqKadOwX/+Yy8GT5liu4/sdp49xymlcuVOIugkIu2MMX8BiMgxY0wlD8flURsOnGDFrqOcTnUA0KRu0LmNRGD9Z/DjRDixF1r0h96PQU3n9e8KgeCvfdh61KJFcMstsGuXvR1Ui8Qp5RHubMnSnE8JC2T1R5Dh0ag8bPI3m1i+MyFruH6NwOwN4rfCV+MgdiVc2BYGz4MmWqq4xBw/busCzZgBzZvD0qXQtau3o1Kq3HInEbwCfAHUM8Y8CfwD+I9Ho/Kw9IwMOobXZvpN7fH3M+deIP75GTi8CQZPg6gR4KcPJpWoQ4dgzhx46CF4/HEIDCz4PUqp8+ZOn8WzjDGrgd6AAa4WkU0ej8zDKvgbalbJ4wxX+hmoFQ4X31CiMfm0zI3/+PFw0UX2QTG9GKxUiXDnrqEw4DTwtes4EdnrycBKXOop2PiVTQKbv4F6Ed6OyDeIwKxZNgEkJcGAAfZ0kCYBpUqMO6eGvsVeHzBAZaAxsAUoX1vKzfPhy7Fnh/V5AM/buxduvx2++w66dDl7TUApVaLcOTXU1nXYWShunMci8hZHqv05ZhFUbwhBukfqUZlF4g4fhldesV1HapE4pbyi0Pc/isifxphOBbcso4LqQnXtuN5jdu6ERo3scwH/+x80bQrh4d6OSimf5s41gn+5DPoB7YADHotIlU/p6fDCC/YuoOeeg7vvht69vR2VUgr3jgiqufyejr1m8JlnwlHl0po1tkjcn3/CkCG2PpBSqtTINxE4HySrJiL3l1A8qrx57TW4914IDoZPP9VKoUqVQnn2R2CMqSAiDuDSEoxHlRfi7OEhMtJ2HblxoyYBpUqp/I4IVmCvB6wxxswDPgFOZU4Ukc89HJsqi5KS4NFHoWJF23GMFolTqtRzp4eyykACtvroQGCQ86dS2X3/vS0T/eqrkJZ29qhAKVWq5XdEUM95x9B6zj5Qlkm/4eqsY8fgX/+C996z5SGWLoXLLvN2VEopN+WXCPyBqmRPAJk0EaizDh+2F4IffhgeewwqV/Z2REqpQsgvERwUkUklFom3HFgDSYfg0HpvR1K2xMXB7Nn2jqDMInHBwd6OSil1HvJLBOW/B5AzSTC9B1kHOMYPKlX1ZkSlnwi8/75NAKdPw8CBtj6QJgGlyqz8LhaX/8c+HamA2C4nb1kMd/8FQbpBy9Pu3dCvH4waBa1b2wfFtEicUmVenkcEInK0JAPxqgbtoGF7b0dRuqWnQ8+ecOQITJtmq4b6uXPTmVKqtNNOd1X+tm+Hxo1tkbh33oEmTWzROKVUuaG7dCp3aWnw1FMQEWGPAMAeEWgSUKrc8ckjgsSUdC6orrc45unPP22RuDVrbIG4YcO8HZFSyoN87ojg0MkUDsYdZHDtPRC7ytvhlD6vvAIdO9rbQz//HObOhQsu8HZUSikP8rkjgp82H+blitPouWYtrHGOrBTk1ZhKBREwBi6+GG66yfYdUKuWt6NSSpUAn0sEizYf5s4KqcgFbTF9ngD/ihBafjtcK1Bion0iOCDAbvy7drUvpZTP8KlTQylpDn7ddoQaVSpiAmtB054QfplNBr5owQJbJO711+0RgRaJU8on+VQi+H1nAslpDmpWqeTtULwrIQFGjoT+/SEoCJYtgxdftKeGlFI+x6cSweLNhwms6E/1yj53Riy7hAT44gv473/hr7+gSxdvR6SU8iKPJgJjTD9jzBZjzHZjzL9zmf4vY8xGY8zfxphFxhiP3aQuIizadJjLmtfBzxf3fA8etB3FiECLFrBnD0yaZK8NKKV8mscSgbO/42lAf6A1MMIY0zpHs7+AGBGJBD4FnvNUPFsPJbH/eDK9W9bz1CJKJxH7RHCrVvYIYPt2O17vCFJKOXnyiKAjsF1EdopIKjAHGOzaQER+EpHTzsHfgRBPBbN8xxEAurWo66lFlD67dsEVV9iHw6KiYO1aLRKnlDqHJ0+WNwT2uQzHAvndpzka+C63CcaYW4FbAcLCws4rmDPpGQDUrOIjdwilp0OvXvZ6wBtvwK23apE4pVSuSsVVU2PMDUAM0D236SIyHZgOEBMTo/c45mfbNlsYrkIFePddaNoUQkO9HZVSqhTz5C7ifsB1CxTiHJeNMeZy4FHgKhE548F4yre0NJg82T4X8NprdlyPHpoElFIF8uQRwUqguTGmMTYBDAeuc21gjLkYeAvoJyKHPRhL+bZqlb0O8PffMHw4jBjh7YiUUmWIx44IRCQduBNYCGwC5orIBmPMJGPMVc5mU4CqwCfGmDXGmHmeiqfcmjoVOnWyHcZ89ZXtR7iej90ZpZQqEo9eIxCR+cD8HOMec/n9ck8uv1zLLBIXE2OPBp57DmrW9HZUSqkyqFRcLFaFcPIkPPQQVK4ML70El15qX0opdZ70fsKyZP5822PY9On2riAtEqeUKgaaCMqCI0fghhvgyiuhRg347TeYMkWLxCmlioUmgrLg2DH4+mt4/HHbjWQnH+4/QSlV7PQaQWm1fz/MmgUPPGDLQuzZoxeDPSwtLY3Y2FhSUlK8HYpS561y5cqEhIRQsaL7VRQ0EZQ2IvD223D//fYhsf/7P2jWTJNACYiNjaVatWqEh4dj9LSbKoNEhISEBGJjY2ncuLHb79NTQ6XJjh3Qu7etC9SunX1ArFkzb0flM1JSUggODtYkoMosYwzBwcGFPqrVI4LSIj3dJoGjR+Gtt2DMGC0S5wWaBFRZdz7/w5oIvG3LFlsYrkIFmDnT/h7isWrcSil1Dt3l9JbUVJg4Edq2hWnT7Lju3TUJKKVKnCYCb1ixAtq3hwkT4Jpr4PrrvR2RKiWMMdxwww1Zw+np6dStW5eBAwcWaj7h4eEcOXKkSG38/f2Jjo6mTZs2DBo0iOPHjwOwe/duAgMDiY6OznqlpqbmOo977rmHhg0bkpGRkTVuwoQJPP/883nGEhcXx/Dhw2natCnt27dnwIABbN261a319raZM2fSvHlzmjdvzsyZM3Nts2bNGjp37kx0dDQxMTGsWLEia9qSJUuIjo4mIiKC7t1zrcrvEXpqqKS9/DLcdx/Ur2+fDSjkF1yVjIlfb2DjgZPFOs/WDarz+KCIfNsEBQWxfv16kpOTCQwM5IcffqBhw4bFGoe7AgMDWbNmDQAjR45k2rRpPProowA0bdo0a1peMjIy+OKLLwgNDeXnn3+mZ8+eBS5TRBgyZAgjR45kzpw5AKxdu5ZDhw7RokWLQq+Dw+HA39+/0O87H0ePHmXixImsWrUKYwzt27fnqquuolaObmEffPBBHn/8cfr378/8+fN58MEHWbJkCcePH2fcuHEsWLCAsLAwDh8uuYLMekRQUjLLQXTsCLfcAhs2aBJQuRowYADffvstALNnz2aES1nxo0ePcvXVVxMZGUnnzp35+++/AUhISOCKK64gIiKCMWPGIC7lRz788EM6duxIdHQ0t912Gw6Ho9AxdenShf37z+lOJF9LliwhIiKCsWPHMnv2bLfe89NPP1GxYkVuv/32rHFRUVF07dr1nLZXX3017du3JyIigunTp2eNr1q1Kvfddx9RUVEsX748z/UfO3YsMTExRERE8Pjjjxdq3XKzcOFC+vTpQ+3atalVqxZ9+vRhwYIF57QzxnDypN3JOHHiBA0aNADgo48+4v/+7/+yemGsV4JVhPWIwNNOnIAHH4TAQHs0cMkl9qVKtYL23D1p+PDhTJo0iYEDB/L3339z880388svv9i4Hn+ciy++mC+//JLFixdz0003sWbNGiZOnMhll13GY489xrfffsuMGTMA2LRpEx9//DHLli2jYsWKjBs3jlmzZnHTTTe5HY/D4WDRokWMHj06a9yOHTuIjo4G4NJLL2Va5nUuF5lJbPDgwTzyyCOkpaUV+JDT+vXrad++vVtxvfPOO9SuXZvk5GQ6dOjA0KFDCQ4O5tSpU3Tq1IkXXniBTZs28eyzz+a6/k8++SS1a9fG4XDQu3dv/v77byIjI7MtY8qUKcyaNeucZXfr1o1XXnkl27j9+/cT6tIRVEhISK7J8+WXX6Zv377cf//9ZGRk8NtvvwGwdetW0tLS6NGjB4mJiYwfP75Qf6ei8JlEEB7/E9sCHqTiU869oSYFH6YW2ddfw+23Q1ycfUAss3S0UvmIjIxk9+7dzJ49mwEDBmSb9uuvv/LZZ58B0KtXLxISEjh58iRLly7l888/B+DKK6/MOh2xaNEiVq9eTYcOHQBITk52e08zOTmZ6Oho9u/fT6tWrejTp0/WtIJODaWmpjJ//nxefPFFqlWrRqdOnVi4cCEDBw7M8/bGwt72+Morr/DFF18AsG/fPrZt20ZwcDD+/v4MHToUyH/9586dy/Tp00lPT+fgwYNs3LjxnETwwAMP8MADDxQqroK88cYbvPTSSwwdOpS5c+cyevRofvzxR9LT01m9ejWLFi0iOTmZLl260Llz5/M6JVZYPpMIaiTvpaJxkNZlPBUrVYZmfQp+0/mKj4fx420nMW3bwpdfgvMfUSl3XHXVVdx///0sWbKEhISE856PiDBy5EiefvrpQr838xrB6dOn6du3L9OmTePuu+92670LFy7k+PHjtG3bFoDTp08TGBjIwIEDCQ4O5uDBg9naJyYmUrNmTSIiIvj0008LnP+SJUv48ccfWb58OVWqVKFHjx5ZD1FVrlw567pAXuu/a9cunn/+eVauXEmtWrUYNWpUrg9hFeaIoGHDhixZsiRrODY2lh49epzz3pkzZzJ16lQArrnmGsaMGQPYI4jg4GCCgoIICgqiW7durF27tkQSASJSpl7t27eX87H8/f+KPF5dTiUeP6/3F8q2bSI1aohMnChy5oznl6eKxcaNG70dggQFBYmIyL59+2Tq1KkiIvLTTz/JlVdeKSIid911l0yaNClrfHR0dNb4J554QkRE5s+fL4DEx8fLhg0bpFmzZnLo0CEREUlISJDdu3eLiEijRo0kPj6+wFhERP78808JCwuTtLQ02bVrl0REROS7HiNGjJCPPvooazgpKUnq1q0rp06dkrVr10qbNm3k5MmTIiLy2WefSc+ePUVEJCMjQzp27ChvvfVW1nvXrl0rS5cuzTb/L7/8UgYOHCgiIps2bZKAgAD56aefzok7r/Vfs2aNREZGisPhkLi4OKlXr568++67+a5TQRISEiQ8PFyOHj0qR48elfDwcElISDinXcuWLbNi/fHHH6Vdu3YiYv//evXqJWlpaXLq1CmJiIiQdevWnVcsuf0vA6skj+2q1zfshX2V2kSwd6/IU0+JZGTY4eMlkHBUsSpNicCVayJISEiQwYMHS9u2baVTp06ydu1aERE5cuSI9OnTR1q3bi1jxoyRsLCwrI38nDlzJCoqStq2bSvt2rWT5cuXi0jhEoGIyMCBA+X9998vMBGcOnVKatWqJSdOnMg2fsiQITJnzhwREXnzzTclMjJSoqKipE+fPrJjx46sdvv375drrrlGmjRpIq1bt5YBAwbI1q1bs80rJSVF+vXrJy1btpTBgwdL9+7dc00E+a3/yJEjpXnz5tKrVy8ZMmRIkROBiMiMGTOkadOm0rRpU3nnnXeyxo8ePVpWrlwpIiK//PKLtGvXTiIjI6Vjx46yatWqrHbPPfectGrVSiIiIuSll1467zgKmwiMlLHOTWJiYmTVqlWFft/vHzxG5x1TOX3/XqpUrVF8AWVk2I5iHnwQHA5Yu1brA5VRmzZtolWrVt4OQ6kiy+1/2RizWkRicmuvt48WxbZt0KsXjB1rbwtdt06TgFKqzPGZi8XFLj0d+vSB48dhxgz45z/1jiBV5iQkJNC7d+9zxi9atIjg4GAvRKS8QRNBYW3aZDuKqVABPvjAFolzPhCiVFkTHBxc4BPCqvzTU0PuOnPGdhUZGQmvvWbHde2qSUApVebpEYE7fv8dRo+GjRvhxhvtSymlygk9IijICy/YkhCJiTB/Prz/Pui5U6VUOaKJIC+ZZXO7dLFlItavh/79vRuTUkp5gCaCnI4ft6eBxo+3w5dcAq+/DtWrezcu5RNKU38ETz75JBEREURGRhIdHc0ff/zBxIkTefjhh7O1W7NmTdY96+Hh4edUCs3s0yAvL7/8MpUrV+bEiRNZ49577z3uvPPObO169OhB5jNESUlJ3HbbbVl9FvTo0YM//vgj3/UtLRYsWMBFF11Es2bNeOaZZ3Jts3fvXnr27MnFF19MZGQk8+fPB2DFihVZfUBERUVl1VoqKr1G4OrLL2HcODh82D4gpkXifNd3/4a4dcU7zwvbQv/cv/iZSkt/BMuXL+ebb77hzz//JCAggCNHjpCamsqIESPo169ftto9c+bMyVYqOzExkX379hEaGsqmTZsKXNbs2bPp0KEDn3/+Of/85z/dim/MmDE0btyYbdu24efnx65du9i4cWPhV5SS7bPA4XBwxx138MMPPxASEkKHDh246qqraN26dbZ2kydP5tprr2Xs2LFs3LiRAQMGsHv3btq0acOqVauoUKECBw8eJCoqikGDBlGhQtE25XpEAHbDf+21MGQIXHCB7UHsqac0CSivKA39ERw8eJA6deoQEBAAQJ06dWjQoAEtWrSgVq1a2fa+586dmy3Ga6+9lo8//jjX+HPasWMHSUlJTJ482e0+C3bs2MEff/zB5MmT8fOzm7DGjRtz5ZVXntM2rz4HwsPDeeihh2jXrh2ffPIJ33//PV26dKFdu3Zcc801JCUlATBp0iQ6dOhAmzZtuPXWWylqJYYVK1bQrFkzmjRpQqVKlRg+fDhfffXVOe3y6rOgSpUqWRv9lJSU8+qoPld51Z4orS+P1Bratk2kZk2RJ58USU09r/mrsq+01Bpau3atDB06VJKTkyUqKipbraE777xTJkyYICIiixYtkqioKBGxRecmTpwoIiLffPNNVtG5jRs3ysCBAyXV+X89duxYmTlzpojkX2soMTFRoqKipHnz5jJ27FhZsmRJ1rQpU6bIPffcIyIiy5cvF9fvZKNGjWTz5s3SpUsXERGJjo6WDRs25FmbaPLkyTJp0iRxOBwSFhYmcXFxIiLy7rvvyh133JGtbffu3WXlypXy1VdfydVXX+3W55lZ9C09PV26d++eVZupUaNG8uyzz4qISHx8vHTt2lWSkpJEROSZZ57J+ixdi8bdcMMNMm/evHOW8eGHH0pUVNQ5r6FDh57T9pNPPpHRo0dnDb///vvnrKeIyIEDB6RNmzbSsGFDqVmzZrZ6RL///ru0bt1agoKC5PPPP891vQtba8h3jwj27oUnn7Snf5o1s8OPPAIFdJyhlKcV1B/Bjc7bl3P2R5B5bSGv/giio6NZtGgRO3fuLDCGqlWrsnr1aqZPn07dunUZNmwY7733HgDDhg3j008/JSMj45zTQmAfUqtVqxZz5syhVatWVKlSJc/lzJ49m+HDh+Pn58fQoUP55JNPgLz7JijsHvDcuXNp164dF198MRs2bMh2+mjYsGEA/P7772zcuJFLL72U6OhoZs6cyZ49ewDbY1qnTp1o27YtixcvZsOGDecs4/rrr2fNmjXnvNwpp52X2bNnM2rUKGJjY5k/fz433nhjVr/PnTp1YsOGDaxcuZKnn3461/LZheXRawTGmH7AVMAfeFtEnskxPQB4H2gPJADDky3ACQAADJJJREFURGS3J2MiI8Ne/H3oIfv7sGE2EVSr5tHFKlUYpaE/An9/f3r06EGPHj1o27YtM2fOZNSoUYSGhtK4cWN+/vlnPvvsM5YvX37Oe4cNG8Ydd9yRlTxys27dOrZt25bV4U1qaiqNGzfmzjvvJDg4mGPHjmVrf/ToUerUqUPNmjVZu3Ztgef2C+pzICgoCLCfUZ8+fc45NZWSksK4ceNYtWoVoaGhTJgwIdeN7qxZs5gyZco545s1a3ZOMmjYsCH79u3LGo6Njc31GtCMGTOyurns0qULKSkpHDlyJFunQq1ataJq1aqsX7+emJhca8m5zWNHBMYYf2Aa0B9oDYwwxrTO0Ww0cExEmgEvAc96Kh4AjjgI6H8l3HGHvS10wwYtEqdKpZtvvpnHH388q2OXTF27ds3qKGXJkiXUqVOH6tWr061bNz766CMAvvvuu6yNaO/evfn000+zOkI/evRo1t5ufrZs2cK2bduyhtesWUOjRo2yhkeMGMG9995LkyZNCAkJOef9Q4YM4cEHH6Rv3755LmP27NlMmDCB3bt3s3v3bg4cOMCBAwfYs2cPHTp0YNmyZcTFxQGwatUqzpw5Q2hoKE2bNiUmJobHH38865z97t27s66rZDp58iRBQUHUqFGDQ4cO8d133+UaR+fOnVm2bBnbt28H4NSpU2zdujVro1+nTh2SkpLy3MMvzBFBhw4d2LZtG7t27SI1NZU5c+Zw1VVXndMuLCyMRYsWAbaSaEpKCnXr1mXXrl2kp6cDsGfPHjZv3kx4eHien7Hb8jpnVNQX0AVY6DL8MPBwjjYLgS7O3ysAR8CWxs7rdd7XCN59VKSGkYyaNUTeffdsvwFKOZWWawQ5eaM/glWrVkmXLl2kVatW0rbt/7d3/kFWlWUc/3xzF1dQsQlrLER2a6lWV1zdMYNRdHSIwRmMkUKLdBvJxsImJKcmNBoiw0xndGpQVAZM82chW2SrIcw6CoICIivp4I+xtR/SZkykFerTH+976brc5Z717r3Xc8/zmTmz7znned/3ee69e57zvu85z9Nq06dPf4fsrl27rK6uzpYsWfKOeoXaHCh/QWNjo+3YseMdx+bOnWuLFy82s5B8pq2tzcaPH28TJ060J598cp/c7t27bfbs2dbU1GTHHnusTZo0yTZu3LhfHwPlHOiv55o1a6y9vd1aW1uttbXVVq1aZWZm8+fPt6amJpswYYJ1dHTYggULCn5eg2H16tXW3NxsTU1NtmjRon3Hr7zyyn399vT02IQJE/blbOjq6jKzsKbQ0tJi48ePt7a2Nlu5cmXBPt4z+QgkzQCmmNnsuP8l4FNmNidPZnuU6Y37z0eZv/Vr62LgYoAxY8aclOSOpj9bHryd4auWcMy822ho+ui7NcupYTwfgVMrDDYfQSreIzCzpcBSCIlp3k0bbZNnweRZxQUdx3EyRjkdwSvA0Xn7o+OxQjK9kuqAkYRFY8dxKoDnI3CgvI5gE9AsqZFwwT8P+EI/mU7gQmA9MAN42Mo1V+U4CTCzoXtJJwV4PoLa491cQsv21JCZvQnMISwI7wDuMbMeSQsl5ZbJbwU+IGkncBnwnXLp4zjFaGhooK+vr+S3Rx2nWpgZfX19NDQ0DKpeZpLXO04x9u7dS29v75C8oOM41aKhoYHRo0dT3+/l2NQvFjtOJaivr6exsbHaajhOxcluiAnHcRwHcEfgOI6TedwROI7jZJzULRZL2gUM/tXiwChCGIss4TZnA7c5G5Ri8zFmdmShE6lzBKUg6YmBVs1rFbc5G7jN2aBcNvvUkOM4TsZxR+A4jpNxsuYIllZbgSrgNmcDtzkblMXmTK0ROI7jOPuTtRGB4ziO0w93BI7jOBmnJh2BpCmSnpW0U9J+EU0lHSzp7nj+cUljK6/l0JLA5sskPSNpm6Q1ko4p1E6aKGZznty5kkxS6h81TGKzpM/H77pH0i8qreNQk+C3PUbSWklb4u97ajX0HCokLZP0aszgWOi8JN0QP49tkk4sudOBclimdQMOAp4HmoBhwFNASz+ZrwE3xvJ5wN3V1rsCNp8BDI/lS7Jgc5Q7DOgGNgDt1da7At9zM7AFeH/c/2C19a6AzUuBS2K5BXip2nqXaPNpwInA9gHOTwUeAAScAjxeap+1OCI4GdhpZi+Y2X+Bu4Bz+smcA6yI5fuAM5XubCRFbTaztWb2etzdQMgYl2aSfM8APwCuBmohtnQSm78C/MzMXgMws1crrONQk8RmAw6P5ZHAnyqo35BjZt3A3w8gcg5wmwU2AEdIOqqUPmvREXwE+GPefm88VlDGQgKd3UCa8/IlsTmfiwh3FGmmqM1xyHy0ma2upGJlJMn3PA4YJ+lRSRskTamYduUhic3fB2ZJ6gV+C1xaGdWqxmD/34vi+QgyhqRZQDswqdq6lBNJ7wOuAzqqrEqlqSNMD51OGPV1S2o1s39UVavycj6w3MyulfRp4OeSjjOzt6utWFqoxRHBK8DRefuj47GCMpLqCMPJvopoVx6S2Iyks4D5wDQz+0+FdCsXxWw+DDgOWCfpJcJcamfKF4yTfM+9QKeZ7TWzF4HnCI4hrSSx+SLgHgAzWw80EIKz1SqJ/t8HQy06gk1As6RGScMIi8Gd/WQ6gQtjeQbwsMVVmJRS1GZJbcBNBCeQ9nljKGKzme02s1FmNtbMxhLWRaaZWZrznCb5bd9PGA0gaRRhquiFSio5xCSx+WXgTABJnyQ4gl0V1bKydAIXxKeHTgF2m9mfS2mw5qaGzOxNSXOALsITB8vMrEfSQuAJM+sEbiUMH3cSFmXOq57GpZPQ5muAQ4F747r4y2Y2rWpKl0hCm2uKhDZ3AZMlPQO8BVxuZqkd7Sa0eR5ws6S5hIXjjjTf2Em6k+DMR8V1jwVAPYCZ3UhYB5kK7AReB75ccp8p/rwcx3GcIaAWp4Ycx3GcQeCOwHEcJ+O4I3Acx8k47ggcx3EyjjsCx3GcjOOOwHlPIuktSVvztrEHkN0zBP0tl/Ri7GtzfEN1sG3cIqkllr/b79xjpeoY28l9Ltsl/VrSEUXkT0h7NE6n/Pjjo857Ekl7zOzQoZY9QBvLgd+Y2X2SJgM/MbPjS2ivZJ2KtStpBfCcmf3wAPIdhKirc4ZaF6d28BGBkwokHRrzKGyW9LSk/SKNSjpKUnfeHfOp8fhkSetj3XslFbtAdwMfi3Uvi21tl/TNeGyEpNWSnorHZ8bj6yS1S1oMHBL1uCOe2xP/3iXp7Dydl0uaIekgSddI2hRjzH81wceynhhsTNLJ0cYtkh6T9PH4Ju5CYGbUZWbUfZmkjVG2UMRWJ2tUO/a2b74V2ghvxW6N20rCW/CHx3OjCG9V5ka0e+LfecD8WD6IEG9oFOHCPiIe/zbwvQL9LQdmxPLngMeBk4CngRGEt7J7gDbgXODmvLoj4991xJwHOZ3yZHI6TgdWxPIwQhTJQ4CLgSvi8YOBJ4DGAnruybPvXmBK3D8cqIvls4BfxnIH8NO8+lcBs2L5CEIsohHV/r59q+5WcyEmnJrhDTM7IbcjqR64StJpwNuEO+EPAX/Jq7MJWBZl7zezrZImEZKVPBpDawwj3EkX4hpJVxDi1FxEiF+z0sz+FXX4FXAq8DvgWklXE6aTHhmEXQ8A10s6GJgCdJvZG3E66nhJM6LcSEKwuBf71T9E0tZo/w7goTz5FZKaCWEW6gfofzIwTdK34n4DMCa25WQUdwROWvgicCRwkpntVYgo2pAvYGbd0VGcDSyXdB3wGvCQmZ2foI/Lzey+3I6kMwsJmdlzCrkOpgKLJK0xs4VJjDCzf0taB3wGmElItAIh29SlZtZVpIk3zOwEScMJ8Xe+DtxASMCz1symx4X1dQPUF3CumT2bRF8nG/gagZMWRgKvRidwBrBfzmWFPMx/NbObgVsI6f42ABMl5eb8R0gal7DPR4DPShouaQRhWucRSR8GXjez2wnB/ArljN0bRyaFuJsQKCw3uoBwUb8kV0fSuNhnQSxkm/sGME//D6WeC0XckSf6T8IUWY4u4FLF4ZFCVFon47gjcNLCHUC7pKeBC4A/FJA5HXhK0hbC3fb1ZraLcGG8U9I2wrTQJ5J0aGabCWsHGwlrBreY2RagFdgYp2gWAIsKVF8KbMstFvfjQUJioN9bSL8IwXE9A2xWSFp+E0VG7FGXbYTELD8GfhRtz6+3FmjJLRYTRg71UbeeuO9kHH981HEcJ+P4iMBxHCfjuCNwHMfJOO4IHMdxMo47AsdxnIzjjsBxHCfjuCNwHMfJOO4IHMdxMs7/AEcyCxwzKdLlAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Specify model\n",
        "model_ANN = MLPClassifier(hidden_layer_sizes=(5, 3), random_state=SEED)\n",
        "\n",
        "# Fit model on single split\n",
        "performance_measures = model_training_and_validation(model_ANN, \"ANN\", splits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jeWNf9DcULvk",
        "outputId": "da1ad4a3-8154-435e-c43d-78f619329630"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sensitivity: 0.83\n",
            "Specificity: 0.64\n",
            "AUC: 0.83\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "r2_ANN = model_ANN.score(static_test_x, static_test_y)\n",
        "r2_ANN"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w_E19TO2rTYN",
        "outputId": "e2858705-e59a-4766-fa33-445f05f3e51c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7605177993527508"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Append ANN model\n",
        "models.append({\"label\": \"Model_ANN\", \"model\": model_ANN})\n",
        "# Plot roc curve\n",
        "plot_roc_curves_for_models(models, static_test_x, static_test_y, True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 582
        },
        "id": "-L9IChGeUQ87",
        "outputId": "e4ffd9ab-e512-4e03-f3cb-72c200b168f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-43-bfcf35ae90f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"label\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"Model_ANN\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"model\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmodel_ANN\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# Plot roc curve\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mplot_roc_curves_for_models\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatic_test_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatic_test_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-15-5a65b02e7941>\u001b[0m in \u001b[0;36mplot_roc_curves_for_models\u001b[0;34m(models, test_x, test_y, save_png)\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[0;31m# Save plot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msave_png\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0mfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavefig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{DATA}/roc_auc\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdpi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox_inches\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"tight\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'DATA' is not defined"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd3hU1daH35VJQgoBktBJIPQeWpQiIEWKCCKiV7FfRcSK9wpeyycCdrGiiCIqFsSCoigICgioKL33DgmEQAgpJCGZmf39sSdhEibJAJlMyn6fZ56Zc/Y+e//OmZmzzm5riVIKg8FgMFRcfLwtwGAwGAzexRgCg8FgqOAYQ2AwGAwVHGMIDAaDoYJjDIHBYDBUcIwhMBgMhgqOMQSGC0JEtolIL2/rKC2IyFMiMsNLdc8Ukee9UXdxIyK3isivF3ms+U1eIsYQlGFE5KCIZIhImojEO24MlT1Zp1KqtVJqmSfryEFEKonISyJy2HGee0RknIhISdTvQk8vEYl13qeUelEpNdJD9YmIPCIiW0XkjIjEisi3ItLWE/VdLCIyQUS+uJQylFKzlFL93ajrPONXkr/J8ooxBGWfIUqpykB7oAPwpJf1XDAi4ltA0rdAX2AQEALcDowC3vaABhGR0vZ/eBsYAzwChAHNgB+Aa4q7okK+A4/jzboNDpRS5lVGX8BB4Cqn7VeB+U7bXYCVwGlgE9DLKS0M+AQ4CiQBPzilDQY2Oo5bCUTnrxOoC2QAYU5pHYCTgJ9j+25gh6P8RUADp7wKeBDYAxxwcW59gUwgMt/+zoANaOLYXga8BKwGUoAf82kq7BosA14A/nKcSxPg3w7NqcB+4D5H3mBHHjuQ5njVBSYAXzjyRDnO607gsONaPO1UXyDwqeN67AAeB2IL+G6bOs7z8kK+/5nAVGC+Q+8qoLFT+tvAEcd1WQf0cEqbAMwBvnCkjwQuB/52XKtjwLuAv9MxrYHfgFPAceApYCCQBWQ7rskmR96qwEeOcuKA5wGLI+0uxzV/E0h0pN0F/OlIF0dagkPbFqAN+iEg21FfGvBT/v8BYHHo2ue4JuvI9xsyLxe/JW8LMK9L+PLy/gEiHH+Ytx3b9Rx/skHoll8/x3YNR/p84GsgFPADrnTs7+D4A3Z2/KnudNRTyUWdS4F7nfRMBt53fB4K7AVaAr7A/wErnfIqx00lDAh0cW4vA8sLOO9DnLtBL3PcaNqgb9bfce7GXNQ1WIa+Ybd2aPRDP203dtyMrgTSgY6O/L3Id+PGtSH4EH3TbwecBVo6n5PjmkcAm/OX51TuaOBQEd//TMf5XO7QPwv4yin9NiDckfYYEA8EOOnOBq5zXJtAoBPacPo6zmUH8Kgjfwj6pv4YEODY7pz/GjjVPRf4wPGd1EQb6pzv7C7ACjzsqCuQvIZgAPoGXs3xPbQE6jid8/OF/A/Gof8HzR3HtgPCvf1fLe0vrwswr0v48vQfIA395KOAJUA1R9r/gM/z5V+EvrHXQT/ZhroocxrwXL59uzhnKJz/dCOBpY7Pgn767OnY/gW4x6kMH/RNtYFjWwF9Cjm3Gc43tXxp/+B40kbfzF92SmuFfmK0FHYNnI6dVMQ1/gEY4/jcC/cMQYRT+mrgZsfn/cAAp7SR+ctzSnsa+KcIbTOBGU7bg4CdheRPAto56V5RRPmPAnMdn0cAGwrIl3sNHNu10AYw0GnfCOB3x+e7gMP5yriLc4agD7AbbZR8XJxzYYZgFzDUE/+38vwqbX2ihgvnOqVUCPom1QKo7tjfALhRRE7nvIDuaCMQCZxSSiW5KK8B8Fi+4yLR3SD5+Q7oKiJ1gJ5o4/KHUzlvO5VxCm0s6jkdf6SQ8zrp0OqKOo50V+UcQj/ZV6fwa+BSg4hcLSL/iMgpR/5BnLum7hLv9DkdyBnAr5uvvsLOP5GCz9+duhCRsSKyQ0SSHedSlbznkv/cm4nIz46JBynAi075I9HdLe7QAP0dHHO67h+gWwYu63ZGKbUU3S01FUgQkekiUsXNui9Ep8GBMQTlBKXUcvTT0muOXUfQT8PVnF7BSqmXHWlhIlLNRVFHgBfyHReklJrtos4k4FfgJuAW9BO8cirnvnzlBCqlVjoXUcgpLQY6i0ik804R6Yz+sy912u2cpz66y+NkEdfgPA0iUglt3F4DaimlqgEL0AasKL3ucAzdJeRKd36WABEiEnMxFYlID/QYxL/QLb9qQDLnzgXOP59pwE6gqVKqCrqvPSf/EaBRAdXlL+cIukVQ3em6V1FKtS7kmLwFKjVFKdUJ3cJrhu7yKfI4R92Ni8hjyIcxBOWLt4B+ItIOPQg4REQGiIhFRAIc0x8jlFLH0F0374lIqIj4iUhPRxkfAqNFpLNjJk2wiFwjIiEF1PklcAdwg+NzDu8DT4pIawARqSoiN7p7Ikqpxeib4Xci0tpxDl0c5zVNKbXHKfttItJKRIKAScAcpZStsGtQQLX+QCXgBGAVkasB5ymNx4FwEanq7nnk4xv0NQkVkXrAQwVldJzfe8Bsh2Z/h/6bReQJN+oKQffDnwB8RWQ8UNRTdQh6cDZNRFoA9zul/QzUEZFHHdN6QxxGGfR1icqZdeX4ff0KvC4iVUTER0Qai8iVbuhGRC5z/P78gDPoSQN2p7oKMkiguxSfE5Gmjt9vtIiEu1NvRcYYgnKEUuoE8BkwXil1BD1g+xT6ZnAE/VSV853fjn5y3okeHH7UUcZa4F500zwJPeB7VyHVzkPPcIlXSm1y0jIXeAX4ytHNsBW4+gJPaTjwO7AQPRbyBXomysP58n2Obg3FowcyH3FoKOoa5EEpleo49hv0ud/iOL+c9J3AbGC/o8vDVXdZYUwCYoED6BbPHPSTc0E8wrkuktPoLo9hwE9u1LUIfd12o7vLMim8KwpgLPqcU9EPBF/nJDiuTT9gCPo67wF6O5K/dbwnish6x+c70IZ1O/pazsG9ri7QButDx3GH0N1kkx1pHwGtHNf/BxfHvoH+/n5FG7WP0IPRhkKQcy15g6HsISLL0AOVXlndeymIyP3ogWS3npQNBk9hWgQGQwkhInVE5ApHV0lz9FTMud7WZTCYFX0GQ8nhj5490xDd1fMVehzAYPAqpmvIYDAYKjima8hgMBgqOGWua6h69eoqKirK2zIMBoOhTLFu3bqTSqkartLKnCGIiopi7dq13pZhMBgMZQoROVRQmukaMhgMhgqOMQQGg8FQwTGGwGAwGCo4xhAYDAZDBccYAoPBYKjgeMwQiMjHIpIgIlsLSBcRmSIie0Vks4h09JQWg8FgMBSMJ1sEM9HxTAviarTXyqboWKTTPKjFYDAYDAXgsXUESqkVIhJVSJahwGeOQCb/iEg1Eanj8GVuMBgMXmH1sdWsjl9dLGXF792ILTXhksvxO2slODWLtt3uYFD3O4pBWV68uaCsHnn9o8c69p1nCERkFLrVQP369UtEnMFgcI+E1Ex+2RJPUnqWt6VcNGnWk+xJX4bCzobUnDAMUugx+RGXftvUJfe7XL49jWdnHiUt0MK8ug3KnSFwG6XUdGA6QExMjPGSZzB4mbNWG0t3JDBnXSzLdp/AZnf3b2nHL/RvxJLpUX0Xin/YCsRyLkbQ2ZNXknWi6DhKt1oWc6dlEc184grMs6r1eDrf+NiFizp9GsaNgxkzoEkTmDGDlld6JnSFNw1BHHljtkY49hkMBg8RdzqDHzbEcSleh+NTMvl58zFOp2dTq0olRvVsxPCOETSuEVzgMX/E/cGOxB3sPLWTxYcXX3TdnqSBXxV+rDsUASRK0MHVCsGWhaz4GFW9OarL29B6KFQ6P4ppZ5+LaBLYbNCtG+zaBY8/DhMmQKDnAq150xDMAx4Ska+AzkCyGR8wGIqP33clsP1oSp597y7dS0a27ZLK9ff1YUDr2tzQKYLuTapj8XHdhXIk5QiLDi0C4O31b+fu9/PxY9agWTQLbXZJOvJwYifsWnDxx698B5/Mw8hul5McC0V6PQFtrr/4up1JTISwMLBY4IUXIDISYmKKp+xC8JghEJHZQC+guojEAs8CfgBKqfeBBcAgdEzcdODfntJiMJRnMrNtfLX6MBnZ9jz7X1m402X+RjWCWfRoz4uuz0ekwJu/M5/v+JzZO2fnbj/c4WH+3ebf+OCDxcdyLmNWur6JJxcVUrkQFk+4+GNzqFofHl4HciFjAwKWYriNKgWzZsGYMfDyy3DvvTBs2KWX6yaenDU0ooh0BTzoqfoNhorC3/sTmfCT626McQOaM7JHwzz7/Hx88HHjRu6KM9ln+GHvD2TZih4Y3pa4jbCAMH674TcA/C3+5xKVgiOrYMMXsO0HyEq9KD15aNQLbvnm4o/38QVnA1VSHDkCo0fDggXQpQtccUWJSygTg8UGg6FgbDbd3z9ndFfa1MvbRx3gV7w3tuVbv+TlzVPczh8dUBP/VdPz7sw8DVu/g1P7wS8YWl8H7W+Buh0v8Gk8H5ZKcDH98d5k9my47z49JvDWW/DQQ7pbqIQxhsBgKKMs3XmcAyfT2R2vn6Yr+VqK/cafy4YvYM0MbCm7oEZ1vos9RoTVWuRhldRh2OEifkhUD+j5OLQcApUqe0BwGSE0FDp3hunToWHDovN7CGMIDIYyyujP15Nl0+MC/r4+hFf2L+IIN0g/pZ/W7U4DyllpsPQ5CG8CbW+C+CUE3reCoMoRF1eHjy/4B1261rKI1QpvvglZWfD00zBwIAwYcGktoWLAGAKDoYySbbdzX89GPNC7CZV8fc5rDayJX8OepD1FF5R1BhK2gy0LDq2ENBcrYUMqQ5t+bPV1dL34h0BAlWI4iwrEpk1wzz2wbh386196nETE60YAjCEwGMoWiftg31IAbvPZxuUnahC4KYwFKbvJtOftqnnh+LILLz8ACAhznXZ4PgCBvoFUqWSMgNucPQvPP69nA4WFwbffwvDhpcIA5GAMgcHgDZSC7T9CxqkCs+w8lkpyZjZnVTZrbYewKhtRx87NlW8bBoknYdyZAJYGu+5quS05hVGnU1ymARBUHdpcB62ug8q19T6/YPCrVOAhAb4BBPp6bnFTuWPPHnjlFbjlFnjjDQgP97ai8zCGwGAoSY5ugGObIOkg/PlmoVlbON7nhATzSXXHzaN6AU/rwFd9plE7sGbutogQ6l8VKezJMzC0eObBG/KSlgY//gi33gpt2sDOndCokbdVFYj5BRgMhZEz3/3k7uIpb97DebeHfwRR3c/LdvBUEsO+nEntBnuJzdZ1f9L1PUL8KmMPqAYWP3xECAvWA8QBvgGE+IcUj0bDpfHbbzBqFBw6BB07QsuWpdoIgDEEBoNrss7A31Nh45eQdKB4y251HWrAiyzfn0LC2cqc2nyII+nbUCjsys6GhA3EZa9C6maRIXV5uMPDDGk0hDqV6xSvDkPxkpQEY8fCxx9Ds2awfLk2AmUAYwgMBlds+wF+fwEaXAFX/g+irgC59MVKvx3YxvKTcZxa9jO/7TyEb5Ut+AYdzJvJHkCDSldwW+sbuKltd3zK2iKpiojNplcE794NTz4J48dDQIC3VbmNMQQGgzP7l0HqcTi8Um8PnwFV6l5wMXa7YvGO45zJOjeTx65sPLP2VSyV9PTMgNoQ7h/JtY0fpGvtnvhb9ABtk/A6VA2ooPPsyxonT55zEvfii1C/vu4OKmMYQ2AwJO6DuPXa9cGCsef2WyqBf9GrXncfT2XHsbwzczbHJvPRn7pLSSxn8Av7E7+q67FUSqaqiubNqyYS6OtL61oNCh/MNZROlILPP4dHH9XTQkeNguuu87aqi8YYAkPF5Gwq7F2sV9B+d0/etL7j9XTKwNDzFk2tP5xEbFIGACdSz/LDhji2xCUXWM3UWzqyOvlTvt//O51qdKVvxDXc0HIggYVMzzSUcg4d0v6BFi3SMQN6Xrwn19KCMQSG8k/CTr1y1pl/3oPYNee263WCYdPB4gfV6rtc7GO3K27+4B+HWwcblsq7iAzz46Ze4bSsHXKev7Oz9jS+iXuC9QnrqRlUk5mDpp9XpqGM8cUXcP/9ukXwzjvwwANlz9GdC4whMJR/vr1TBy5xgbr/b/acyGC/tTr2Yw5fPbHxLvPalSLLZufOrg1o3eQYz639jERgwXH9ckVUlSjGdBzDtY2vLYYTMXidGjX0oPAHH0CDBt5WU2wYQ2AoO2Sc1r5w7EV7vcxDeiI0GwhXTczdlXjmLD/vzeaLWSfYk5AGuL7558cnIBZrYDKxGTr/m73epGFV114j/Xz8iAyJNGMAZZnsbHj9df3+zDPaQVz//qXKPURxYAyBofRz8C9Y+xHs+BlsZ4vO74JD9hrsOlGN1EwrP28+yvLdJ7Ar6NQglJeub0uH+tUQCv9z25WdWxf3Yd6xLHAEVW0V3oq6lS98VpGhDLBhg3YSt2ED3HxzqXISV9wYQ2Ao3dis8Nm1evZOpzuh9TBwcnimUOyMTyU5PZtdx1OZvdp1uMN9W+uSvXUdAHWqBnB/r8YM7xhBoxp5ZwVtPbmVkxknXZZhV3ay7Vnc1Pwmbmx2I1X8q5hFXuWRzEyYNAlefRWqV4fvvoPriykmcSnFGAKDd0iOhePbis5ny9ZdQd0exnrFf1l94BSZGTaUgq1xKXy3PpbDp9IdmQWoz6s3RNOqjmvvmL4WoWnNEJcxd1OyUrhl/i0oVKGSGlZtSPOw5kVrN5RN9u6F116DO+7Q3UKhod5W5HGMITB4h+/uPbdoyx0CqvLHnpP8e+aaPLu7NQ5nTN+mNKul/ewEV7Kc95TvLlm2LBSKu9vcTf+o/i7zWMRC02pNL6p8QykmLQ3mzoXbb9dO4nbt8mrEsJLGGAKDd8hOh8guMPDFovOKBWq3JX2rXpE7ZUQHGoQFUbNKJepULX53yHWD69I6vHWxl2sopSxapBeEHTkCMTHaP1AFMgJgDIHBmwRU1fP3i2BvQhrx+5LYGa9X77aoHZLbAnCHY2nHOJhysMB0pRRrj69l3r55AAT5GfcOFYLERPjvf+Gzz6BFC/jjjzLjJK64MYbAUCpJOpPFjmMpWO2KOz5enSctuFLRP9sMawZbTmxBoRj568gi8/uID1fUvYKxl42lX/1+F63bUEbIcRK3d6+OHfx//1emnMQVN8YQGEolT83dwi9bz83tvya6Dnd0rc/JrL0czdzK0SKm/b+74V3WJ6zP3W4e2pynOj9VYP76VepTPbD6Jes2lHJOnNARwiwWHTWsQQNo397bqryOMQSGUofNrkhKz6JJzcq8cF0bfC1CdEQ1VsWv5IkV919QWZ8M+ATQ8/1Nl08FRimYOVN3Bb38svYVNHSot1WVGowhMHidXfGprDl4iu3HUth+NIWd8SlkZtvp1jiczo3C2XVqF6+umcaCAzpe76Ruk4gIiSiy3LqV61Kvcj1PyzeUdg4e1IPBv/0GPXpA797eVlTqMIbA4BlSj+u4vAVgO5tGqiWM9TuPc/fMtQBUCfClVd0q3HJ5A1rVrULPptWx2q3cMv8WRIQ+kX24vtn1dKnTpYROwlDm+fxz7SROBN57T7cEyoGTuOLGGAJD8WE9C/FbAQUf9QNlLzCrBfjjRHUe3quNwIvD2jLi8vP98mTbssmyZ/Fwh4cZFT3Kg+IN5ZJatbSb6Pff10FjDC4xhsDgPtkZ57tzdmbpC7Bvybnt6s1g4Msus47/cRv7/Zvw2YDL8LP4EBMVmscIpGensz95P1aHg7mi/AAZDIB2Dvfqq3pW0Pjx2kFcf9eLAw3nMIbA4D4Ln4R1nxSd79Y5+j3iMgislrvbZlfsjE/Bboe/sNAgJJiezWq4LGL8yvEsOrgodzvQt/gXjhnKGevXw913w6ZNcMst55zEGYrEGAKD+5xNgZA6MOTtgvOENYLqrl0wfPb3QSb+dK5F0aIAf0AAaVlpRFWJYtxl4/ARH2JqxVysakN5JyMDJk7U/oFq1NCuIspw2Ehv4FFDICIDgbfRXcIzlFIv50uvD3wKVHPkeUIptcCTmgwXj82usPoEsq9y14IzZQFHU1wm7TuRBsD02zuRZc+gerU0dp3a5TJvWnYaVfyr0DOi7IcBNHiY/fvhjTfgrrtg8uQK4SSuuPGYIRARCzAV6AfEAmtEZJ5SyrmT+f+Ab5RS00SkFbAAiPKUJsOlse1oMsFJGQya8sdFl1HJ14e+LWpy3+JRrFq/qtC8l9W+7KLrMZRzUlLg++/1zb91a9izp1xFDCtpPNkiuBzYq5TaDyAiXwFDAWdDoICc/oGqwFEP6jFcItk2ha+P8P5tRfsHcuZkZhyHz+zkaMY+ErMP0Pvbl0g6m0SgbyAvdX+pwONahLe4VMmG8siCBTB6NMTFQefO2j+QMQKXhCcNQT3AOUpILNA5X54JwK8i8jAQDFzlqiARGQWMAqhvpoCVOKv2J/Ltulh6J2dSy1cY2KZ2kcecyT7D8fTjHE07yrjFejWwv48/TUKb0Lt+b5qHNqdP/T7UDi66LIMBgJMn4T//0QHkW7WCv/6qsE7iihtvDxaPAGYqpV4Xka7A5yLSRqm8E9CVUtOB6QAxMTGFRw0xFAun07M4dSaL5btPMPGn7VSu5MttoYHUoJJbx/974b/ZcWpH7vakbpMY3Hgwfj5+npJsKM/kOInbv19PC33qKajk3m/RUDSeNARxQKTTdoRjnzP3AAMBlFJ/i0gAUB1I8KAuQxFk2+x0f+V30s6eCxI//5HuNPj9azimV2WmZaVxKvNUgWWczDhJp1qduKn5TQRYAuge0d0YAcOFc/y4nglksehZQQ0aQHS0t1WVOzxpCNYATUWkIdoA3Azcki/PYaAvMFNEWgIBwAkPajK4gdWmSDtrZUi7ulzVoga1SKSBzwnIOpObZ/i84Rw9U/iQTu/I3lzd8GpPyzWUR5SCjz+Gxx7TTuJGj4YhQ7ytqtziMUOglLKKyEPAIvTU0I+VUttEZBKwVik1D3gM+FBE/oMeOL5LKWW6fkqalGNgz87dlGw7jSWOW7P/psufCzl5eh9HcxbmhDeDtKMkZibSvV53BjUcVGCxXesWMs3UYCiI/fvh3nth6VK48kq4yuXQoaEYkbJ2342JiVFr1671toyyT2YynE2FbXPh1/8rMNs/9Ttyr+Wky7R7297LIx0f8ZRCQ0Xk00/hgQd0V9DkydogGCdxxYKIrFNKuVyZ6e3BYoM3yDgNrzcHa2burp/qPsLquGwyrTaqBfrTKSqMK/oM4VTmXvjjfzzS4ZE8gVt8xMcs9jIUP3XrQp8+MG0aRBTtatxQPBhDUBE5mwLWTHbUGsLPpxuwNTWY1Uc6cnXb2tzQMYIujcLJtGVwJvsMKYf1KuGrGlxFw6oVK6C3oQTIytJjAHY7TJgA/frpl6FEMYagAnDqTBY2+7kuwDOJ6UQBH8fW4XD9wdzQL4KpbetQ2REL2Gq3MuC7AZw+ezr3GDPjx1DsrFmjncRt3Qq3326cxHkRYwjKObNWHeLpuVsBsGCjCmeoK6eYXwmGd4ygy/DzB3Sz7dmcPnuavvX70q1uN0IDQk2kL0PxkZ6u1wK8+SbUqQPz5pkZQV7GGIJyzvFkPQ7w3HVt6L3uISJOrMhNax/l2gV0DtE1ovlX8395VJ+hAnLgALzzjh4IfuUVqFrV24oqPMYQlCOsNjtnsmx59p212hGB27s0gA1JULM1dLoLfP0JaHMtoN1B2NS54zKdBpENhmIhOVk7ifv3v7WTuL17ITKy6OMMJYIxBOWIf33wN+sPnz5vfzVLFmSmgLJBaEPoPIosWxZZtiyW7pvH038+7bI8XzE/D0MxMH++jhV87Bh07QotWhgjUMow//RyRNzpDNpHVmNIu7q5+5rG/UDPHRMgJxJE9aakZaXRb04/0rLTcvM90O4BKvtXzt22iMWsCjZcGidOwKOPwpdfQps2ukXQwniULY0YQ1DOaFE7hHu6NwRbNtitsDxZJ/R/Qb83uYqUrBTSstMYGDWQNtXbUCu4FgOjBnpPtKH8YbNB9+56PGDiRHjiCfD397YqQwEYQ1AeSTkK73SC7HS97eMH3R7Carey8uhKvt06FYAeET24tvG1XhRqKHfEx0PNmnpl8OuvQ1SUbg0YSjVuGwIRCVJKpXtSjKGYSDuujUC7EVCjOYTrGML/WfYflh1ZRmilUG5reRu9Int5V6eh/GC3w4cfwrhxeibQ/ffD4MHeVmVwkyINgYh0A2YAlYH6ItIOuE8p9YCnxRkKxm5X5PcSdZ7bqFZDofm5fv5jace4rPZlfHDVB/hZzAIxQzGxd6+eCrpsmXYPMWCAtxUZLhB3WgRvAgOAeQBKqU0iYpzMeJHdx1O59t0/ycy2n5fm41P4yszKfpWNETAUH598op3E+fvrFsE995jVwWUQt7qGlFJHJO+Xaysor8HzHD2dQWa2nRGXR1KnamDufh9lZXjV3fD7LL1DtNfGg8kHmbdvHodTDxMRYhx5GYqR+vV1C2DqVKhnVp+XVdwxBEcc3UNKRPyAMcCOIo4xlAA3xkTSsX5o7vaomTF8EHdWbzSsD3+P1S+0t9Budbtxd5u7vSHVUF44exZeekmPCUyaBH376pehTOOOIRgNvI0ORh8H/AqY8QEvMOSdP9l+LAW7YzDAJ18TfJ89k5a+gVzZ5nbwseTur1apGv0a9KNmUM0S1WsoZ6xapbt+tm2DO+80TuLKEe4YguZKqVudd4jIFcBfnpFkKIhd8am0i6hKt8bVCQnwpXXdKuflaeUXyoMmWIyhODlzBp55Bt56S3f//PwzXHONt1UZihF3DME7QEc39hkuAavNTv+3VhCblFFgniybna6Nwxk7oHkJKjNUeA4dgvfe03GDX34Zqpz/AGIo2xRoCESkK9ANqCEi/3VKqoKOQWwoRjKybew/cYaujcJpF1ktT5q/7Qw10vdSO303XZJ/hg+2Q+JesOcbs68TCqalbigOTp+GOXNg5Eho1UpPETURw8othbUI/NFrB3yBEKf9KcANnhRVkenbsiYjezQ6t2P5ZPj9+XPbgWFQJxo63AaWfEv2jy+EGi1LRqih/PLjj3pBWEKCdhPRooUxAuWcAg2BUmo5sFxEZiqlDpWgJoMzx7dA5Vpw7TtQuy2E1MkzQE/tybYAACAASURBVDdn9xzeWPsGCkWaPRMJCvOiWEOZJiEBHnkEvv4aoqN1wBjjJK5C4M4YQbqITAZaAwE5O5VSfTymqgLx0i87mL3q8HmrhPMQGArNXK/W3HlqJ1n2LG5sdiMiYnwHGS4Omw2uuAIOH4bnn4fHHwc/s/CwouCOIZgFfA0MRk8lvRM44UlRFYmNh08T6G/h6jZ18LMIV7et49ZxGdYMlhxewqpjqwj2C+Z/l//Pw0oN5ZKjR6F2be0k7u23tZO4Vq28rcpQwrhjCMKVUh+JyBin7qI1nhZWHrHa7CzffYLv1sey+sAp7ApSMrKJiQplwrWtz2U8ugG+uhWsmTqgTHgTAJRSbEjYwI/7fmTRwUWcyT5Dvcr1eLD9g146I0OZxW6HDz6A//1PzwR64AEYNMjbqgxewh1DkO14PyYi1wBHAdMRfYGkZmYz8K0/iDudQViwP31a1CTQT0++6tsy70Kv7YeWM6aqIsO3KhYJ1XO0vr6SbHs2qVmpBPoG0r9Bf4Y2GUqnWp3wcbiSMBjcYvdu7SRuxQq46iq42gQgqui4YwieF5GqwGPo9QNVgEc9qqqcYbMrvlkbS9zpDB7u04SH+zTF39f1zftM9hlmnVhDvK8vXau3JjLs3JoBEaFt9bb0a9CPIL+gkpJvKE989BE89BAEBMDHH8Ndd5nVwYaiDYFS6mfHx2SgN+SuLDa4wZ97TvL8/O3sjE+lY/1qjOzeyKURsNlt/LD3B97Z8A6JmYkMS01j4jUTkOpNvKDaUG6JitItgKlToY5741GG8k9hC8oswL/QPoYWKqW2ishg4CkgEOhQMhLLJnsTUnlxwU6W7kwgMiyQqbd0ZFDb2oiLp6+VR1fy2trX2JO0h/Y12jMl8lqif51ontQMl87Zs/Dcc/rz888bJ3EGlxTWIvgIiARWA1NE5CgQAzyhlPqhJMSVJU6mneW2GatIzbQCEJ+SSZCfhacGteDOblFU8nW9GPu9je8xbdM06lWux+v26vTbtRHJ+rMkpRvKKytXaidxO3fC3XcbJ3GGAinMEMQA0Uopu4gEAPFAY6VUYslIK1scOZXOzvhUrmgSTu0qgdSqUol7ujckvHKlQo9bE7+G5qHN+bLv+/hPbgx1O0BUdwgOh9CokhFvKF+kpcHTT8M770BkJCxcaKKGGQqlMEOQpZSyAyilMkVk/4UaAREZiHZhbQFmKKVedpHnX8AEQAGblFK3XEgdpY2RPRrRu3nh7p7XH1/PhL8nYLVbSUhPoG31tvjnuItocwN0e6gElBrKLYcP66mhDz4IL74IISFFH2Oo0BRmCFqIyGbHZwEaO7YFUEqp6MIKdowxTAX6AbHAGhGZp5Ta7pSnKfAkcIVSKklEKoTD/K0nt3Ig+QD9G/QnukY0V9W/ytuSDGWdpCT49lsYNUovCNu/H+rW9bYqQxmhMENwqd7LLgf2KqX2A4jIV8BQYLtTnnuBqUqpJAClVMIl1lniPPn9ZlYfOOUyfrAzmdZM7vvtPpLOJpF8NhmACd0mEOLveFrLTPG0VEN5Ze5cvSDsxAm48kpo3twYAcMFUZjTuUt1NFcPOOK0HQt0zpenGYCI/IXuPpqglFqYvyARGQWMAqhfv/4lyipeftt+nCB/X9rXr0a3xuF0yOdCOocT6SdYn7CettXb0iy0GXWD61LZr3IJqzWUK+Lj4eGHtbvo9u1h/nxtBAyGC8St4PUerr8p0AuIAFaISFul1GnnTEqp6cB0gJiYmEL9s3mDHk2r88Kwtm7lHdFiBEMaD/GwIkO5x2aDHj3gyBE9DjB2rHESZ7hoPGkI4tDTT3OIcOxzJhZYpZTKBg6IyG60YSj1vozSs6z8siU+d7poYZzMOMmP+34sAVWGck9srO72sVhgyhRo2NC4ijZcMm45qRGRQBG50DbnGqCpiDQUEX/gZmBevjw/oFsDiEh1dFfR/gusp8T5cWMclz2/mMe+3UStKgFc3abgFZpvrH2Dq769ig82f0B0jWg61jIRPg0Xgd2up4O2aAHTpul9V19tjIChWCiyRSAiQ4DX0BHLGopIe2CSUqpQx/dKKauIPAQsQvf/f6yU2iYik4C1Sql5jrT+IrIdsAHjysI6hX/2JyIifHNfVy6LCnW5WjiHP+L+oFloM17u+TKNqjY6P8M/02DTV6Bs56cZDKAXhI0cCX/9pdcDDB7sbUWGcoY7XUMT0DOAlgEopTaKSEN3CldKLQAW5Ns33umzAv7reJUpgvwtXN7QPSesESERro0AwI6f4fQhiLgcqjWAJmb5v8GJGTO0k7igIPj0U7j9drM62FDsuOWGWimVnO+pt9QN2JZparaGW7/xtgpDaaRxYxgyBN59F2rV8rYaQznFHUOwTURuASyOBWCPACs9K6v0cSw5g7HfbiIjy8bhU+n4FPJUtiZ+De9seAe7snMk9QgNq7rVgDIYIDMTJk3Sn198EXr31i+DwYO4M1j8MDpe8VngS7Q76goXj2D70RT+2puITUHLOlW4tXODAvOuPLqSDQkbCPINomPNjgyMGliCSg1llr/+0usBXnpJLw5TpuFtKBncaRG0UEo9DTztaTFlgeeGtiY6wvWiMWd8fXyZ3n96CSgylHlSU+Gpp3SMgAYNYNEi6N/f26oMFQh3WgSvi8gOEXlORNp4XJHBUNGIjdWDwg8/DFu2GCNgKHGKNARKqd7oyGQngA9EZIuI/J/HlRkM5ZnExHPrAVq21E7i3n4bKhu3I4aSx62VxUqpeHRwmt+Bx4HxwPOeFFbWSMtKY/zK8aRmpXI45XDhmeO3wuJnwW6FQ39Cg+4lI9LgfZSC777TLqJPnYI+fbR/IBM20uBFimwRiEhLEZkgIlvQwetXot1FGJzYn7yf3w79RkJ6AjWDanJ9k+sLznxgOexdDGdT9fqB1teVnFCD9zh2DIYPhxtv1AFj1q41TuIMpQJ3WgQfA18DA5RSRz2sp8wzNmYsPSJ6uJf59rkQUNWzggylgxwncXFx8Oqr8J//gK+3fT4aDJoif4lKqa4lIaTcY8uGn8bAmZOQdNDbagwlxZEjUK+edhI3dap2EtesmbdVGQx5KLBrSES+cbxvEZHNTq8tTpHLDO6SfAQ2zoITO8AvEFoOAX8TQrDcYrNp76DOTuIGDDBGwFAqKaxFMMbxbjxcFSe9n4Z2N3tbhcGT7NgB99wDf/+tPYQOMfEnDKWbwiKUHXN8fEAp9T/nNBF5Bfjf+UeVP974dRe7jqeSkHrWZfonWz9h04lNueEnDRWc6dP1eoCQEPj8c7j1VuMkzlDqcWdBWT8X+64ubiGllXd/38vag0lkZNm4LCqUBmHBedJnbpvJmvg1JGclE10jmibVmnhJqaFU0LQpDBsG27fDbbcZI2AoExTYIhCR+4EHgEb5xgRCgL88Law0cUvn+jzWv+BpfgOjBvJM12dKUJGh1JCRARMm6Bv+yy8bJ3GGMklhYwRfAr8ALwFPOO1PVUqd8qgqg6EssGKFDhizZw+MHq0Xi5kWgKEMUljXkFJKHQQeBFKdXoiIexFZDIbySEoKPPAAXHmlnh20ZImeGWSMgKGMUlSLYDCwDh2IxvlXroACQm4ZDOWco0dh5kz473917IDg4CIPMRhKM4XNGhrseDdRVQyGkyfhm290S6BFCzhwwEQMM5Qb3PE1dIWIBDs+3yYib4hIfc9LMxhKAUrB119Dq1bw6KOwe7feb4yAoRzhjrOTaUA7EWkHPAbMAD4HrvSkMG/y196TzNuo3SrZLzVI1JY5sH+ZdjBnKFscPQr33w/z5kFMjB4LMCuDDeUQdwyBVSmlRGQo8K5S6iMRucfTwrzJpysPsnRnAtUrV6JetUC3IpIVyIrXIOkABIZBaBTUbFlsOg0exGaDnj21k7jXXoMxY4yTOEO5xZ1fdqqIPAncDvQQER/Az7OyvE+TmpVZ+GjPizs4YSesng7KDqlHoWl/uOnz4hVo8AyHDkFEhHYS99570KgRNDGLBA3lG3dWFt+EDlx/tyNATQQw2aOqyjpbvoG1H8HO+WCpBJGXe1uRoShsNnjjDR0tLMdJXP/+xggYKgTuuKGOF5FZwGUiMhhYrZT6zPPSyjBKgY8fjNvjbSUGd9i6VTuJW70aBg+G60ygIEPFwp1ZQ/8CVgM3Av8CVonIDZ4WZjCUCO+/Dx076pjBX36pB4YjTAA+Q8XCnTGCp4HLlFIJACJSA1gMzPGksJLmZNpZZvxxgCyrnZ3xqQT5W7wtyeBJctxBtGypQ0e+9RbUqOFtVQaDV3DHEPjkGAEHibg3tlCmWLbrBO8v30ewvwUfEbo1NsHEyyXp6TB+vB4MfuUV7SbiynI7E9pgcAt3DMFCEVkEzHZs3wQs8Jwk76CUXjCw8NGeRIYFeVmNwSMsW6adxO3bp1cIGydxBgPgxpO9Umoc8AEQ7XhNzx+oxmAo1SQnw333nXMPvXSpjh9sjIDBABQej6Ap8BrQGNgCjFVKxZWUMIOh2Dh2DL74AsaOhYkTIci0+AwGZwprEXwM/AwMR3sgfedCCxeRgSKyS0T2isgTheQbLiJKRGIutA6DwSUnTsA7jp9sixZw8CBMnmyMgMHggsLGCEKUUh86Pu8SkfUXUrCIWICp6FCXscAaEZmnlNqeL18IMAZYdSHle5P9p/fzy8FfUEqRYc3wthyDM0rB7NnwyCM6bsCAAdo/kJkRZDAUSGGGIEBEOnAuDkGg87ZSqijDcDmwVym1H0BEvgKGAtvz5XsOeAUYd4HavcasHbP4Zvc3CIKI0KiaCc1QKjhyRDuJmz8fOneGjz4yTuIMBjcozBAcA95w2o532lZAnyLKrgcccdqOBTo7ZxCRjkCkUmq+iBRoCERkFDAKoH5973vAtikbNQJrsPRfS89PtNvgzImSF1XRsVqhVy+Ij4c334SHH9ZTRA0GQ5EUFpjGoxG4Hc7r3gDuKiqvUmo6MB0gJibmUh1DewZrFqyYDBu+0I7mwpt6W1HF4OBBiIzUnkE/+EA7iWtkWmgGw4XgyYVhcUCk03aEY18OIUAbYJmIHAS6APO8NWBclHXJtGbyxfYvmLFlBjtP7Tw/w/EtsOJVCGsIN86E0X96QqYhB6tVu4du2VJ7CQW46ipjBAyGi8CTDtbXAE1FpCHaANwM3JKTqJRKBqrnbIvIMvQU1bUe1OSS9Cwrs/45RKCfhWpBrj1srz2+llfWvJK73alWp7wZziTq9+7/gab9PCXVALB5s3YSt3YtDB0Kw4d7W5HBUKbxmCFQSllF5CFgEWABPlZKbRORScBapdQ8T9V9IVhtdh6ctZ4tcclMvz2GkADXhsBmtwHw+dWf0yq8Fb4+vjrq2LYfYOOXcHgliAVCapek/IrHe+/pIDGhoTqE5I03moVhBsMlUqQhEBEBbgUaKaUmOeIV11ZKrS7qWKXUAvK5o1BKjS8gby+3FBczz/y4ld93neDFYW25qlUhcWgzkwHw2/o9/gF/QPxW2DEPstP1eEDfZyH6Jqhar4SUVzBy3EG0aQM336wHhKtXL/o4g8FQJO60CN4D7OhZQpOAVOA74DIP6ioRUjKzmb36CLd3acAtnV3PRlpxcDEHt85m3+E/ILgS/DMNsrKgUlV9429/K0TEmKdST3HmDPzf/+nB4MmTdfjInhcZOc5gMLjEHUPQWSnVUUQ2ACilkkTE38O6PMq2o8msPnCK9Czd3dOoRvD5mZSCrd/x6LqJZAsQXAl/Hz/CR/8NwbXANxAsJoatR1myBO69Fw4c0NNBjZM4g8EjuHMny3asElaQG4/A7lFVHub5n3fw9/7E3O06VQPzZjixG358AGLXYIuqz131+jKq53P4W/ypZKlUwmorIKdPa79AH30ETZvCihXQo4e3VRkM5RZ3DMEUYC5QU0ReAG4A/s+jqjyM1W7n8qgwpt/RCYuPnD9AvPxlSNgBQ6fClsn4hzcmxD/EO2IrIsePw1dfwf/+B88+C4GBRR9jMBguGndiFs8SkXVAX7R7ieuUUjs8rszD+FqEakEF9HBZz0JoFHS4DbZMLlFdFZacm/+YMdC8uV4oZgaDDYYSwZ1ZQ/WBdOAn531KqcOeFFbiZJ2B7T9qI7DzZ6jZ2tuKKgZKwaxZ2gCkpcGgQbo7yBgBg6HEcKdraD56fECAAKAhsAsoX3fKnQvgh/vPbZv1AJ7n8GEYPRp++QW6dj03JmAwGEoUd7qG2jpvOxzFPeAxRd7ClqXfRy6BKvUg2DyRepQcJ3EJCTBlig4daZzEGQxe4YLnPyql1otI56JzllGCa3DUR7HmgF4HZ1dleoJU6WP/fmjQQK8L+PBDaNwYoqK8rcpgqNC4M0bwX6dNH6AjcNRjirxEpj2bZcFBZB9ZyvhNU7DarblpYQFhXlRWTrBa4fXX9SygV1/VgWP69vW2KoPBgHstAud5k1b0mMF3npFT8qyJX8Px9OP8cugHVtSsDht0yIWoKlFMu2oaFrFQO9iMF1wSGzdqJ3Hr18OwYdo/kMFgKDUUaggcC8lClFJjS0hPiXIm+wz3LLoH5eSEenaPN6havQU1gmoQ4BvgRXXlhHffhf/8B8LDYc4c4ynUYCiFFGgIRMTX4UH0ipIUVJJY7VYUinvb3svQLCHk1/GEXdsMqkQWfbChcHLcQURHw623whtvQJjpYjMYSiOFtQhWo8cDNorIPOBb4ExOolLqew9rKzHCA8NpIIDdDAxfMmlp8PTT4OenA8cYJ3EGQ6nHnQhlAUAi2vvoYGCI491gyMuvv2o30e+8A9nZulVgMBhKPYW1CGo6Zgxt5dyCshzMP9xwjqQk+O9/YeZM7R5ixQro3t3bqgwGg5sUZggsQGXyGoAcjCEwnCMhQQ8EP/kkjB8PAWaQ3WAoSxRmCI4ppSaVmJISJJskMiSOlUeT9I6EHZBh866oskZ8PMyerWcE5TiJCw/3tiqDwXARFGYIym0EkHj/zzhj2c7jK/R25TUfQ9oZEB/wr+xdcaUdpeCzz7QBSE+HwYO1fyBjBAyGMkthg8XldtmnnSwC7Q2Y3fd9vo07xpCG18C9S+GRDRBsbmgFcvAgDBwId90FrVrphWLGSZzBUOYpsEWglDpVkkJKGh8CaRPWErKyoV4n/TIUjNUKvXvDyZMwdar2GurjzqQzg8FQ2jFBdw2Fs3cvNGyoncR9/DE0aqSdxhkMhnKDeaQzuCY7G158EVq31i0A0C0CYwQMhnJHhWwR2OwK3wp55m6yfr12Erdxo3YQd9NN3lZkMBg8SIVrERxPySQrO4vqvpkQu9bbckofU6bA5Zfr6aHffw/ffAO1anlblcFg8CAV7rn4950JNJJj1EhLgy8d7pD9g70rqjSQ4ySuQwe44w4dOyA01NuqDAZDCVDhDMGSnQn4ih3lFwy3/wAWP4gsvwHXiiQ1Va8IrlRJ3/x79NAvg8FQYahQhiAz28afe04SXV8Qix807u1tSd5l4UK47z44cgQeffRcq6CCkp2dTWxsLJmZmd6WYjBcNAEBAURERODn5+f2MRXKEPyzP5GMbBu+lgo3NJKXxETtJO6zz6BlS/jrL+ja1duqvE5sbCwhISFERUUhFdggGsouSikSExOJjY2lYcOGbh9Xoe6IS3cmEOhnweJTwf/kiYkwdy488wxs2GCMgIPMzEzCw8ONETCUWUSE8PDwC27VetQQiMhAEdklIntF5AkX6f8Vke0isllEloiIxyapK6VYsiOB7k2rl18nSoVx7JgOFKMUNGsGhw7BpEl6bMCQizEChrLOxfyGPWYIHPGOpwJXA62AESLSKl+2DUCMUioamAO86ik9u4+nEXc6g74tanqqitKJUnpFcMuWugWwd6/eb2YEGQwGB55sEVwO7FVK7VdKZQFfAUOdMyilfldKpTs2/wEiPCXm730nAejZrIanqih9HDgA/fvrxWHt2sGmTcZJnMFgOA9PGoJ6wBGn7VjHvoK4B/jFVYKIjBKRtSKy9sSJExcl5qxVxyOuFuT+SHqZxmqFPn1g1SqYNg1+/113CRlKNSLCbbfdlrtttVqpUaMGgwdfWHTYqKgoTp48eUl5LBYL7du3p02bNgwZMoTTp08DcPDgQQIDA2nfvn3uKysry2UZjz76KPXq1cPuFA98woQJvPbaawVqiY+P5+abb6Zx48Z06tSJQYMGsXv3brfO29t8+umnNG3alKZNm/Lpp5+6zLNx40a6dOlC+/btiYmJYfXq1blpy5Yto3379rRu3Zorr7yypGSXjsFiEbkNiAEmu0pXSk1XSsUopWJq1KhAT/QXw549YLNpJ3GffALbthlPoWWI4OBgtm7dSkZGBgC//fYb9eoV9vzkOQIDA9m4cSNbt24lLCyMqTk+p4DGjRuzcePG3Je/v/95x9vtdubOnUtkZCTLly93q06lFMOGDaNXr17s27ePdevW8dJLL3H8+PGLOgebreQCTp06dYqJEyeyatUqVq9ezcSJE0lKSjov3+OPP86zzz7Lxo0bmTRpEo8//jgAp0+f5oEHHmDevHls27aNb7/9tsS0e3L6aBwQ6bQd4diXBxG5CngauFIpddaDeso32dnwyivw3HPw6qswZgz06uVtVWWWiT9tY/vRlGIts1XdKjw7pHWR+QYNGsT8+fO54YYbmD17NiNGjOCPP/4A9M3m7rvvZv/+/QQFBTF9+nSio6NJTExkxIgRxMXF0bVrV5Q6F032iy++YMqUKWRlZdG5c2fee+89LBbLBWnv2rUrmzdvvqBjli1bRuvWrbnpppuYPXs2vXsXvW7n999/x8/Pj9GjR+fua9euncu81113HUeOHCEzM5MxY8YwatQoACpXrsx9993H4sWLmTp1KgcPHnR5/vfffz9r1qwhIyODG264gYkTJ17Q+eVn0aJF9OvXj7CwMAD69evHwoULGTFiRJ58IkJKiv5tJScnU7duXQC+/PJLrr/+eurXrw9AzZolN57pycfENUBTEWkoIv7AzcA85wwi0gH4ALhWKZXgQS0opRDf08SlxZJZ3kIur10LMTF6MPj66yHfD89Qtrj55pv56quvyMzMZPPmzXTufG7l+7PPPkuHDh3YvHkzL774InfccQcAEydOpHv37mzbto1hw4Zx+PBhAHbs2MHXX3/NX3/9xcaNG7FYLMyaNeuC9NhsNpYsWcK1116bu2/fvn253UIPPvigy+NyjNiwYcOYP38+2dnZRda1detWOnVyLzbIxx9/zLp161i7di1TpkwhMTERgDNnztC5c2c2bdpEeHh4gef/wgsvsHbtWjZv3szy5ctdGrrJkyfn6QLLeT3yyCPn5Y2LiyMy8tyzb0REBHFx5z378tZbbzFu3DgiIyMZO3YsL730EgC7d+8mKSmJXr160alTJz777DO3rkNx4LEWgVLKKiIPAYsAC/CxUmqbiEwC1iql5qG7gioD3zqmPB1WSl1bYKGXwM60pVRu+g7X/wQI9Cgva+neflsvDqtdG378Ea71yOWrcLjz5O4poqOjOXjwILNnz2bQoEF50v7880++++47APr06UNiYiIpKSmsWLGC77//HoBrrrmGUMessCVLlrBu3Touu+wyADIyMtx+0szIyKB9+/bExcXRsmVL+vXrl5uW0zVUEFlZWSxYsIA33niDkJAQOnfuzKJFixg8eHCB0xsvdNrjlClTmDt3LgBHjhxhz549hIeHY7FYGD58OFD4+X/zzTdMnz4dq9XKsWPH2L59O9HR0XnqGDduHOPGjbsgXUUxbdo03nzzTYYPH84333zDPffcw+LFi7Faraxbt44lS5aQkZFB165d6dKlC81KYGzPo3dDpdQCYEG+feOdPl/lyfqdybDrptiTlz1D5VXv0sEnqKSq9gw57iBiYvSsoFdfhWrVvK3KUExce+21jB07lmXLluU+6V4MSinuvPPO3KfOCyFnjCA9PZ0BAwYwdepUl0/Crli0aBGnT5+mbdu2AKSnpxMYGMjgwYMJDw/n2LFjefKnpqZSrVo1WrduzZw5c4osf9myZSxevJi///6boKAgevXqlbuIKiAgILfrq6DzP3DgAK+99hpr1qwhNDSUu+66y+UirMmTJ7tsQfXs2ZMpU6bk2VevXj2WLVuWux0bG0svF92zn376KW+//TYAN954IyNHjgR0CyI8PJzg4GCCg4Pp2bMnmzZtKhFDUOFGEAdEXc21KohIyujsoZQUuP9+3QoAuOIKmD7dGIFyxt13382zzz6beyPNoUePHrk3pmXLllG9enWqVKlCz549+fLLLwH45Zdfcgcp+/bty5w5c0hI0D2vp06d4tChQxekJSgoiClTpvD6669jtVrdOmb27NnMmDGDgwcPcvDgQQ4cOMBvv/1Geno6PXv2ZN68eaSmpgLw/fff065dOywWC3369OHs2bNMnz49t6zNmzfnjpHkkJycTGhoKEFBQezcuZN//vnHpY6Czj8lJYXg4GCqVq3K8ePH+eUXlxMWGTduXJ5B8ZxXfiMAMGDAAH799VeSkpJISkri119/ZcCAAeflq1u3bu7g+dKlS2nqmNI9dOhQ/vzzT6xWK+np6axatYqWLVsWdamLhXLSP1JBWLBAO4k7elQbggruJK48ExER4fLpe8KECdx9991ER0cTFBSUO0Xx2WefZcSIEbRu3Zpu3brlDji2atWK559/nv79+2O32/Hz82Pq1Kk0uMBIcx06dCA6OprZs2fTowjvtOnp6SxcuJD3338/d19wcDDdu3fnp59+4qabbuKhhx6ie/fuiAg1a9ZkxowZgO4emjt3Lo8++iivvPIKAQEBREVF8dZbb+WpY+DAgbz//vu0bNmS5s2b06VLF5daCjr/Ll260KFDB1q0aEFkZCRXXHHFBV0PV4SFhfHMM8/kdkONHz8+d+B45MiRjB49mpiYGD788EPGjBmD1WolICAg1+i1bNmSgQMHEh0djY+PDyNHTom4gAAAGm9JREFUjqRNmzaXrMsdxHl2QVkgJiZGrV174QFl7pv3GiuTPmXZjSsJ/+pG7X76zp88oNADnDypvYPOmqVDR370EXSuwK6zPcSOHTtK7AnMYPAkrn7LIrJOKRXjKn+F6xoqkyQlwU8/wbPP6jCSxggYDIZixHQNlVbi4nQLYNw47Rbi0CEzDmAodhITE+nbt+95+5csWUJ4eLgXFBm8gTEEpQ2lYMYMGDtWLxK7/npo0sQYAYNHCA8PL3QaqKFiYLqGShP79kHfvjBqFHTsCJs3ayNgMBgMHsS0CEoLVqs2AqdOwQcfwMiRxj+QwWAoEYwh8Da7dkHjxtpJ3Kef6s8RHvPGbTAYDOdhHjm9RVYWTJwIbdtCjlfHK680RsBgMJQ4xhB4g9WroVMnmDABbrwRbr3V24oMpYTSFI/ghRdeoHXr1kRHR9O+fXtWrVrFxIkTefLJJ/Pk27hxY+6c9aioqPMWnOXENCiIt956i4CAAJKTk3P3zZw5k4ceeihPvl69epGzhigtLY377rsvN2ZBr169WLVqVaHnW1pYuHAhzZs3p0mTJrz88ssu8xw+fJjevXvnLuRbsEB76lm9enWu47t27drl+lq6VEzXUEnz1lvw2GNQp45eG3CBf3BDCfHLExC/pXjLrN0Wrnb9x8/BOR5BYGCg1+IR/P333/z888+sX7+eSpUqcfLkSbKyshgxYgQDBw7M47vnq6++yuNqOTU1lSNHjhAZGcmOHTuKrGv27NlcdtllfP/9/7d37vFVVVce/y4IBAgi1qSIECAYHubBvYZASC0TLEUsymsSCYyiQR42Aoq1I1S0PJq2UrR+UPTDyCtQnBsBGUSnI3V4CCgoUI0kAQTklRI1gAgZHoGw5o9zcpvk3pAb8iK5+/v5nE/OY5+z1zpJzjp777PXbw1jxozxyb5x48YRFhbGgQMHaNSoEYcPHyYnJ6fyjmJlV61sWu7rpaioiIkTJ/Lhhx/Svn17evXqxZAhQ4iIKK3im5aWxogRI0hNTSUnJ4dBgwZx5MgRoqKi2LVrFwEBAeTl5eFwOBg8eDABAVV7lJsWQW1RPIO7d28YP94SjDFBwOCFYj0C+Gcq52JOnz7NsGHD6NGjB3369HGnTj516hT33nsvkZGRjBs3zkOPoHfv3jidTh5//HGfxFry8vIIDg4mMDAQgODgYG6//Xa6du3KLbfcUurte+XKlaVsHDFiBG+//bZX+8ty6NAhCgoKSEtLw+Vy+XJ7OHToEJ9++ilpaWk0sj+oCAsL4/777/com5qaSmxsLJGRkcyYMcO9v1OnTkydOpWYmBhWrVrF3/72N+Lj44mJieHBBx+koKAAgNmzZ9OrVy+ioqKYMGECVc3E8NlnnxEeHk7nzp1p2rQpI0eO5N133/UoV55mQYsWLdwP/YsXL16XUL1XVLVeLT179tTrYcK7czUqPUpP/t9Z1cUDVdMfuK7rVJozZ1QnTFB96qnaqc9w3eTk5NS1CRoUFKSZmZmamJioFy5cUIfDoZs2bdL7779fVVUnTZqkM2fOVFXVDRs2qMPhUFXVyZMn66xZs1RV9f3331dA8/PzNScnRx944AEtLCxUVdXU1FRdtmyZqqp27NhR8/Pzvdpx7tw5dTgc2qVLF01NTdXNmze7j82dO1enTJmiqqrbt2/Xkv+THTt21H379ml8fLyqqjqdTs3OztbIyEiv9aSlpens2bO1qKhIO3TooN98842qqi5dulQnTpxYqmxCQoLu3LlT3333XR02bJhP9/PUqVOqqnrlyhVNSEjQzMxMt51z5sxRVdX8/Hzt27evFhQUqKrqiy++6L6Xxeerqj788MO6bt06jzpWrFihDofDY0lMTPQou2rVKh07dqx7e/ny5R5+qqqeOHFCo6KitF27dtq6dWvdtWuX+9iOHTs0IiJCg4KCdM2aNV799va3jJX+3+tz1W9aBLcUHAKg+dxQOLYdpBaagu+9BxER1gSxwMB/tgoMhmtQkR7B6NGjAU89guKxhfL0CJxOJxs2bODrr7+u0IaWLVuye/du3nzzTUJCQkhOTiY9PR2A5ORkVq9ezdWrVz26hcCapHbLLbeQkZHBnXfeSYsW5ad8d7lcjBw5kkaNGpGYmOiWZ6wuzYKVK1cSExPDXXfdRXZ2dqnuo+TkZAB27NhBTk4Od999N06nk2XLlrkztG7atIm4uDiio6PZuHEj2dnZHnU89NBDXjOU+pJOuzxcLhcpKSnk5uby17/+ldGjR7t1n+Pi4sjOzmbnzp388Y9/9Jo+u7L4zRhB8ytWWt4rvZ+AZi0hfEAFZ1SB/HxLKtLlsr4KWrsW7IyEBoMv3Ah6BI0bN6Zfv37069eP6Oholi1bRkpKCqGhoYSFhfHRRx/xzjvvsH37do9zk5OTmThxojt4eGPPnj0cOHDALXhTWFhIWFgYkyZN4tZbb/XQ+z19+jTBwcG0bt2azMzMCvv2K9IcCAoKAqx7NGDAAI+uqYsXL/LEE0+wa9cuQkNDmTlzpteH7ltvvcXcuZ5y6+Hh4R7BoF27dhw/fty9nZub63UMaPHixXzwwQeAJRN68eJFTp48WUpU6M4776Rly5ZkZWURG+s1l5zP+E2LoJjL8U/CPc9BaA0+mH/4wUoZPWuWJSNpgoChktS1HsH+/fs5cOCAe/uLL74olbp61KhRPP3003Tu3Jn2Xj55Hj58OM8++6zXfPzFuFwuZs6c6dYsOHHiBCdOnODo0aP06tWLjz/+mG+++QaAXbt2cenSJUJDQ7njjjuIjY1lxowZ7j77I0eOuMdVivFVc6BPnz58/PHHHDx4ELCkLr/66iv3Qz84OJiCgoJy3/Ar0yLo1asXBw4c4PDhwxQWFpKRkVFKArSYDh06sGHDBsDKJHrx4kVCQkI4fPiwWxPi6NGj7Nu3j06dOpV7j33Fb1oENc7x47BiBUybZqWFOHoUbr65rq0y1FPqWo+goKCAyZMnc+bMGQICAggPDy8lFvPggw/y5JNP8tprr3k9/6abbmLq1KnXrCMjI8P9WWQxw4cPJyMjg6lTpzJv3jwGDRrE1atXadmyJS6Xyz04vGjRIp555hnCw8Np3rw5wcHBHm/lDofDJ82BkJAQ0tPTGTVqFJcuXQKsr3a6du3K+PHjiYqK4rbbbnPrDFSFgIAA5s+fz8CBAykqKuKxxx4jMtKSRf3tb39LbGwsQ4YM4eWXX2b8+PG88soriAjp6emICNu2bePFF1+kSZMmNGrUiDfeeIPg4OAq2+U3egSzlo9ktWazeeiH3Nr6tuoz6OpVSyHs2WehqAgyM01+oHqK0SMwNBSMHkFtcuAA/OxnlnRk796wZ48JAgaDod5huoaulytXYMAAOHPGUgwbM8bIRhrqHUaPwAAmEFSevXstoZiAAPjLX6wkcfZkD4OhvmH0CAxguoZ859IlSyqyRw+YP9/a17evCQIGg6HeY1oEvrBjB4wdCzk5MHq0tRgMBkMDwbQIKuLll+EnP4Fz56y5AcuXg+k7NRgMDQgTCMrDns5NfDz88peQlQW/+EXd2mQwGAw1gAkEZTlzxuoGeuopa/snP4E33oBWrerWLoNfcCPpEQCsXbsWEWHfvn3ufUeOHEFESk0mmzRpkjudREpKCu3atXNPzjp58uQ1Z796q2Pz5s0ePqekpLhn616+fJlp06bRpUsXYmJiiI+PL3fm8I3G7t27iY6OJjw8nCeffNJrRtMffviBwYMH43A4iIyMZOnSpYA1mzgmJgan00lkZCQLFiyoFpvMGEFJ1q6FJ56A776zJoipmk9C/ZQ5n81h3+l9FResBN1/1J2pva892/ZG0SMoxuVy8dOf/hSXy8WsWbPc+3/84x8zb948Hn/8cZo2bepxXuPGjVmyZAmpqanXXce1eOGFF8jLyyMrK4vAwEC+/fZbPvroI98dK8GVK1eqnM+/MqSmprJw4ULi4uIYNGgQH3zwAb8o09vw+uuvExERwXvvvUd+fj7dunXjoYceom3btmzfvp3AwEAKCgqIiopiyJAh7jTV14tpEYD14B8xAoYPhzZtLAWxP/zBBAFDnXAj6BGAlWZi27ZtLF68mIyMjFLHQkJC6N+/vzvFRVmmTJnCK6+84s6Lcz11lMf58+dZuHAhr732mlsvoU2bNowYMcKjbHl6Av369WPKlCnExsYyb948du/eTUJCAj179mTgwIHk5eUBsHDhQnr16oXD4SAxMZHz58/7ZGN55OXlcfbsWfr06YOI8Mgjj7B27VqPciLCuXPnUFUKCgr40Y9+REBAAE2bNnX7fOnSJXdG0ipTXn7qG3W5Xj2CmcuSLT2C7/M8Dx44oNq6tervf69q5203+B9Gj6A0K1as0Mcee0xVVePj49058Q8fPqyRkZF66NAh7dq1q165ckUnTpyoS5cuVVXVRx99VFetWqVjxozRJUuWaH5+vnbs2LFSdZT0uZji62ZmZqrT6fTpfpanJ5CQkKCpqamqqlpYWKjx8fH63XffqapqRkaGjhkzRlVVT5486T5/+vTp+uqrr3rUsXHjRq96BMWaDCXZuXOn9u/f3729ZcsWDz9VVc+ePav9+vXT2267TYOCgvT99993Hzt27JhGR0dr8+bNdf78+V79rqwegf92DR07Zk0Ie+45Ky3EsWNw0011bZXBUKEewTvvvAN46hGsWbMGKF+PAODChQulUhlfC5fLxVP2WNnIkSNxuVz07NnTfbxz587ExcW5s56W5Te/+Q1Dhw71qhxWUR3VpUewadMm/vSnP3H+/HlOnz5NZGQkgwcPBv6pR7B//36ysrLc6bCLiopo27YtAFlZWTz//POcOXOGgoICr9lU77nnnmqflLd+/XqcTicbN27k0KFDDBgwgL59+9KqVStCQ0P58ssvOXHiBMOGDSMpKYk2bdpUqb4aDQQich8wD2gMLFLVF8scDwSWAz2BU0Cyqh6pSZu4etUa/J061VpPTrYCgQkChhuIutYjOH36NBs3bmTPnj2ICEVFRYiIR4bP5557jqSkJBISEjyu0aVLF5xOJytXrqx0HdfSIwgPD+fYsWOcPXuWVtf4iKMiPYGSegSRkZFedRVSUlJYu3YtDoeD9PR0Nm/e7FFm06ZNPP300x77W7RowSeffFJqX7t27cjNzXVvl6dHsHTpUqZNm4aIEB4eTlhYGPv27aN3797uMrfffjtRUVFs3bqVpKSkcu+DL9TYGIGINAZeB34BRACjRCSiTLGxwPeqGg68AsypKXsAOuVdotUD/woTJ1qfhWZnmyRxhhuSutYjWL16NaNHj+bo0aMcOXKE48ePExYWxtatW0uV6969u3tQ0xvTp0/npZdeqnQdXbp04cSJE+zduxewvpbJzMzE6XTSokULxo4dy1NPPUVhYSEA+fn5bnWzYnzVE+jWrRv5+fnuQHD58mW3Etm5c+do27Ytly9fdt/3shS3CMouZYMAQNu2bWnVqhU7duxAVVm+fDlDhw71KFdSj+Dbb79l//79dO7cmdzcXC5cuADA999/z7Zt2+jWrZtXuypDTQ4W9wYOqurXqloIZABlPR4KFI82rQb6S7WpMZdGiq6y4KUjNM7ZB0uXwvr1UA2CDgZDTXAtPYLdu3fTo0cPpk2bVkqPYMuWLURGRrJmzRqvegQ9evRgwIAB7oHQa+FyuRg+fHipfYmJiV4F5qdPn17qLbckkZGRxMTEVLqOwMBAVqxYwZgxY3A6nSQlJbFo0SJutjU+0tLSCAkJISIigqioKB544AGP1kHr1q3degIDBw4sV0+gadOmrF69mqlTp+JwOHA6ne6H+O9+9zvi4uK4++676d69u9fzK8sbb7zBuHHjCA8P54477nB/MbRgwQL356AvvPACn3zyCdHR0fTv3585c+YQHBzM3r17iYuLw+FwkJCQwK9//WuPl4Xrocb0CEQkCbhPVcfZ26OBOFWdVKJMll0m194+ZJc5WeZaE4AJAB06dOjpyxtNWRatm0H+tlVMnLCaVuFlGyYGg9EjMDQcKqtHUC8Gi1X1TeBNsIRpruca44bMgiG+faNsMBgM/kRNBoJ/AKElttvb+7yVyRWRAOBmrEFjg8FQCxg9AgPUbCDYCXQRkTCsB/5I4N/KlFkHPApsB5KAjVpTfVUGgw+oaqU/UazPGD2Chsf1PEJrbLBYVa8Ak4D1wF5gpapmi8hsERliF1sM3CoiB4FfAdNqyh6DoSKaNWvGqVOnrusfyWC4EVBVTp06RbNmzSp1nt+I1xsMFXH58mVyc3NLfWtuMNQ3mjVrRvv27WnSpEmp/fV+sNhgqA2aNGlCWFhYXZthMNQ6JumcwWAw+DkmEBgMBoOfYwKBwWAw+Dn1brBYRPKByk8ttggGri3J1PAwPvsHxmf/oCo+d1TVEG8H6l0gqAoisqu8UfOGivHZPzA++wc15bPpGjIYDAY/xwQCg8Fg8HP8LRC8WdcG1AHGZ//A+Owf1IjPfjVGYDAYDAZP/K1FYDAYDIYymEBgMBgMfk6DDAQicp+I7BeRgyLikdFURAJF5G37+Kci0qn2raxefPD5VyKSIyJfisgGEelYF3ZWJxX5XKJcooioiNT7Tw198VlERti/62wR+c/atrG68eFvu4OIbBKRz+2/70F1YWd1ISJLROQ7W8HR23ERkVft+/GliHjXAq0MqtqgFqAxcAjoDDQFMoGIMmWeABbY6yOBt+va7lrw+R6ghb2e6g8+2+VuArYAO4DYura7Fn7PXYDPgVvs7R/Xtd214PObQKq9HgEcqWu7q+jzvwAxQFY5xwcB/wMI0Af4tKp1NsQWQW/goKp+raqFQAYwtEyZocAye3010F/qtxpJhT6r6iZVPW9v7sBSjKvP+PJ7BvgdMAdoCLmlffF5PPC6qn4PoKrf1bKN1Y0vPitQrFx/M3CiFu2rdlR1C3D6GkWGAsvVYgfQWkTaVqXOhhgI2gHHS2zn2vu8llFLQOcHoD7r8vnic0nGYr1R1Gcq9NluMoeq6n/XpmE1iC+/565AVxH5WER2iMh9tWZdzeCLzzOBh0UkF/grMLl2TKszKvv/XiFGj8DPEJGHgVggoa5tqUlEpBHwZyCljk2pbQKwuof6YbX6tohItKqeqVOrapZRQLqqviwi8cBfRCRKVa/WtWH1hYbYIvgHEFpiu729z2sZEQnAak6eqhXragZffEZEfg5MB4ao6qVasq2mqMjnm4AoYLOIHMHqS11XzweMffk95wLrVPWyqh4GvsIKDPUVX3weC6wEUNXtQDOs5GwNFZ/+3ytDQwwEO4EuIhImIk2xBoPXlSmzDnjUXk8CNqo9ClNPqdBnEbkL+A+sIFDf+42hAp9V9QdVDVbVTqraCWtcZIiq1medU1/+ttditQYQkWCsrqKva9PIasYXn48B/QFE5E6sQJBfq1bWLuuAR+yvh/oAP6hqXlUu2OC6hlT1iohMAtZjfXGwRFWzRWQ2sEtV1wGLsZqPB7EGZUbWncVVx0ef5wItgVX2uPgxVR1SZ0ZXER99blD46PN64F4RyQGKgH9X1Xrb2vXR52eAhSLyNNbAcUp9frETERdWMA+2xz1mAE0AVHUB1jjIIOAgcB4YU+U66/H9MhgMBkM10BC7hgwGg8FQCUwgMBgMBj/HBAKDwWDwc0wgMBgMBj/HBAKDwWDwc0wgMNyQiEiRiHxRYul0jbIF1VBfuogctuv6uz1DtbLXWCQiEfb6c2WOfVJVG+3rFN+XLBF5T0RaV1DeWd+zcRpqHvP5qOGGREQKVLVldZe9xjXSgfdVdbWI3Au8pKo9qnC9KttU0XVFZBnwlar+/hrlU7Cyrk6qblsMDQfTIjDUC0Skpa2j8HcR2SMiHplGRaStiGwp8cbc195/r4hst89dJSIVPaC3AOH2ub+yr5UlIlPsfUEi8t8ikmnvT7b3bxaRWBF5EWhu2/GWfazA/pkhIveXsDldRJJEpLGIzBWRnXaO+cd9uC3bsZONiUhv28fPReQTEelmz8SdDSTbtiTbti8Rkc/sst4ythr8jbrOvW0Ws3hbsGbFfmEv/4U1C76VfSwYa1ZlcYu2wP75DDDdXm+MlW8oGOvBHmTvnwr81kt96UCSvf4g8CnQE9gDBGHNys4G7gISgYUlzr3Z/rkZW/Og2KYSZYptHA4ss9ebYmWRbA5MAJ639wcCu4AwL3YWlPBvFXCfvd0KCLDXfw68Y6+nAPNLnP8H4GF7vTVWLqKguv59m6VulwaXYsLQYLigqs7iDRFpAvxBRP4FuIr1JtwG+KbEOTuBJXbZtar6hYgkYImVfGyn1miK9Sbtjbki8jxWnpqxWPlr/ktV/8+2YQ3QF/gAeFlE5mB1J22thF//A8wTkUDgPmCLql6wu6N6iEiSXe5mrGRxh8uc31xEvrD93wt8WKL8MhHpgpVmoUk59d8LDBGRX9vbzYAO9rUMfooJBIb6wkNACNBTVS+LlVG0WckCqrrFDhT3A+ki8mfge+BDVR3lQx3/rqqrizdEpL+3Qqr6lVhaB4OANBHZoKqzfXFCVS+KyGZgIJCMJbQCltrUZFVdX8ElLqiqU0RaYOXfmQi8iiXAs0lVh9sD65vLOV+ARFXd74u9Bv/AjBEY6gs3A9/ZQeAewENzWSwd5m9VdSGwCEvubwdwt4gU9/kHiUhXH+vcCgwTkRYiEoTVrbNVRG4HzqvqCqxkft40Yy/bLRNvvI2VKKy4dQHWQz21+BwR6WrX6RW11OaeBJ6Rf6ZSL05FnFKi6DmsLrJi1gOTxW4eiZWV1uDnmEBgqC+8BcSKyB7gEWCflzL9gEwR+RzrbXuequZjPRhdIvIlVrdQd18qVNW/Y40dfIY1ZrBIVT8HooHP7C6aGUCal9PfBL4sHiwuw9+whIH+Vy35RbACVw7wd7FEy/+DClrsti1fYgmz/An4o+17yfM2ARHFg8VYLYcmtm3Z9rbBzzGfjxoMBoOfY1oEBoPB4OeYQGAwGAx+jgkEBoPB4OeYQGAwGAx+jgkEBoPB4OeYQGAwGAx+jgkEBoPB4Of8P3oH5QRPJs/5AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def crossvalidation(ml_model, df, n_folds=5, verbose=False):\n",
        "    \"\"\"\n",
        "    Machine learning model training and validation in a cross-validation loop.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    ml_model: sklearn model object\n",
        "        The machine learning model to train.\n",
        "    df: pd.DataFrame\n",
        "        Data set with SMILES and their associated activity labels.\n",
        "    n_folds: int, optional\n",
        "        Number of folds for cross-validation.\n",
        "    verbose: bool, optional\n",
        "        Performance measures are printed.\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    None\n",
        "\n",
        "    \"\"\"\n",
        "    t0 = time.time()\n",
        "    # Shuffle the indices for the k-fold cross-validation\n",
        "    kf = KFold(n_splits=n_folds, shuffle=True, random_state=SEED)\n",
        "\n",
        "    # Results for each of the cross-validation folds\n",
        "    acc_per_fold = []\n",
        "    sens_per_fold = []\n",
        "    spec_per_fold = []\n",
        "    auc_per_fold = []\n",
        "\n",
        "    # Loop over the folds\n",
        "    for train_index, test_index in kf.split(df):\n",
        "        # clone model -- we want a fresh copy per fold!\n",
        "        fold_model = clone(ml_model)\n",
        "        # Training\n",
        "\n",
        "        # Convert the fingerprint and the label to a list\n",
        "        train_x = df.iloc[train_index].fp.tolist()\n",
        "        train_y = df.iloc[train_index].active.tolist()\n",
        "\n",
        "        # Fit the model\n",
        "        fold_model.fit(train_x, train_y)\n",
        "\n",
        "        # Testing\n",
        "\n",
        "        # Convert the fingerprint and the label to a list\n",
        "        test_x = df.iloc[test_index].fp.tolist()\n",
        "        test_y = df.iloc[test_index].active.tolist()\n",
        "\n",
        "        # Performance for each fold\n",
        "        accuracy, sens, spec, auc = model_performance(fold_model, test_x, test_y, verbose)\n",
        "\n",
        "        # Save results\n",
        "        acc_per_fold.append(accuracy)\n",
        "        sens_per_fold.append(sens)\n",
        "        spec_per_fold.append(spec)\n",
        "        auc_per_fold.append(auc)\n",
        "\n",
        "    # Print statistics of results\n",
        "    print(\n",
        "        f\"Mean accuracy: {np.mean(acc_per_fold):.2f} \\t\"\n",
        "        f\"and std : {np.std(acc_per_fold):.2f} \\n\"\n",
        "        f\"Mean sensitivity: {np.mean(sens_per_fold):.2f} \\t\"\n",
        "        f\"and std : {np.std(sens_per_fold):.2f} \\n\"\n",
        "        f\"Mean specificity: {np.mean(spec_per_fold):.2f} \\t\"\n",
        "        f\"and std : {np.std(spec_per_fold):.2f} \\n\"\n",
        "        f\"Mean AUC: {np.mean(auc_per_fold):.2f} \\t\"\n",
        "        f\"and std : {np.std(auc_per_fold):.2f} \\n\"\n",
        "        f\"Time taken : {time.time() - t0:.2f}s\\n\"\n",
        "    )\n",
        "\n",
        "    return acc_per_fold, sens_per_fold, spec_per_fold, auc_per_fold"
      ],
      "metadata": {
        "id": "1zuuMlr8VXR5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "N_FOLDS = 10"
      ],
      "metadata": {
        "id": "Xs_3cFU8VeIv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for model in models:\n",
        "    print(\"\\n======= \")\n",
        "    print(f\"{model['label']}\")\n",
        "    crossvalidation(model[\"model\"], chembl_df, n_folds=N_FOLDS)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sLOCCUQEURDT",
        "outputId": "f3eb74a7-6de4-4a61-da4e-2f5d13416ea0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======= \n",
            "Model_RF\n",
            "Mean accuracy: 0.77 \tand std : 0.03 \n",
            "Mean sensitivity: 0.83 \tand std : 0.04 \n",
            "Mean specificity: 0.67 \tand std : 0.07 \n",
            "Mean AUC: 0.84 \tand std : 0.03 \n",
            "Time taken : 4.76s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_SVM\n",
            "Mean accuracy: 0.78 \tand std : 0.04 \n",
            "Mean sensitivity: 0.87 \tand std : 0.05 \n",
            "Mean specificity: 0.62 \tand std : 0.08 \n",
            "Mean AUC: 0.85 \tand std : 0.03 \n",
            "Time taken : 11.57s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.77 \tand std : 0.03 \n",
            "Mean sensitivity: 0.86 \tand std : 0.04 \n",
            "Mean specificity: 0.60 \tand std : 0.06 \n",
            "Mean AUC: 0.84 \tand std : 0.03 \n",
            "Time taken : 8.67s\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Reset data frame\n",
        "compound_df = chembl_df.copy()"
      ],
      "metadata": {
        "id": "Pl17-_yWVpRU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Use Morgan fingerprint with radius 3\n",
        "compound_df[\"fp\"] = compound_df[\"smiles\"].apply(smiles_to_fp, args=(\"morgan3\",))\n",
        "compound_df.head(3)\n",
        "# NBVAL_CHECK_OUTPUT"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "UmUOy83KVr6m",
        "outputId": "0be81119-bd8f-409f-a878-cd24a5f90141"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id                                             smiles  \\\n",
              "0      CHEMBL4279047  C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...   \n",
              "1      CHEMBL4278845  C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...   \n",
              "2      CHEMBL4283785  CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...   \n",
              "\n",
              "   pIC50  active                                                 fp  \n",
              "0  10.40    1.00  [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
              "1   9.60    1.00  [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
              "2   9.52    1.00  [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2d98cfad-98e9-4ddb-ab73-9d95289ffc33\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "      <th>active</th>\n",
              "      <th>fp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL4279047</td>\n",
              "      <td>C[C@@H](Nc1nc(N[C@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>10.40</td>\n",
              "      <td>1.00</td>\n",
              "      <td>[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL4278845</td>\n",
              "      <td>C[C@H](Nc1nc(N[C@@H](C)C(F)(F)F)nc(-c2cccc(Cl)...</td>\n",
              "      <td>9.60</td>\n",
              "      <td>1.00</td>\n",
              "      <td>[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL4283785</td>\n",
              "      <td>CC(Nc1nc(NC(C)C(F)(F)F)nc(-c2cccc(Cl)n2)n1)C(F...</td>\n",
              "      <td>9.52</td>\n",
              "      <td>1.00</td>\n",
              "      <td>[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2d98cfad-98e9-4ddb-ab73-9d95289ffc33')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2d98cfad-98e9-4ddb-ab73-9d95289ffc33 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2d98cfad-98e9-4ddb-ab73-9d95289ffc33');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for model in models:\n",
        "    if model[\"label\"] == \"Model_SVM\":\n",
        "        # SVM is super slow with long fingerprints\n",
        "        # and will have a performance similar to RF\n",
        "        # We can skip it in this test, but if you want\n",
        "        # to run it, feel free to replace `continue` with `pass`\n",
        "        continue\n",
        "    print(\"\\n=======\")\n",
        "    print(model[\"label\"])\n",
        "    crossvalidation(model[\"model\"], compound_df, n_folds=N_FOLDS)"
      ],
      "metadata": {
        "id": "5MqfKsELVu0D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Data visualization"
      ],
      "metadata": {
        "id": "12DZancAzSfP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "plt.figure(figsize=(5,5))\n",
        "plt.scatter(x=y, y=y_pred, c=\"#7CAE00\", alpha=0.3)\n",
        "\n",
        "# Add trendline\n",
        "# https://stackoverflow.com/questions/26447191/how-to-add-trendline-in-python-matplotlib-dot-scatter-graphs\n",
        "z = np.polyfit(y, y_pred, 1)\n",
        "p = np.poly1d(z)\n",
        "\n",
        "plt.plot(y,p(y),\"#F8766D\")\n",
        "plt.ylabel('Predicted pIC50')\n",
        "plt.xlabel('Experimental pIC50')"
      ],
      "metadata": {
        "id": "soRqRC06zUn8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Novel drugs prediction"
      ],
      "metadata": {
        "id": "d2JHkKTYrhO-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "smis = [\"CN2C(=O)N(C)C(=O)C1=C2N=CN1C\"]\n",
        "\n",
        "cans = [Chem.MolToSmiles(Chem.MolFromSmiles(smi),True) for smi in smis]\n"
      ],
      "metadata": {
        "id": "lT1tdGN5sFZX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame({'smile': ['CN2C(=O)N(C)C(=O)C1=C2N=CN1C']\n",
        "                   })"
      ],
      "metadata": {
        "id": "YZ_pwLXHtbi1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        },
        "id": "q-rkcUbxtH8a",
        "outputId": "0c075707-63ab-47ab-f60c-931b477325ad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-82-144c7b0decef>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5272\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5273\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5274\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5276\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'dtype'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.to_csv('drugs.csv')"
      ],
      "metadata": {
        "id": "7IzoN5Rouiw5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pred = model_RF.predict([df])\n",
        "print(\"pIC50 prediction\", pred)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 345
        },
        "id": "58LYkwX3sqn1",
        "outputId": "c0cadfad-c5b8-4681-8b63-632b28a174f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-86-a19da2d7db34>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_RF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pIC50 prediction\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/ensemble/_forest.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    627\u001b[0m             \u001b[0mThe\u001b[0m \u001b[0mpredicted\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    628\u001b[0m         \"\"\"\n\u001b[0;32m--> 629\u001b[0;31m         \u001b[0mproba\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    630\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    631\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_outputs_\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/ensemble/_forest.py\u001b[0m in \u001b[0;36mpredict_proba\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    671\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    672\u001b[0m         \u001b[0;31m# Check data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 673\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_X_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    674\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    675\u001b[0m         \u001b[0;31m# Assign chunk of trees to jobs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/ensemble/_forest.py\u001b[0m in \u001b[0;36m_validate_X_predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    419\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    420\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 421\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimators_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_X_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    422\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    423\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/tree/_classes.py\u001b[0m in \u001b[0;36m_validate_X_predict\u001b[0;34m(self, X, check_input)\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0;34m\"\"\"Validate X whenever one tries to predict, apply, predict_proba\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    387\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcheck_input\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 388\u001b[0;31m             \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDTYPE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"csr\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    389\u001b[0m             if issparse(X) and (X.indices.dtype != np.intc or\n\u001b[1;32m    390\u001b[0m                                 X.indptr.dtype != np.intc):\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m                           FutureWarning)\n\u001b[1;32m     72\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)\u001b[0m\n\u001b[1;32m    597\u001b[0m                     \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcasting\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"unsafe\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m                     \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mComplexWarning\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m                 raise ValueError(\"Complex data not supported\\n\"\n",
            "\u001b[0;31mValueError\u001b[0m: could not convert string to float: 'CN2C(=O)N(C)C(=O)C1=C2N=CN1C'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "drugs = pd.read_csv('drugs.csv')\n",
        "pred = model_RF.predict([drugs])\n",
        "print(\"pIC50 prediction\", pred)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "0kkaWU7Yrg4o",
        "outputId": "7c030c75-3dde-4a9e-cc07-5240ffd74d89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-87-27c37a586b65>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mdrugs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'drugs.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_RF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdrugs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pIC50 prediction\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/ensemble/_forest.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    627\u001b[0m             \u001b[0mThe\u001b[0m \u001b[0mpredicted\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    628\u001b[0m         \"\"\"\n\u001b[0;32m--> 629\u001b[0;31m         \u001b[0mproba\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    630\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    631\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_outputs_\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/ensemble/_forest.py\u001b[0m in \u001b[0;36mpredict_proba\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    671\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    672\u001b[0m         \u001b[0;31m# Check data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 673\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_X_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    674\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    675\u001b[0m         \u001b[0;31m# Assign chunk of trees to jobs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/ensemble/_forest.py\u001b[0m in \u001b[0;36m_validate_X_predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    419\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    420\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 421\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimators_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_X_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    422\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    423\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/tree/_classes.py\u001b[0m in \u001b[0;36m_validate_X_predict\u001b[0;34m(self, X, check_input)\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0;34m\"\"\"Validate X whenever one tries to predict, apply, predict_proba\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    387\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcheck_input\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 388\u001b[0;31m             \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDTYPE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"csr\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    389\u001b[0m             if issparse(X) and (X.indices.dtype != np.intc or\n\u001b[1;32m    390\u001b[0m                                 X.indptr.dtype != np.intc):\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m                           FutureWarning)\n\u001b[1;32m     72\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)\u001b[0m\n\u001b[1;32m    597\u001b[0m                     \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcasting\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"unsafe\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m                     \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mComplexWarning\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m                 raise ValueError(\"Complex data not supported\\n\"\n",
            "\u001b[0;31mValueError\u001b[0m: could not convert string to float: 'CN2C(=O)N(C)C(=O)C1=C2N=CN1C'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "generate model as Pickle Object"
      ],
      "metadata": {
        "id": "LaTzuIhrviJf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle"
      ],
      "metadata": {
        "id": "ffuVsyvmvWyt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pickle.dump(model_RF, open('IDH_model.pkl', 'wb'))"
      ],
      "metadata": {
        "id": "qV5fj0rovYW9"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}