Python/Sklearn-Workflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sklearn Modeling Workflow\n",
    "\n",
    "An example of using Sklearn to perform cross validation and create models for a binary outcome (survived titanic). Hyperparameters are trained with random search. \n",
    "\n",
    "Lift curves are generated for analyzing the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import pandas as pd\n",
    "import joblib\n",
    "import numpy as np\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import classification_report\n",
    "import time\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import svm\n",
    "from sklearn import tree\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV,StratifiedKFold,cross_val_score,RandomizedSearchCV\n",
    "from sklearn.metrics import accuracy_score, brier_score_loss\n",
    "from sklearn.preprocessing import MinMaxScaler,RobustScaler\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from scipy.stats import randint as sp_randint # use this random number generator for randomized grid search\n",
    "from scipy.stats import uniform as sp_uniform\n",
    "\n",
    "# Warning\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Return a dataframe with a count column for specified group by variable(s)\n",
    "def count_groupby(df,groupbyvars):\n",
    "    return(df.groupby(groupbyvars).size().reset_index().rename(columns={0: 'n'}))\n",
    "\n",
    "pd.set_option('display.max_columns', 500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "titanic = sns.load_dataset('titanic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(891, 15)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titanic.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "survived          int64\n",
       "pclass            int64\n",
       "sex              object\n",
       "age             float64\n",
       "sibsp             int64\n",
       "parch             int64\n",
       "fare            float64\n",
       "embarked         object\n",
       "class          category\n",
       "who              object\n",
       "adult_male         bool\n",
       "deck           category\n",
       "embark_town      object\n",
       "alive            object\n",
       "alone              bool\n",
       "dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titanic.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>class</th>\n",
       "      <th>who</th>\n",
       "      <th>adult_male</th>\n",
       "      <th>deck</th>\n",
       "      <th>embark_town</th>\n",
       "      <th>alive</th>\n",
       "      <th>alone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Cherbourg</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived  pclass     sex   age  sibsp  parch     fare embarked  class  \\\n",
       "0         0       3    male  22.0      1      0   7.2500        S  Third   \n",
       "1         1       1  female  38.0      1      0  71.2833        C  First   \n",
       "2         1       3  female  26.0      0      0   7.9250        S  Third   \n",
       "\n",
       "     who  adult_male deck  embark_town alive  alone  \n",
       "0    man        True  NaN  Southampton    no  False  \n",
       "1  woman       False    C    Cherbourg   yes  False  \n",
       "2  woman       False  NaN  Southampton   yes   True  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titanic.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test/Train split\n",
    "X_train, X_test, y_train, y_test = train_test_split(titanic[['adult_male','fare','alone']], \n",
    "                                                    titanic['survived'].values, \n",
    "                                                    test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run model pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Estimator: Random Forest\n",
      "Best parameters: {'clf__criterion': 'entropy', 'clf__max_depth': 8, 'clf__max_features': 'sqrt', 'clf__min_samples_split': 12}\n",
      "Kfold CV Mean Brier Score Loss: 0.143\n",
      "Brier Score on Training Data: 0.097\n",
      "Test data accuracy: 0.793 \n",
      "Test brier score: 0.150\n",
      "Precision, recall & f1-score:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.85      0.83       105\n",
      "           1       0.77      0.72      0.74        74\n",
      "\n",
      "    accuracy                           0.79       179\n",
      "   macro avg       0.79      0.78      0.78       179\n",
      "weighted avg       0.79      0.79      0.79       179\n",
      "\n",
      "runtime: 4.920530796051025\n",
      "\n",
      "Estimator: Neural Network\n",
      "Best parameters: {'clf__alpha': 0.020584494295802447, 'clf__hidden_layer_sizes': (3,)}\n",
      "Kfold CV Mean Brier Score Loss: 0.158\n",
      "Brier Score on Training Data: 0.157\n",
      "Test data accuracy: 0.788 \n",
      "Test brier score: 0.161\n",
      "Precision, recall & f1-score:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.82      0.82       105\n",
      "           1       0.74      0.74      0.74        74\n",
      "\n",
      "    accuracy                           0.79       179\n",
      "   macro avg       0.78      0.78      0.78       179\n",
      "weighted avg       0.79      0.79      0.79       179\n",
      "\n",
      "runtime: 7.835153102874756\n"
     ]
    }
   ],
   "source": [
    "MySeed = 42\n",
    "# Grid Searches for each model:\n",
    "jobs = -1\n",
    "score_fn = 'brier_score_loss'\n",
    "Mycv = StratifiedKFold(n_splits=5, shuffle=True, random_state=MySeed)\n",
    "\n",
    "pipe_nn = Pipeline([\n",
    "    ('scaler', MinMaxScaler()),\n",
    "                    ('clf', MLPClassifier(random_state=MySeed,solver='adam',max_iter=300))\n",
    "                   ])\n",
    "\n",
    "pipe_rf = Pipeline([\n",
    "    ('scaler', RobustScaler()),\n",
    "                    ('clf', RandomForestClassifier(random_state=MySeed,n_estimators=100))\n",
    "                   ])\n",
    "\n",
    "\n",
    "# specify parameters and distributions to sample from\n",
    "rf_param_dist = {\n",
    "              \"clf__max_depth\": sp_randint(1, 10),\n",
    "        #      \"clf__max_features\": sp_randint(1, 11),\n",
    "              \"clf__min_samples_split\": sp_randint(2, 20),\n",
    "              \"clf__criterion\": [\"gini\", \"entropy\"],\n",
    "              'clf__max_features' : ['sqrt','log2']\n",
    "                }\n",
    "\n",
    "nn_param_dist = {'clf__hidden_layer_sizes': [(1,),(3,),(5,)],\n",
    "       #     'clf__learning_rate': ['constant'],\n",
    "        #    'clf__solver' : ['adam'],\n",
    "        #    'clf__activation': ['relu'],\n",
    "            'clf__alpha' : sp_uniform(0.0,1)\n",
    "                }\n",
    "\n",
    "# Randomized Search objects\n",
    "gs_rf = RandomizedSearchCV(pipe_rf,\n",
    "            rf_param_dist,\n",
    "            scoring=score_fn,\n",
    "            random_state=MySeed,\n",
    "            cv=Mycv,\n",
    "            n_iter=30,\n",
    "            n_jobs=jobs)\n",
    "\n",
    "gs_nn = RandomizedSearchCV(pipe_nn,\n",
    "            nn_param_dist,\n",
    "            scoring=score_fn,\n",
    "            random_state=MySeed,\n",
    "            cv=Mycv,\n",
    "            n_iter=30,\n",
    "            n_jobs=jobs)\n",
    "\n",
    "\n",
    "# Dictionary of 'model name' : modelPipeline\n",
    "grids = {\n",
    "     #    'Logistic Regression' : gs_lr, \n",
    "    #  'Logistic Regression w PCA' : gs_lr_pca,\n",
    "         'Random Forest' : gs_rf,\n",
    "     #    'Random Forest w PCA' : gs_rf_pca,\n",
    "         'Neural Network': gs_nn\n",
    "        }\n",
    "\n",
    "best_models = dict()\n",
    "cv_results = dict()\n",
    "for model_name, gs in grids.items():\n",
    "    print('\\nEstimator: %s' % model_name)\n",
    "    start = time.time()\n",
    "    \n",
    "    # Fit grid search:\n",
    "    gs.fit(X_train, y_train)\n",
    "    \n",
    "    # Best parameters:\n",
    "    print('Best parameters: %s' % gs.best_params_)\n",
    "    \n",
    "    # Best training data accuracy:\n",
    "    # Average of all cross validation score (\"folds\") of the best estimator\n",
    "    # for a single combination of the parameters you specify in the tuned_params\n",
    "    print('Kfold CV Mean Brier Score Loss: %.3f' % abs(gs.best_score_))\n",
    "    \n",
    "    print('Brier Score on Training Data: %.3f' % brier_score_loss(y_train,gs.predict_proba(X_train)[:,1]))\n",
    "    best_models[model_name] = gs.best_estimator_\n",
    "    cv_results[model_name] = gs.cv_results_\n",
    "\n",
    "    # Predict on test data with best parameters:\n",
    "    y_pred = gs.predict(X_test)\n",
    "    y_prob = gs.predict_proba(X_test)[:,1] # extract probabilities for y=1 (match)\n",
    "    \n",
    "    # Test data accuracy of model with best parameters:\n",
    "    print('Test data accuracy: %.3f ' % accuracy_score(y_test, y_pred))\n",
    "    print('Test brier score: %.3f' % brier_score_loss(y_test, y_prob))\n",
    "    # Precision, Recall and F1:\n",
    "    print('Precision, recall & f1-score:\\n',classification_report(y_test, y_pred))\n",
    "    print('runtime: %s'%(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Save Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(best_models['Neural Network'], open('NN.pickle', 'wb'))\n",
    "pickle.dump(best_models['Random Forest'], open('RF.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "imp_features = pd.DataFrame()\n",
    "imp_features['Feature'] = X_train.columns.values.tolist()\n",
    "imp_features['Importance'] = best_models['Random Forest'].named_steps['clf'].feature_importances_.tolist()\n",
    "imp_features.sort_values('Importance',ascending=False,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature</th>\n",
       "      <th>Importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>fare</td>\n",
       "      <td>0.592487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>adult_male</td>\n",
       "      <td>0.379428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>alone</td>\n",
       "      <td>0.028086</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Feature  Importance\n",
       "1        fare    0.592487\n",
       "0  adult_male    0.379428\n",
       "2       alone    0.028086"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cv_results_summary(CVResults,model_name):\n",
    "    CompareCVResults = pd.DataFrame()\n",
    "    CompareCVResults['Score Rank'] = CVResults['rank_test_score'].tolist()\n",
    "    CompareCVResults['Model Name'] = model_name\n",
    " #   CompareCVResults['Hidden Layer'] = CVResults['param_clf__hidden_layer_sizes'].tolist()\n",
    "    CompareCVResults['params'] = CVResults['params']\n",
    "    CompareCVResults['Mean Test Score'] = [ abs(score) for score in CVResults['mean_test_score'].tolist() ]\n",
    " #   CompareCVResults['L2'] = CVResults['param_clf__alpha']\n",
    "    CompareCVResults['Mean Fit Time'] = CVResults['mean_fit_time']\n",
    "    CompareCVResults.sort_values('Mean Test Score',ascending=True,inplace=True)\n",
    "    return(CompareCVResults)\n",
    "\n",
    "all_cv_results = pd.concat([cv_results_summary(cv_results[model_name],model_name) for model_name in cv_results]).\\\n",
    "    sort_values('Mean Test Score')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score Rank</th>\n",
       "      <th>Model Name</th>\n",
       "      <th>params</th>\n",
       "      <th>Mean Test Score</th>\n",
       "      <th>Mean Fit Time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>27</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>{'clf__criterion': 'gini', 'clf__max_depth': 1, 'clf__max_features': 'sqrt', 'clf__min_samples_split': 6}</td>\n",
       "      <td>0.182798</td>\n",
       "      <td>0.122743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>{'clf__criterion': 'entropy', 'clf__max_depth': 8, 'clf__max_features': 'sqrt', 'clf__min_samples_split': 3}</td>\n",
       "      <td>0.144903</td>\n",
       "      <td>0.140536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>30</td>\n",
       "      <td>Neural Network</td>\n",
       "      <td>{'clf__alpha': 0.9832308858067882, 'clf__hidden_layer_sizes': (1,)}</td>\n",
       "      <td>0.213813</td>\n",
       "      <td>0.409822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>{'clf__criterion': 'gini', 'clf__max_depth': 1, 'clf__max_features': 'log2', 'clf__min_samples_split': 4}</td>\n",
       "      <td>0.182798</td>\n",
       "      <td>0.124231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>9</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>{'clf__criterion': 'entropy', 'clf__max_depth': 7, 'clf__max_features': 'log2', 'clf__min_samples_split': 10}</td>\n",
       "      <td>0.143928</td>\n",
       "      <td>0.147869</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Score Rank      Model Name  \\\n",
       "24          27   Random Forest   \n",
       "4           14   Random Forest   \n",
       "15          30  Neural Network   \n",
       "23          27   Random Forest   \n",
       "25           9   Random Forest   \n",
       "\n",
       "                                                                                                           params  \\\n",
       "24      {'clf__criterion': 'gini', 'clf__max_depth': 1, 'clf__max_features': 'sqrt', 'clf__min_samples_split': 6}   \n",
       "4    {'clf__criterion': 'entropy', 'clf__max_depth': 8, 'clf__max_features': 'sqrt', 'clf__min_samples_split': 3}   \n",
       "15                                            {'clf__alpha': 0.9832308858067882, 'clf__hidden_layer_sizes': (1,)}   \n",
       "23      {'clf__criterion': 'gini', 'clf__max_depth': 1, 'clf__max_features': 'log2', 'clf__min_samples_split': 4}   \n",
       "25  {'clf__criterion': 'entropy', 'clf__max_depth': 7, 'clf__max_features': 'log2', 'clf__min_samples_split': 10}   \n",
       "\n",
       "    Mean Test Score  Mean Fit Time  \n",
       "24         0.182798       0.122743  \n",
       "4          0.144903       0.140536  \n",
       "15         0.213813       0.409822  \n",
       "23         0.182798       0.124231  \n",
       "25         0.143928       0.147869  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', 200)\n",
    "all_cv_results.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate lift and associated metrics\n",
    "def calculate_lift(model,model_name):\n",
    "    lift_curve_pd = pd.DataFrame({\n",
    "    'prob' : model.predict_proba(X_test)[:,1],\n",
    "    'pred' : model.predict(X_test),\n",
    "    'truth' : y_test\n",
    "    }).sort_values('prob',ascending=False).reset_index(drop=True)\n",
    "    lift_curve_pd['model'] = model_name\n",
    "    lift_curve_pd['cum_truth'] = lift_curve_pd['truth'].cumsum()\n",
    "    lift_curve_pd['percentile'] = (lift_curve_pd.index+1)/len(lift_curve_pd)\n",
    "    lift_curve_pd['capture_percent'] = lift_curve_pd['cum_truth'] / lift_curve_pd['cum_truth'].max()\n",
    "    lift_curve_pd['lift']  = lift_curve_pd['cum_truth'] / \\\n",
    "       (lift_curve_pd['percentile']*lift_curve_pd['cum_truth'].max())\n",
    "    return(lift_curve_pd)\n",
    "\n",
    "all_lifts = pd.concat([calculate_lift(best_models[model_name],model_name) for model_name in grids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract Lift at Specific Junctions\n",
    "def summarize_lifts(df):\n",
    "    # Which percentiles we want to return\n",
    "    x_range = np.arange(0.1,0.51,0.1)\n",
    "    \n",
    "    # Interpolate to find the row that is closest to each percentile\n",
    "    q = df[[\"model\",\"percentile\"]].groupby('model').quantile(x_range, interpolation = 'nearest').reset_index()\n",
    "    df_summ = df.merge(q,on=['model','percentile'],how='inner').\\\n",
    "       pivot(index='level_1',columns='model',values='lift')\n",
    "    df_summ.columns.name =''\n",
    "    df_summ.reset_index(inplace=True)\n",
    "    df_summ.rename(columns={'level_1':'Percentile'},inplace=True)\n",
    "    return(df_summ)\n",
    "\n",
    "all_lifts_summ = summarize_lifts(all_lifts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEuCAYAAACgf0TJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hU1dbA4d9KJQ0SktASQiihd4L0IoiCoqBYUBHBQrkqYG/3u9d71WvvioiIKCKigKIoqIAU6YTeeyDUJEAIJXX298cZIIQJCWEmk7Le55knM2f2ObPOYcjKPruJMQallFLqSnm4OwCllFIlkyYQpZRShaIJRCmlVKFoAlFKKVUomkCUUkoViiYQpZRSheLl7gCuVFhYmImOjnZ3GEopVaLExcUlGWPCnXnMEpdAoqOjWbVqlbvDUEqpEkVE4p19TL2FpZRSqlA0gSillCoUTSBKKaUKRROIUkqpQtEEopRSqlA0gSillCqUEteNFyD5SAIZaaddcuxyXp6EBPhc3UF8y4NfsHMCUkqpYsplCUREqgNfA1UAGzDWGPNBHmVbA8uAu4wxU/M79r6vh9Li9N/ODNe5vMrBk1vBL8TdkSillMu4sgaSBTxpjFktIkFAnIj8aYzZnLOQiHgCbwC/F/TA3u2GsiLxBudGC6SmZTF74yEGtKlBs+qFrEEc3QJLP4bkXRAZ69wAlVKqGHFZAjHGHAIO2Z+nisgWIALYnKvoY8A0oHVBj9244y3OCvMiR1PTeHDdXJpWakyzFjUKd5Ajm60EcnyvJhClVKlWJI3oIhINtACW59oeAdwKjMln/yEiskpEViUmJroqTEL8rbaP46czCn+Q4Cjr5wmnzxqglFLFissTiIgEYtUwRhljTuZ6+33gWWNM9uWOYYwZa4yJNcbEhoc7dS6wi3h7ehBUzotjV5NAfAPBPxRO7HNeYEopVQy5tBeWiHhjJY9JxpjpDorEAt+JCEAYcKOIZBljfnJlXJcT4u/D8TNXkUAAgmvAca2BKKVKN1f2whLgC2CLMeZdR2WMMTVzlJ8AzHRn8gAICfC5uhoIQEgNOLTOOQEppVQx5coaSAfgPmCDiKy1b3sBiAIwxly23cNdKvp7k3gq/eoOEhwFW2aCzQYeOlZTKVU6ubIX1t+AXEH5Qa6K5UqEBPiw/cipqztIcA2wZULqIagQ4ZzAlFKqmNE/j3Op6Iw2kBB7F2DtiaWUKsU0geQSEuDDmYxs0jIv2zHs8oKjrZ/aE0spVYppAsmlon0erKuqhVSItH5qTyylVCmmCSSXc4MJr6onlnc5CKqqt7CUUqWaJpBcztdATmde3YGCa1i3sGzZYIwTIlNKqeJFE0guIf7eABxzRkP6gTh4Owa+utkJkSmlVPGiCSSXkAAnzIcFENUOPL3BJxASVlpjQpRSqhTRBJJLsJ+9BnK1CSR2MDy3DzqMhKw0SD3ohOiUUqr40ASSi5enBxX8vK9+LMg5obWtn8m7nHM8pZQqJjSBOFDRGfNhnT+YPYEc0wSilCpdNIE4EOLvxBpI+QhriVutgSilShlNIA5UDPC5+m6853h4QEhNOLbbOcdTSqliQhOIA05ZEySn0NpaA1FKlTqaQBw41wayZGcSH8zZcfUHDK0Nx/dYgwqVUqqU0ATiQLC/D+lZNu4bv4L35mznVHrW1R2wYm3IzoCU/Ze+l3oY5rwE43rA6eSr+xyllCpCLl3StqQKC7QGE1YM8CExNZ3E1HQCfa/iUuXsyhsSbT1P2gFLPoR131nJBWDfEmigo9aVUiWDJhAHejWpio+XB+XLeTN4wkoSU9OpGRZQ+AOe78q7G07EwF+vwbrJ4OULLQdC7APwaQc4slkTiFKqxNAE4kCgrxd9mkew5dBJABJTr3KJ26Aq4B0Ay0bD7y8AAu0egQ6jIDDcKlOxFhzZeHWfo5RSRUgTyGVUCvIF4Ghq2tUdSAQq1YeDa6D5PdDlOQiufnGZyo3gyKar+xyllCpCLksgIlId+BqoAtiAscaYD3KVuRd41v7yFDDcGLPOVTFdqRB/H7w85OprIAB3TABbllXTcKRyY9jyC2ScBp/L3C4zxroVFhINHp5XH5dSShWSK3thZQFPGmMaAG2BR0SkYa4ye4AuxpimwMvAWBfGc8U8PISwQF/nJJDgqLyTB0DlhoCBxK2O37fZYNNPMK47fNQS1ky8+piUUuoquCyBGGMOGWNW25+nAluAiFxllhhjjttfLgMiXRVPYYUH+XL0ChLI4ZQ0Zm88fOUfVLmR9TP3bSxbNmycBp+2hx/uh7PHoVwF2Pv3lX+GUko5UZGMAxGRaKAFsPwyxR4EZuWx/xARWSUiqxITE50f4GWEBxW8BhIXf5zeH/3NsG/i2Jd85so+KDjaamg/l0CMsWocn7aHqQ8ABvp9AY+ugppdrDVGlFLKjVyeQEQkEJgGjDLGnMyjzLVYCeRZR+8bY8YaY2KNMbHh4eGuC9aBSgWsgXy/aj93j11GRpY12nxP8ukr+yAPD+s21pFNkBAH43taNQ6A27+E4Uuhye1Wu0dkazi+F04VbTJVSqmcXJpARMQbK3lMMsZMz6NMU2Ac0McYU+yGYocH+XLsdDrZNsfrmmfbDC/P3MwzU9fTumYI04a3ByD+ShMIWLex4pfAuG5WQ/ktH8HwJdD4NivBnBPZ2vp5YNWVf4ZSSjmJyxKIiAjwBbDFGPNuHmWigOnAfcaY7a6K5WqEB/liM5B8+tJaSGa2jVFT1vLF33sY1D6arwZfQ51Kgfj7eLI36QpvYYF1a8rTBzo9BSNWW4MMHfW0qtoMPLz0NpZSyq1cOQ6kA3AfsEFE1tq3vQBEARhjxgD/AkKB0Va+IcsYE+vCmK7Y+bEgJ9OpFFTu/PaMLBv/mLSaOVuO8GzP+gzvWvv8e1EV/QtXA2l0q1XbyI+Pv9XtVxOIUsqNXJZAjDF/A5JPmYeAh1wVgzOE2xNI4qkLNZCsbBujpqxhzpYj/LdPIwa2i75on+jQAHYcTb3yD5PLXq6LRba2pkPZ8ovVI6vHf62pUZRSqojobLz5OFfrONcTyxjD89M38NuGw/zzpgaXJA+AGmH+7D92Ns92E6eIbA0Zp2DKAFg+BvYuct1nKaWUA5pA8hEWaK+B2BPIx/N28kNcAiO6x/BQJ8cDA6NDA8jItnEo5azrAqtzHdTpAb3eBPGE+KWu+yyllHJAE0g+/Hw8CfL1IjE1nZnrD/LOn9u5rUUEj18Xk+c+NUL9AYi/0rEgVyIgFAZMhTZDoWpT2Lfs8uXTU2HRu/BRbP5llVKqAHQyxQIIL+/Lyr3HmLJyP7E1QnitXxPkMu0V0aHWXFZ7k0/ToU6Y6wOMag+rvoCs9EvbQbLSYeUXsOhtOJMM4mGtQRLV1vVxKaVKNa2BFEB4oC+bDp7E38eTT+5tia/X5ScxrFK+HD5eHq6tgeQU1Ray0uBQjnkobdlWovgoFn5/Hqo0gYfmQt1esGuuNdJdKaWugiaQAqhcvhwi8EH/FlQuXy7f8h4eQo2K/uxNKkRX3sKIamf9jF9i/dy3DMZ2hR+Hgn8I3PcjDJwBkbFQpxuc2GetjpibzQbZmUUTs1KqxNNbWAXwWLc69GlejY4xBb8dVSM0oOhqIIHhEBpDxpZZrFu5mNYpf0D5CGvurEa5RrHX7mb93DUPwupYz7MzYe231m0uv4owZP6VdSlWSpVJWgMpgJjKQXRvUPmK9okO9Sf+2GlsruzKa2ezGXb6NcbnwDKanpjHR1l9OXr/IvvcWbn+iSvWgpCa1m0smw02TIWPW8MvIyAzDQ6thaRiOSmAUqqY0QTiIjXDA0jLtHHo5FWuZpiPnUdTuX3MEkbuas0c/xv5+/qZvJN1J0v2XeZz63SHPQthbBeY9iB4+8PdU6yaB8DWX10as1KqdNAE4iK1wgIB2J14yiXHt9kMExbv4aYP/2ZP0mkevKMP3Z/+lq7t2lLBz5vFO5Py3jnmesg8A2kn4NaxMGwR1OsJFSKgWktNIEqpAtE2EBepHW515d119BSdYpw7Bf3hlDSenrqORTuS6Fa/Eq/3a3J+xLynQLtaoSzZlYwxxnF345jr4YHfoVqLS7v91r8J5r0MJw9B+apOjVspVbpoDcRFwoN8CfT1YreTe2L9tfUoN7y/kFV7j/PqrY354v7YiyZ5BOhQJ5QDJ87m3YgvYnX9dTR3Vv3e1s9tvzk1bqVU6aMJxEVEhNrhAexOdE4CsdkM7/25nQe+Wkm1YD9+G9mJe9vUcFjDaFfb6i22ZFchllcJr2c1tOttLKVUPjSBuFCt8ECntIGcOJPBA1+t5IO5O7itRSTTh7enZlhAnuVrhwdQubwvi3ddph0kLyLWbaw9CyHN4QKSSikFaAJxqVphARxMSeNMRtb5bfuSz5CaVvDBenuSTtPnk8Us2ZnMq7c25u07muLnc/mR8CJCh9phLN2VXLhuxPV7gy0Tdv555fsqpcoMTSAuVCv8XE8s6zZWXPwxrntvAW//vq1A+6/ed5zbRi8mNS2LyUPa5nnLypH2dcI4djqDrYcLsS5JZGsICL/4NpYxcHSrToGilDpPE4gL1a5k3WbanXSavUmnefjrODKybGw4kJLvvr9vOszdY5dRwc+b6cPb06pGyBV9doc6oQAsKcxtLA9PqNcLdvwJWRlwaD1MuAlGt7Hm11JKKTSBuFR0aAAisDr+OIMnrMQYQ7f6ldh2OPWyt5a+Xb6PYd/E0aBqeaYNb0/0Zdo78lK1gh+1wgIuaUjflXgKU5BaRL2bIP0kfHsHfNYZjm6xaiXrJl9xLEqp0sllCUREqovIXyKyRUQ2ichIB2VERD4UkZ0isl5EWroqHnco5+1JRLAfE5bs5cCJs3w+MJbrGlTmdEY2B044Xmzqm2XxvPDjBrrWDWfyw20JDSz8MrXtaoeyfHcymdk2UtMyeXzKWrq/s4ApK/fnv3OtLuATaC2X23Y4jFgNrQZbjeuphwsdk1Kq9HBlDSQLeNIY0wBoCzwiIg1zlekFxNgfQ4BPXRiPW9S2t4O8c0czYqMrUq9KEIDDtolJy+P5508b6V6/EmPua5VvY3l+OtQJ43RGNl8vjefGDxcxY+0BAn29mL2pAAnA2w8G/QqPrICer4FfiDW3FgY2/XhVcSmlSgeXJRBjzCFjzGr781RgCxCRq1gf4GtjWQYEi0ipGv78RI+6jBnQkpubVQM4n0C2Hb64i+yk5fG8+ONGutWvxOgB+a85UhDtaoUiAi/P3IzNBt8PbcedsdVZsiv5op5hearWHEJrX3gdXs9aV2TD1KuOTSlV8hVJG4iIRAMtgOW53ooAct5PSeDSJFOiNaseTM/GF3JioK8X1Sv6sSVHDeTHNQnnk8enTkoeACEBPtzeMpI7WkUya1QnYqMr0r1BJTKybCzeWYhBhgCNb4cDq+DYbqfEqJQquVyeQEQkEJgGjDLG5B6Z5qhP6iUtvCIyRERWiciqxMREV4RZpOpVLs82ewJZsiuJZ6aup12tUKcmj3PeuqMZb93RjPLlvAFoHV2RIF8v5m45ckXHiU8+zSszN5MUbZ/qZOM0p8aplCp5XJpARMQbK3lMMsZMd1AkAaie43UkcDB3IWPMWGNMrDEmNjzcuRMTukODqkHsSTrNxgMpDJ0YR3RoAGPua+X05OGIj5cHneuGM2/r0QINMsy2GcYt2s0N7y9k3N97GLchy1oBccNUHROiVBnnyl5YAnwBbDHGvJtHsZ+BgfbeWG2BFGPMIVfFVFzUqxJEts1wz+fLKOftyZeDW1PBz7vIPr9b/UocTU1n48HLj0fZdjiV2z5dwiu/bqFD7TCuia7Iz2sPYGvUDxK3wpFNVsFTiTDjUfj9xSKIXilVXLhyOvcOwH3ABhFZa9/2AhAFYIwZA/wG3AjsBM4Ag10YT7FR396Qnplt+OahWCJD/Iv086+tXwkRmLvlKE0jgy9532YzjF+8hzdnbyOwnBcf3t2Cm5tW5ed1Bxn53VpWB3UhVjxhw/eQsBLm/BvSUgCxuvxWiCzS81FKuYfLEogx5m8ct3HkLGOAR1wVQ3FVMyyQvs2rcWvLSIe/wF2tYoAPLaNCmLf1KI/3qHvRe4dSzvLk9+tYsiuZ6xtW5rXbmpwfi9KjYWX8fTyZtjWN2NrXwuIPrJ2iO0GHkTDpdlg7Gbo8XdSnpJRyAx2J7gaeHsL7/VvQpa772nO6N6jEhgMpHMmx5O6sDYe44b2FrN1/gjf6NeGz+1pdNJDR38eLGxpV4df1h8iMfRgq1oa+Y+D+XyCmh5VI1k6y2kZs2dpGolQppwmkjOpevzIA87YeJSPLxn9+2cTwSaupGR7IryM6cVfrKIcTN/ZpXo2TaVnMzWpujU5vfrc1BTxAiwFwfA/MegbeqAl/vVqUp6SUKmKaQMqoupUDiQj2Y/rqBPqPXcqXi/cyuEM0Pwxtd9m1RjrWCSMs0IcZaw9c+maDW8AnCFaMBWODFZ9D5sVTtqRnZfPxvB0MmxhHela2s09LKVWEdE30MkpEuK5BJb5aGk+Ajycf39OC3k2r5bufl6cHvZtW49vl+0g5m3lx7zEff+g3DrLSwL8ifHUzbPrJqqUA87cd5aWfN7HXvtTubxsOcWsLbXBXqqTSGkgZdl+7aHo3rcqMRzsWKHmc07dFBBnZNmZvdNDjul5PaNTXag8JjYFV4zmcksawiXEM+nIlHiJ8/cA11AoPYMKSeCeejVKqqGkCKcPqVArk43taUqdS4BXt1yyyAjXDAvhxjYPbWOeIYFoNgoQVPPLuV8zffpRnetZj1qhOdK4bzqD20azbf4K4+ONMX53An5uvbGS8Usr9NIGoKyYi9GlejeV7juU5Lf3+Y2cYtrEe6caboQELmT2yM//oWuf8aPvbWkYS5OvFveOW8cT36xj13RpSzhZ8qV+llPtpAlGF0q9lJMbA9LiEi7bbbIYJi/dw/XsLWXzAxoHInvTIWkB0UI4uvempBKbsZHDHmoT4+/Bkj7qczsjm2+X7ivgslFJXQxOIKpTqFf1pW6siU1cnnF/hMOH4Gfp/voyXftlMm1oV+ePxztTqOQLJSIWN9ingt/8Bn7SBT9vxREtPlj7fnce6x9CxThhfLt5DRpbNjWellLoSmkBUod3RqjrxyWdYsecYP687SK8PFrH54Eneur0pXw5qTbVgP4hsDZUbW116pw+xlsj1DQLxhOWfnT/Ww51rUf3Uev5cssKNZ6SUuhKaQFSh9WpShUBfLx6dvIYRk9dQp1Igv43oxB2x1S8MQhSB2MFwZKM1BXyXZ2HoQmt1wzXfwNkTkHqEzuueYZrvf6g7fzjGprUQpUoCHQeiCs3fx4s+zasxecU+RnSPYUS3Onh5OvibpNk9cPIQNLoVqjS2trUZBusmw0//gPi/kcyzHKrUmZijC1m76Gead+lbtCejlLpiYkrYfEWxsbFm1apV7g5D2aVlZpOYmk71ioWYUfjLGyF+MUS1h5s/ICMokpOvN+Cgby2aPv4TrP3WWgExsOSvAaOUu4lInDEm1pnH1BqIuirlvD0LlzwA+nxi3dqqdxN4eOAD7Kp1H212f0TWBy3xOpsEhzdA39FOjVkp5RzaBqLcp2JNaHAzeFz4Gja4ZRTJpjyHbMFQvzes+w6O7wUg6VQ6j09ZS8/3F3I6PctNQSulztEEooqV8sFhTGg9gy4nXyK+zUvg4YlZ9B4/rNrPde8uYOb6g2w9nMrXS3UaFKXcTROIKnYGdmmEt6cXo+POcrJBf7JXf8O7U/8iplIgs0Z2omu9cD5buIvUtEzi4o+x+eBJh8ex2Qy7E08VcfRKlR2aQFSxEx7ky12tqzNtdQK3rG2NwTCh3hKmDGlHnUpBjLquLifOZHLr6CX0+3Qp93+5gjMZF9/S2pCQwm2fLqHbOwv4db2DSR+VUldNE4gqloZ0roW/jycNGzQiq9Gd1EuYjsdpa8LF5tWD6dGwMvuSz3BXbHUSU9OZsGQvACfOZPDPnzZwyyd/k3D8LFEV/Xlt1hbSMnXtEaWczWW9sERkPNAbOGqMaezg/QrAN0CUPY63jTFfuioeVbJEhviz9l/X4+EhkBwMm6fAko/gBmuVww/7t+BMRhahgb4knkpnzPxdBPp68f6cHZw4k8Gg9tE83qMu6/enMOCL5UxYspdhXWq7+ayUKl1cWQOZAPS8zPuPAJuNMc2ArsA7IuLjwnhUCePhYR/NHlobmtwBq8bDqUQA/Hw8z6/X/uT1dTmZlsW/ZmyidngAv47oxL9vbkT5ct50jAmje/1KfDJvJ0mn0t11KkqVSi5LIMaYhcCxyxUBgsSa8yLQXlb7ZirHOj9trXT493uXvNWoWgX+d2sTPujfnO+HtqNB1fIXvf/8jQ04k5nN+3O2F1W0SpUJ7mwD+RhoABwENgAjjTE6CZJyLCwGmt0NK8fByYOXvH1Pmyj6NI+w5uA6eQj++h+kHgashbMGtIni2+X72H4ktagjV6rUcmcCuQFYC1QDmgMfi0h5RwVFZIiIrBKRVYmJiUUZoypOujwLxgYL33L8vs1m3eb65BpY8AbMeub8WyOvq0uArxev/rrlij/2UMpZnvphHfePX0F6ljbGK3WOOxPIYGC6sewE9gD1HRU0xow1xsQaY2LDw3VepDIrpAa0HAirv4Zjey5+73g8TOwDMx+Has0h9gHYPAP2LgagYoAPI7rFsGB7IvO2Fmz53NPpWbz7xzaufXs+P689yILtiXw8b6ezz0qpEsudCWQf0B1ARCoD9YDdboxHlQSdnwYPL1jwpvXaGKvW8Wl7OLAaer8PA3+G61+F8hHw+wtWzQS4v300tcIDeHmm1a13xtoDLNudfMlHGGOYvjqBa9+ez4fzdtKjYRXmPtmF21pGMHr+LjYeSCnKM1aq2HLZbLwiMhmrd1UYcAT4N+ANYIwZIyLVsHpqVQUEeN0Y801+x9XZeBW/vwjLRsO9P1hde3fPh5pdoM/HEBx1ody6KfDjEOj7KTS/B4D5244y6MuVVC7vy5GT6ZQv58W8p7oSZu/RtfngSf41YyOr4o/TrHow/+rdkFY1QgBrjEmP9xYSHujLjEc74O1o6nqliilXzMar07mrkud0EnzQDDJOgXcAXP+ydcvq3CJW59hs8MV1kHIAHosD30AAhn8Tx4o9x3ioUy3e/XMbfZtH8M/eDXn3j21MXBZPsL8Pz/Wsz+2tIi90Jbb7fdNhhk6M44kedRnRPaaozlipq6bTuSsFEBBmJY1d8+D6VyAk2nE5Dw+44TUYfz0s/gC6vQjAx/e0xBiDl6cHKWczGbNgF3O2HCHlbCYD2tbgyR71qODv7fCQNzSqws3NqvHRvB3c0KgK9aoEuegklSr+tAaiSr8fBsO23+DRlRff4gLOZGTR+6O/CQ3w4aVbGtGoWoV8D3fsdAY93l1ARIgf04e3d7wKo1LFjCtqIAX65ovIGwXZplSx1OO/gMAf/7ReGwM2qzuuv48Xc5/owg/D2hcoeYDVo+u/fRqzPiGFzxftIS0zm2lxCRw/neGiE1CqeCron049HGzr5cxAlHKZ4OrQ6QmrW++yT2FMJxjbBbKsX/iSu+2kAG5sUoWejarw3pztdH9nAU/+sI7HJq/BZitZNXqlrsZlE4iIDBeRDUB9EVmf47EHWF80ISrlBO1HWG0ls5+DkwespXKXfmS9l3IAMs9e0eFEhJf7NqZ8OW/K+3nzUMea/L0zifGL9+S/s1KlRH6N6MuAWcBrwHM5tqcaYy43z5VSxYt3Oej3BWydCR1Gws+PwYK3rMkZV4yF2t2sbsHnaiP7lsGGqVbDu1+Iw0OGB/ny97PX4uPpgQjsO3aGN2dvo33tMBpWczipglKlSn63sMYZY/YClYwx8TkemjxUyRMZC9e9ZCWEnq+DeMDyT6FqM9j5J2ycBump8OtTML4nrPwcZj132UOW8/bEw0MQEV7v15Rgf29GfrdG1x9RZcJle2GJyBrgJ+Ah4JJpUI0x77ouNMe0F5Zymr2Lrbm1arSHcd3hxH7wKmfd4mozFDx9YMmH0P9bqH9TgQ65cHsiA8ev4P52NfhPn0uWwVHKbdzRC6s/kIZ1qyvIwUOpkiu6A9TsBB6ecPMHkHYCfIPgwT+h1xvQ7f+gchP4ZRScKVilu3PdcB7sWJOvlsbz19ajLj4BpdyrQONARKSXMWZWEcSTL62BKJdJOQAB4eCVY12zwxtgbFdo2Bdu/6JAh0nLzKbvJ4tJOpXO7FGdz0+TopQ7FXkNREQG2J82FJEncj+cGYhSblch4uLkAVCliTWN/MapVjfgAijn7cmHd7fgZFoWz0xdT0kbrKtUQeV3CyvA/jOQS29fBbowLqWKj46PWw3tM5+w5uEqgLqVg3jxxgbM23qUcYu0a68qnS7bjdcY85n9539yvycio1wVlFLFiqc39B0Dn3WG356COyYUaLeB7WqwbHcyr8/eSouoYGKjK7o2TqWK2NVM4qO3sFTZUbkhXPs8bPoRNk4v0C4iwhu3NyUyxI9Hv11D8ql0FwepVNG6mgRy5fM/KFWStR8J1VrCr084XJfdkfLlvPnknpYcO5PB49+v06lOVKlyNQlE/yeossXTC2773JpDa/qQ8xMy5qdxRAVeurkRC7cnMnq+LomrSo/8emGlishJB49UoFoRxahU8RFWB258E/YustYYKaC7r6lO3+bVePfP7SzZVbCGeKWKu8smEGNMkDGmvINHkDFGF6NSZVPze6HRrfDXq5AQV6BdRIRXb21CzbAARkxey9HUNBcHqZTr6Uo4Sl0pEej9PgRVhWkPWvNnFUCArxefDmjFqfRMHp20howsm4sDVcq1XJZARGS8iBwVkY2XKdNVRNaKyCYRWeCqWJRyOr9gqz3kRDz89nSBd6tbOYg3+jVlxd5jvDxzswsDVMr1XFkDmQD0zOtNEQkGRgO3GGMaAXe4MBalnK9GO+j8DKybDOt/KPBufZpHMLRzLSYui+e7FftcGKBSruWyBGKMWQhcbga6e4Dpxph99vI685wqeTo/DdXbWl17jxV8xPkzPevTKSaM/5uxkbh4XR1BlUzubAOpC4SIyHwRiRORgW6MRanC8fSCfl9TsJ0AACAASURBVJ9b7SJTB0NWwQYLenoIH9/dkmrBfgz7ZjWHU7RRXZU87kwgXkAr4CbgBuD/RKSuo4IiMkREVonIqsTExKKMUan8BUdB30/h4Br4/YVL38/KgLSUSzZX8Pfm84GxnEnPYug3cboIlSpx3JlAEoDZxpjTxpgkYCHQzFFBY8xYY0ysMSY2PDy8SINUqkDq3wTtH4OV46ylcM/ZvwJGt4UPW8DxvZfsVrdyEO/e1Zx1+0/wz5826sy9qkRxZwKZAXQSES8R8QfaAFvcGI9SV6f7vyGqHfw8Ag5vhDkvwfgbIDsTbFnw3b2QcfqS3W5oVIWR3WOYGpfA54t2F33cShWSK7vxTgaWAvVEJEFEHhSRYSIyDMAYswWYDawHVmCtv55nl1+lij1Pb7h9PHj7wWed4O/3oMUAGL4Ybv8Sjm6Gn4aDg1rGyO4x3NSkKq/N2srsjYfdELxSV65AKxIWJ7oioSr29iyEOf+xFqKqe/2F7Us+gj/+Cd3+afXeyiUtM5u7P1/GlkMnmTKkHc2qBxdh0Kq0c8ea6EqpK1WzMzw89+LkAdDuUWh6F8x7Bbb+dslu5bw9+XxgLOFBvjz41SoSjp8pooCVKhxNIEoVFRG4+QOo1sKazffo1kuKhAX68uWg1qRnZfPAhJWcTMt0Q6BKFYwmEKWKkrcf3DXJ+vnd3XD2+CVF6lQK4rMBrdideJpHJq0mPvk0Qyeu4sUfN+h6IqpY0TYQpdxh3zKY0BtqtIcB06wG+Fy+X7mfZ6atRwS8PTzIyLbxQIeaNI2swBuzt3L3NVGM6B7jhuBVSeSKNhCdkl0pd4hqC7d8aPXKmvk43PKRdYsrhztbV+f4mQzWH0jh+V71GbdoD+MXW9OlhAX68u6f2/H38eShTrXccQZKaQJRym2a3wPHdsPCtyC0NnR8HGw2OJMMgdaA2aFdap8v/q/eDQn09aJyeV/uah3F41PW8sqvWyhfzps7W1d311moMkwTiFLudO2L1iSMc14Cb3/Y8gvE28eNNOp7UVEPD+GpG+qdf/3eXc1JTc/iuenrCSznxY1NqhZx8Kqs00Z0pdxJBPp8Ys3oO+sZaz6tsHow/WHYs+iyu/p4eTBmQEtaRoUw8rs1LNiu88SpoqUJRCl38y4H/b+FTk/BsL9h8G9QsRZ8dw8c3nDZXf19vPhiUGtiKgUxbGKcTg2vipQmEKWKg4BQ6P5/ULEm+Fe0emb5BsE3/RxOwphTBT9vvn7wGqpWKMegL1eyPuFE0cSsyjxNIEoVRxUirSSSlQ4Tb4PTSdb2E/scrsEeFujLxIfaEOzvzYBxy9l44NLp45VyNk0gShVXlRrAPd/DyQMw6Xb46zVrWvgvroczl96qigj2Y/LDbQkq5829V5pEMs9C0k4nBq/KAk0gShVnUW2sHlmH1sGC16F2d0jeCd/cBmknLykeGeLPd0PaEujrxYAvlrP54KVlLrHjT/ikDXwcCys+d8FJqNJKE4hSxV39G+GeH+DuKXDv93Dn11bj+rd3QcYZq+Yw75XztZLqFf2Z/HBb/Lw9uXecNbuvQ6eTYOqDVu3GyxdqdYXfnoIFbzmccl6p3HQqE6VKoo3TYNpDULkxJO2ArLNQtRkMnAF+IQDEJ5/mrs+WkZFtY/LDbalXJcja1xjY8APMetZqT+n8NHQcBeIBMx6F9d9B20fg+lfAQ//GLC10OnellKVxP2v6k8MbILqjtSb70S3wdV84a/XCqhHix+QHW+HtKfQfu9RqEzmxHybdYY0zCa1jdRvu+qxVA/H0to5zzVBY9gn8/ChkZ7n5RFVxpiPRlSqpWgyAuj3BP9QakOgfClMGwMS+1vK6s5+jZlY6U/tP4e4p+5nx+Us08JqCpwA934BrHgYPz4uP6eEBvd6wuhLPfw3SUqDfF9ZYFaVy0VtYSpUm22ZbScSWCUHVIPM0+JYnPaAqvgdX8Ldpiv9tH9OyWbP8j7XsU5j9HER3gv6TrFtcayZB/ZsgWOfeKmlccQtLE4hSpc3OubBnAXR8whqEOLEvGMPJri/Tb3EU+46f5bP7WtG1XqX8j7VuCsz4B4TGQNoJSD0EgZXh7u8goqXLT0U5T4lqAxGR8SJyVEQ25lOutYhki8jtropFqTKlTnfo8V/wC4ZqzeGRlTByLeXbDmTKsPbUqRTIw1+vYvbGw/kfq9ld1liUlP0QEGZ1KfbyhQk3wbZZrj8XVay5shF9AtDzcgVExBN4A/jdhXEoVbYFhp/vmVUxwIdvH25Lk4gKPPLtaqbFJeS/f53u8ORWGLIQGt8GD82F8PrWXF3Lx7o4eFWcuSyBGGMWAvnN7PYYMA046qo4lFIXq+DnzcQH29C2VkWe/GEdny/cnf9OvkEXuvQGVoJBv0LdXjDraZj9PNiyXRu0Kpbc1o1XRCKAW4Ex7opBqbIqwNeL8YNac1OTqrz62xZen7WVjCwbn/y1k6d/WMep9Hy67/r4w10Toc1wWDYavh8I6aeKJnhVbLizG+/7wLPGmGzJtZRnbiIyBBgCEBUVVQShKVX6+Xp58uHdLQj292bMgl1MjdtP0qkMRGDTwZN8Obg1lctfpvuuhyf0eh1CouH352F8T7j7WwjW/6NlhUt7YYlINDDTGNPYwXt7gHOZIww4Awwxxvx0uWNqLyylnMsYw4dzdzJ9TQIv3NgAXy8PHpm0mgp+3kx44BrqVg7K/yA758APD1iDEftPstZ8V8VKievGe7kEkqvcBHu5qfkdUxOIUq638UAKD0xYydnMbD4b0Ir2dcLy3ylphzU/14l90Ps9aHmf6wNVBVbSuvFOBpYC9UQkQUQeFJFhIjLMVZ+plHKOxhEV+PGRDlStUI77v1zBj2sK0FsrLAYengvRHaxpUGa/oFOhlHI6kFAplaeUs5kMmxjH0t3JPH5dXUZ0r0N+bZZkZ8EfL8LyMVDrWmsqlIDQoglY5alE1UCUUiVfBT9vvnrgGm5rGcF7c7bz2OQ1pGXm02XX08uaT+uWjyB+CYztAgdWF03AqkhpAlFKXZaPlwfv3NGM53rV59cNh7jzs6UcOZmW/44tB8IDs63n42+AuK9cG6gqcppAlFL5EhGGdanN2Pti2XX0FLd8/Dfr9lvTxu8/dobZGw/h8HZ4REsYssCacv6XETDjEWv5XFUqaBuIUuqKbD18koe+WkViajr9W1fn+1UJnM3M5sYmVXjz9mYE+joYXmbLtqaHX/iWtfDVnRMhpAakHrEmfIxqU+TnUdaUuG68rqAJRCn3Sz6VzrBv4li59zjX1gunefUQPpi7nVrhgYwZ0Io6lQId77htFkwfak2L0moQrPwC0k9Cq8HQ8zXw9ivS8yhLNIGgCUSp4iIjy8aWQydpGlkBEWHJziQem7yG9Cwbb9/RlJ6NqzreMXmXNfXJkY3WOuyVGlrToYQ3gDu+hEoNivI0ygxNIGgCUao4O3jiLMMnrWbd/hMM7VKLp6+vh5eng6bWzLNwaD1Uv8ZaTXHnHPhxmDWfVq/XoeX91nblNNqNVylVrFUL9uP7oW25p00Uny3YzcDxK0g+lX5pQW8/q93jXJKocx0MW2xt+2UkTB1sLaerijVNIEopp/L18uR/tzbhzdubsir+ODd/9Der9x3Pf8egyjDgR2s9980/w5iOkKB3G4ozTSBKKZe4M7Y604e3x8NDuHPMUj5bsAubLZ9b5h4e0OkJa/yIwRo/svBtXW+kmNIEopRymcYRFfh1RCd6NKzMa7O2MnjCSpIc3dLKrfo1MGwRNLgZ5r0MX94Ix/a4PmB1RTSBKKVcqoKfN6PvbcnLfRuzdHcyvT5YxJKdSfnv6BdsrcF+2+dwdIt1S2v1RDjX8cdmg4wzrg1eXZYmEKWUy4kI97WtwYxHOlC+nBf3frGcd/7YRla2Lb8doemdMHwxVGthzfI7ZQDsmAOfdYa368Lqry8kFVWktBuvUqpIncnI4t8zNvFDXAKto0P4oH8LqgUXYAChzWaNF5n7H8jOgKBq1uqH+5dZvbhu/hBsmTD/dWu0+zVDrFUTFaDjQABNIEqVFj+tOcCLP27Ay9OD125rwo1N8hh4mNuRzbBrrjWS3TsAVo6DOf8GDy8rsdiyrEdEK2tG4MqNXHoeJYUmEDSBKFWa7Ek6zagpa1m3/wT9Wkby0i0NAfh0/i7qVQnilmbVOJqazqRl8fRpEUHt8DymSDm2G359CspVgOtfgX1LYdYz1liSjo9D56fBy7cIz6z40QSCJhClSpvMbBsfzdvJx/N2UC3YD5vNcDDFmi6+Y50w1iWcIDUtC18vD56+oR6DO9TE00M4mZZJ8qkMaoYFOD7w6WT4/QVY/x2E1bVqI2V4rXZNIGgCUaq0ios/zhPfr8XH04PX+zVh1d7jvPPHdlrWCOap6+sxZsFu5mw5QmyNEG5tGcF7f+4g+XQ6g9vX5Kkb6uLv42AWYLAa3Gc+Din7oPVD1kDFcuWL9uSKAU0gaAJRqjTLthkE8PCwpjg5lZ5FgI8nIoIxhh/XHOClnzdxMi2LJhEVaFStPN+t3E9kiB+v39aUjjFhjg+cfgrmvWItsxtU1Zpvq8EtZWq+rRKVQERkPNAbOGqMaezg/XuBZ+0vTwHDjTHr8juuJhClyrYjJ9NYtfc4NzSqjJenB8t3J/Pc9A3sSTrNnbGRvHhjQ3YmnuKDuTvoVi+c+9pF42lPSOxfadVGjmywem71ehNCa7v3hIpISUsgnbESw9d5JJD2wBZjzHER6QW8ZIzJd1UZTSBKqdzSMrP5YO4Oxi7cTVA5L1LOZuLn7cmZjGyaRFTg1Vsb0zQy2CqcnQUrxsJf/7N6bXV6AjqMAu9y7j0JFytRCQRARKKBmY4SSK5yIcBGY0xEfsfUBKKUysvGAym88utmGlatwBPX12X+tqP895fNJJ5KZ2DbGjx5Qz3Kl/O2Cp88BH+8CBunQUhNuPFtiLnOvSfgQqU5gTwF1DfGPJTH+0OAIQBRUVGt4uPjnRypUqq0OpmWybt/bOerpXsJC/Tl/3o35OamVZFz7R+7/oLfnoLkndCwD9zwP6gQ6daYXaFUJhARuRYYDXQ0xiTnd0ytgSilCmN9wgle/HEjGw6k0CkmjP/2aXyhC3BWOiz5yFqzHYGOo6D9CPDxd2vMzlTqFpQSkabAOKBPQZKHUkoVVtPIYH56pAP/uaURa/ad4Ib3FvLm7K2cTs+yBhl2fgoeXQn1esL81+Dj1tbtrRLWU7UouS2BiEgUMB24zxiz3V1xKKXKDk8P4f720cx9sgu9m1Zl9PxddHtnPj+tOYAxxppb644JMOg38A+BqQ/Al73g4Bp3h14subIX1mSgKxAGHAH+DXgDGGPGiMg4oB9wrkEjqyDVK72FpZRylrj447z08yY2HEghtkYIL93SiMYRFaw3bdmwZiLMfRnOJEOLAdD9XxBYyb1BF1KJawNxBU0gSilnstkMP8Tt583Z2zh2JoP+raN46vq6hAba585KS4EFb1qDEL3KWV1+2z1S4tpHNIGgCUQp5RopZzP5cO4OvlqyF38fT0Z0j+HO1tUZu2A387cfZUjDbLrsG02F+N9JlIrsbTKSlrc8iqdXHlOoFDOaQNAEopRyrR1HUvnvzM0s2pGEl4eQZTPUDAtgT9JpALr77+Jpj2+on7WNvR5RpHT8J0273oF4FO/1+TSBoAlEKeV6xhjmb09k5rpD9GsVQbtaofy+6Qi7Ek9xb5soKpTzYvXvX1F5xRtEmkNs9GmGV8+Xqd+yi7tDz5MmEDSBKKWKj8yMNNZMf4+YraMJ4SRxgV2pdMt/qF63ubtDu0SpGweilFIlmbdPOa7p/zzeT6xjWeQDNEhdSrVJXYl7/y4Ox291d3gupwlEKaWuUmD5irR96D3O/GMNyyvdRePjcwkd354VH93P0QN73B2ey2gCUUopJwmrHEH7Rz7j2MMriAu7meZJv1B+bGuWjh5G0pEEd4fndJpAlFLKyapG1qLtY1+RNGgJG0Ku45oj3+E3uiWLx47kePJRd4fnNJpAlFLKRarVrE/rUd9xaMACtpVvT4eDE/D6sClLPx/FyaTD7g7vqmkCUUopF4uMaUbLJ38i/s4/2BF0DW0SJuD1UVOWf/YPko/sc3d4haYJRCmlikiNhm1o+dTP7LlrDhvLdyL24LcEjG7Jsk8e4kjCLneHd8V0HIhSSrnJvh3rOfTra7Q8/jsGYU3oTUT2fp6IWg2c/lk6kBBNIEqp0ufg3m3s/+V/tEiaiQc2VlfoQaVezxLdoJXTPkMTCJpAlFKlV9KBPez6+TWaHv4RP8lgnX87ynV5nHrXXA/nluAtJE0gaAJRSpV+JxIPsfnnd6i//zsqkso27/qciX2Ept3vKfTsv5pA0ASilCo7Tp86yfqZo4naNp4Ic4T9Uo2DDR6k6U3D8AsIvKJjaQJBE4hSquzJysxk3Z/fELR6NHWztnOM8myJuof6N48iNLxqgY6hCQRNIEqpssvYbGxZNousRe/T9OwK0ow360N7UqXHSKIatL7svppA0ASilFIA+7bGceiP92maPBs/yWCTb3MyY4fS5No7HbaTlKjp3EVkvIgcFZGNebwvIvKhiOwUkfUi0tJVsSilVGkTVb8VbUZM5MyjG1hS81FC0xNovng4h19tyNJJL5NyItnlMbhyJPoEoOdl3u8FxNgfQ4BPXRiLUkqVSqHhVWh//6uEvriFuDbvc9IrjHY73sbrvYYs/egB9m5b57LPdtlq8MaYhSISfZkifYCvjXUPbZmIBItIVWPMIVfFpJRSpZW3tw+teg2GXoPZuXYRJ/76iFZJM/CZPI31vs4bkJiTyxJIAUQA+3O8TrBvuySBiMgQrFoKUVFRRRKcUkqVVHWad4LmnTh2ZD/bZ31Crb1TXPI57pxM0dGwSoct+saYscaYWGNMbHh4uIvDUkqp0qFi5eq0HfQ6IS9uc8nx3ZlAEoDqOV5HAgfdFItSSpVa3t4+LjmuOxPIz8BAe2+stkCKtn8opVTJ4bI2EBGZDHQFwkQkAfg34A1gjBkD/AbcCOwEzgCDXRWLUkop53NlL6y783nfAI+46vOVUkq5lq5IqJRSqlA0gSillCoUTSBKKaUKRROIUkqpQilxs/GKSCrgmlExJU8YkOTuIIoJvRYX6LW4QK/FBfWMMUHOPKA7pzIprG3OnpK4pBKRVXotLHotLtBrcYFeiwtExOnrYOgtLKWUUoWiCUQppVShlMQEMtbdARQjei0u0GtxgV6LC/RaXOD0a1HiGtGVUkoVDyWxBqKUUqoY0ASilFKqUIptAhGRniKyTUR2ishz9m2TRGS9iPwvR7n/E5E+7ovU+USkuoj8JSJbRGSTiIy0b68oIn+KyA77zxD79n72cotEJNS+rbaIfOfO83AmEfEUkTUiMtP+uqaILLdfiyki4mPf/piIbBSR33Js6ygi77ozfmexL/08VUS22r8f7crq90JEHref30YRmSwi5crK90JExovIURHZmGNbXt8DEZEP7b9L14tIS/v2eiISJyLrRKSdfZuXiMwREf8CBWKMKXYPwBPYBdQCfIB1QFNgkv39RUAFoCrwi7vjdcH5VwVa2p8HAduBhsCbwHP27c8Bb9ifL7GXexh4zL5tMhDj7nNx4jV5AvgWmGl//T3Q3/58DDDc/nwd1h9GrwI3Y618+TsQ4u5zcNJ1+Ap4yP7cBwgui98LrOWv9wB+Ob4Pg8rK9wLoDLQENubYltf34EZglv2c2wLL7dvfBXoAMcA0+7bHgPsLGkdxrYFcA+w0xuw2xmQA3wE3AX4i4oH1Hycb+C/wL/eF6RrGmEPGmNX256nAFqz/MH2wfoFg/9nX/twG+AL+QKaIdAIOGWN2FGngLiIikVj//uPsrwXoBky1F8l5LcBad8YfyATuA34zxhwvsoBdRETKY/3i+ALAGJNhjDlBGf1eYA2E9hMRL6xzPEQZ+V4YYxYCx3Jtzut70Af42liWAcEiUhXrOvhx4fsRjJVcvy5oHMV1JHoEsD/H6wSgDbAPWA1MBOpg9SJbU/ThFR0RiQZaAMuBysa+aqMx5pCIVLIX+w/WX1MHgQHY/wor8mBd533gGay/pgFCgRPGmCz76wSs7wzA28AyYBOwGPgJ6Fl0obpULSAR+FJEmgFxwEjK4PfCGHNARN7G+p1wFvgD63qUxe/FOXl9Dxz9Po0APsFKFr7AUKw/xl819qpIQRTXGog42GaMMaOMMc2NMe8ALwP/EpEXReR7EXm4iGN0OREJBKYBo4wxJ/MqZ4z50xjTyhhzM9ZfHb8B9ez3yj8v8P3MYkhEegNHjTFxOTc7KGoAjDETjTEtjDEDsG57fQj0sl+L9+w12JLKC+u2xafGmBbAaaxbFQ6V8u9FCNZf1jWBakAA0MtB0bLwvchPXr9P9xljuhpj2mGtClsN2CoiE+3tR3XzO3BxvWgJQPUcryOx/ooCwN5ovgrrS9PYGHMncF9J/g+Rm4h4YyWPScaY6fbNR+xVT+w/j+baxx+4HxgNvAY8gPVX2b1FFbcLdABuEZG9WLcyu2HVSILtty4g1/cDQESqAa2NMTOAfwJ3AelA9yKK2xUSgARjzHL766lYCaUsfi+uA/YYYxKNMZnAdKA9ZfN7cU5e34PL/j61exX4P2AEMAlrCfJ/5/eBxTWBrARi7D0qfLCq3T/D+V+sI4G3sO7dnatunWsbKfHs9/i/ALYYY3L2EvkZ6xcB9p8zcu36DPCB/T+UH9a1sWFdpxLJGPO8MSbSGBON9T2YZ4y5F/gLuN1ezNG1eBnrPwSUnmtxGNgvIvXsm7oDmymD3wusW1dtRcTf/v/l3LUoc9+LHPL6HvwMDLT3xmoLpJy71QUgIl2AA/a2MX+s65FNQa6Ju3sTXKaXwY1YvY92AS/m2D4Key8BrKrZZGAD9h4HpeEBdMT6Yq8H1tofN2Ld+58L7LD/rJhjn2rYeyjZX9/Bhfu94e4+Jyddl65c6IVVC1gB7AR+AHxzlGsBfJHrO7MJmJ2zXEl8AM2xat/rse7jh5TV7wVWG89WYCNWu6hvWfle2H/vHcJqCE8AHszre2D/PfmJ/XfpBiA2x3EE+BN7bzSgAVY783qgQ35x6FQmSimlCqW43sJSSilVzGkCUUopVSiaQJRSShWKJhCllFKFoglEKaVUoWgCUSWCiGSLyFr7jKo/uGvQqIiMyuuzRaS3WDMGrxORzSIyNJ9jDRKRjwsZxwuF2KfQn6eUI5pAVElx1ljT2DQGMoBhBd1RRDydGMcoHAywsg9wHQvcbIxphjXmYL4TPze3K04gSjmbJhBVEi3CmkwTERkgIivstZPPziULETklIv8VkeVAOxFpLSJL7LWDFSISJNYaI2+JyEr7OglD7ft2FZH5cmHdjUn2UbwjsAbm/SUif+WKKQhrrqpkAGNMujFmm/144SIyzf45K0WkQ+4TyquMiASKyJcissEeYz8ReR1rFtq1IjIpn+swWES2i8gCrGlhlHIed4+o1Ic+CvIATtl/emFN0TAca9TsL4C3/b3RwED7cwPcaX/uA+zGmgMJoLz9OEOAf9q3+WKN8K6JNeI9BWvOIA9gKdDRXm4vEJZHjOOw5h+ajDXPlId9+7c59o/CmqIGrPUrPs6nzBvA+zk+IyTn9bA/d3gdsNaV2QeE26/B4nOfpw99OONRXKdzVyo3PxFZa3++CGuusCFAK2ClNR0SflyYQC4bazJKgHpY62CsBDD2mY1F5HqgqYicmzupAtbiOhnACmNMgr3cWiAa+PtyARpjHhKRJlgT/T2FtVjPIPvrhvYYAcqLSFCu3fMqcx05pmA3jtev6J7HdWgDzDfGJNrPYwqQ7wyrShWUJhBVUpw1xjTPucE+id5XxpjnHZRPM8ZknyvKhUk3LzoE1kp9v+c6blesGVrPyaaA/1eMMRuADSIyEWvFvEFYtZh2xpizuT4n58u8yuQVe+7zuOQ6iEjfAuyrVKFpG4gqyeYCt4t94Ryx1oSu4aDcVqCaiLS2lwuyT/n9OzDc3gCOiNQVkYB8PjOVCwtbnWdvq+iaY1NzIN7+/A/g0RxlL0qE+ZTJvT3E/jTzXNzkfR2WA11FJNRe9o58zk2pK6IJRJVYxpjNWGs6/CEi67FmFa3qoFwG1roPH4nIOnu5clhtFpuB1SKyEfiM/GsaY4FZDhrRBXhGRLbZb3n9B6v2AdYaC7H2RvDNOO5BlleZV4AQe/fldcC1OeJYLyKT8roOxpqy+yWsNpw5WLOsKuU0OhuvUkqpQtEaiFJKqULRBKKUUqpQNIEopZQqFE0gSimlCkUTiFJKqULRBKKUUqpQNIEopZQqFJ3KRBWZuLi4Sl5eXuOAxugfLyWNDdiYlZX1UKtWrY7mW1qVCZpAVJHx8vIaV6VKlQbh4eHHPTw8dARrCWKz2SQxMbHh4cOHxwG3uDseVTzoX4GqKDUODw8/qcmj5PHw8DDh4eEpWLVHpQBNIKpoeWjyKLns/3b6O0Odp18GVaZ4enq2ql+/fsOYmJhG3bp1q5OUlOSU5W63bdvmExMT08gZx8rpiSeeqFapUqWm9evXb1i/fv2G//jHPyKc/RnnLFmyxG/KlCkVXHV8VfpoAlFliq+vr23r1q2bd+zYsSk4ODjrrbfeCnd3TPkZNmzYka1bt27eunXr5tGjRx8o6H5ZWVlX9DmrVq3y//XXXzWBqALTBKLKrLZt254+cOCAD0BKSopHu3bt6jZs2LBB3bp1G37zzTfBYNUsatWq1ah///416tSp06hDhw4xp06dEoBFixb516tXr2Hz5s3rv/vuu5XOHffMmTNy++23R9etW7dhgwYNGv7yyy9BAB9++GHoddddV7tbt251IiIimvzvf/8L///27j2mqbOPA/ivF3ujpbYUOuQiiL23wAYrinbqFiFqWFh4O82YZhFNt727YXCLcyOwRIchOuYg8LacjAAACqtJREFUAy/oCNHMuWQDt0yyICNMccFhuV8GL0Og3CpKK6V4evr+oW2YMsAOxcnv8xfnnOc8fUjJ+fbhnP6ejIwMsUKhUEZERMgHBgZmPRv6/vvveQqFQimVSpV6vT7EZrNRAAACAgI0aWlp/lFRUbLCwkJBY2MjU6fTSVQqlSIqKkpWW1vLAgAoLCwUSCQSlUwmU0ZHR8vGx8cpn3766ZLS0lKBXC5XHj16VDD9CBDCp7DQPNl91hjU1m/hzGWf0qd4Y9n/ibg2m7YEQcCFCxd4KSkpwwAAHA6H/OGHH/4QCoWkyWSix8TEyF955ZUbAADd3d2s4uLiztjY2D83bty4rKioSPDmm29eT0lJCfnss8+6N23aZDUYDIGuvg8cOOAHANDW1tZUW1vL2rhxo6Sjo6Ph7j620WhsstlsVJlMpv744497m5ubm1JSUoIKCgp80tPT73tENj8/X3zmzBkfAIB9+/b1bNiwwWIwGELLyspaw8PD7S+99FJIdna2r+tcFotFXrlypRUAYOXKldIjR478qdFo7OXl5V5vvPFGcHV1dVtWVpZ/WVlZW2ho6O3h4WEai8Vy7tmzp6+mpsarqKio+5+9E2ihwABBC4rdbqfK5XJlb28vQ61WjyUmJo4C3HlM9b333gusrq7mUqlUGBwcZPT09NABAAICAuyxsbE2AICnn356rKuri2k2m2kWi4W2adMmKwDA9u3bzeXl5XwAgIsXL3Lffvvtwbvtx5csWTJRX1/PAgCIjY21CAQCUiAQkFwu16HX628AAGg0mrG6uropA/X1118f+OSTTwZc25cuXWIHBgbaw8PD7QAAr732mjkvL88P7q4Hv23bthGAO7Oq2tparl6vD3OdOzExQQEAiI6OtiYnJ4ckJSWNJCcnT7XOOkIzwgBB82K2M4W55roHYjabaXFxccuzsrL8Pvroo8GCggKh2Wym19fXNzOZTGdAQIDGZrNRAQAYDIb7yTEajea02WxUp9N575rmbtMt0ja5LyqVCiwWy+n6mSCIqTt8gP4BAHg8HgkA4HA4gMfjES0tLU33tjl16lR3eXm5V0lJCT8yMlJ19erVxtm8NkKT4T0QtCD5+Pg4Dh8+3J2Xlye22+2Umzdv0kQi0W0mk+ksLS3l9fX1MaY7XyQSObhcruP8+fNcAICTJ08KXcdWr15tLS4uFgIA1NXVMU0mEyM8PHx8rsYeGRk53tvby2hoaGACABQVFfnodDrLve2EQiEZGBg4UVhYKAAAIEkSLl26xAYAaGxsZD7//PO3cnJy+gQCAdHZ2cnw9vZ2WK1WvCagWcM/FrRgrVq1yqZQKGzHjh0T7Nix47rRaPRSq9WK4uJiYWho6IwX/OPHj3e98847wZGRkXI2m+2eFrz//vuDDoeDIpVKlZs3bw4rKCjomnz8n+JwOM78/PwuvV4fJpVKlVQqFdLS0oamanv69OnOEydOiGQymVIikai+/fbbxQAAqampgVKpVCmRSFQrVqywrFixwrZhwwZLW1sbG2+io9nCNdHRI2M0GrsiIiKG53scyHNGo1EUERERMt/jQI8HnIEghBDyCAYIQgghj2CAIIQQ8ggGCEIIIY9ggCCEEPIIBghCCCGPYICgBYVCoUTt3LnTXbcqPT1dvGvXriUP+3W1Wq2ssrLyvlIlWq1WplarFa7tyspKjlarlU3XV2trKyM/P184XRtPPKyS9OjJhQGCFhQGg+H88ccfBSaTaU7L+JAkCQ6Hw6NzzWYz/cyZM96zbd/e3s78+uuv5zRAHrT0O0IAGCBogaHRaM5t27YN7d+/X3zvsb6+Pnp8fHyYWq1WqNVqRVlZmRfAnUWd0tPT3e0lEomqtbWV4Sr1/uqrrwarVCplR0cHIzk5OVitViuWL1+uSk1NndXM5q233hrIysq6ry1BEGAwGALVarVCKpUqs7OzRQAAe/fuDaipqeHK5XJlZmam35o1a5ZfvnyZDQCgUCiUaWlp/gAA77777pJDhw6JSJIEg8EQKJFIVFKp1P0t83PnzvFiYmKkCQkJoTKZ7C8zj6amJoZCoVD+8ssvc1oxGT1ZsJgimh/f/TcIBpvm9uLkpxyDxLwZizTu3r17UKPRqDIyMvon7zcYDEG7du0aiI+Pt7a3tzPi4+MlnZ2d0xYZ7OrqYh09erSruLi4GwDg0KFDvWKx2EEQBMTGxsouX77MjomJsU3Xh06ns5aWli4uLS3l8fl89zQmJydHxOfzHQ0NDc02m43y7LPPyhMSEkb37dvXe/DgQfGFCxf+ALhTYbi8vJwrkUjsNBrNWV1dzQUAqK6u5m7fvv3PoqKixfX19ezm5uZGk8lE12q1iri4OCsAQF1dnVdtbW2jXC6faG1tZQAAGI1G5pYtW8KOHz/+P1cVYoSmggGCFhyhUEjq9XpzVlaWH5vNJl37f/31V+/29na2a9tqtdJGRkamnaX7+/tPvPDCC7dc21999ZXw5MmTIoIgKENDQ4uMRiNrpgABAPjwww9N+/fv9z9w4ECPa9/PP//s3dLSwikpKREAAFgsFlpTUxNrckVfAIC1a9daPv/8c/GyZcsm4uLiblZUVHhbLBZqT08PMyIiwn748GG/l19++TqdToegoCAiJibGWlVVxeHz+WR4ePgtuVw+4err+vXr9MTExOXffPNNR3R09JwVgERPJgwQND9mMVN4mPbs2TPwzDPPKLds2eKuzeV0OqGmpqaZy+X+5QJNp9OdJOnOGbDb7e6y6xwOx32gpaWFkZubK75y5Uqzr6+vIykpKWR8fHxW/yZ+8cUXLRkZGQFVVVVek8ZDOXjwYHdSUtLo5Lbnzp3jTd5+7rnnxlJSUjiVlZX2+Pj40eHhYXpOTo5Io9Hccv1ef2fy+AEAeDyew9/ff6KiooKLAYJmgvdA0IIkFosdCQkJI6dOnRK59q1evXrUtZogAMDFixfZAAAhISH2q1evegEAVFVVcXp7e5lT9TkyMkJjs9mkUCh0XLt2jV5RUfFA64t/8MEHpi+++OIp1/b69etvfvnll76uwKqrq2OOjo5S+Xy+w2q1upe/ZbFYTn9//9slJSWCdevW3dLpdJa8vLynVq1aZQUAWLNmjeXs2bNCgiCgr6+P/ttvv3F1Ot2t+0cAsGjRIudPP/3Ucfr0aZ+H8aQXerJggKAFa+/evf03btxwz8KPHDly7ffff/eSSqXKsLAwVW5uri/AnRX+RkZGaHK5XJmbm+u7dOnSKT+Zr1y50qZWq8ckEolq69atIVFRUdYHGc/mzZtvCoVC9+NQqampw3K5fFyj0SgkEolq586dS2/fvk3RarU2Op3ulMlkyszMTL+7r20RiUQEj8cj169fbx0YGFi0bt06KwDA1q1bb6hUKptCoVCtXbtWmpmZ2RMcHPy3j115e3uT58+f/yM3N1fsWhseoalgOXf0yGA5938/LOeOJsMZCEIIIY9ggCCEEPIIBghCCCGPYICgR4kkSZIyczP0OLr73pEzNkQLBgYIepQahoaG+Bgi/z4kSVKGhob4ANAw32NBjw/8IiF6ZAiC2NHf33+sv79fDfjh5d+GBIAGgiB2zPdA0OMDH+NFCCHkEfwUiBBCyCMYIAghhDyCAYIQQsgjGCAIIYQ8ggGCEELII/8HYd3X/lBohV8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "ax =sns.lineplot(data=all_lifts,x=\"percentile\",y=\"lift\",hue='model')\n",
    "# delete legend title and position legend outside graph\n",
    "Null, labels = ax.get_legend_handles_labels() \n",
    "ax.legend(labels=labels[1:],loc=9, bbox_to_anchor=(0.5, -0.15))\n",
    "ax.xaxis.set_major_formatter(FuncFormatter('{0:.0%}'.format))\n",
    "plt.ylabel('Lift')\n",
    "plt.xlabel('Percent Selected')\n",
    "plt.xlim([0,1])\n",
    "plt.ylim()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('scaler', MinMaxScaler(copy=True, feature_range=(0, 1))),\n",
       " ('clf',\n",
       "  MLPClassifier(activation='relu', alpha=0.020584494295802447, batch_size='auto',\n",
       "                beta_1=0.9, beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "                hidden_layer_sizes=(3,), learning_rate='constant',\n",
       "                learning_rate_init=0.001, max_iter=300, momentum=0.9,\n",
       "                n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
       "                random_state=42, shuffle=True, solver='adam', tol=0.0001,\n",
       "                validation_fraction=0.1, verbose=False, warm_start=False))]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_models['Neural Network'].get_params().pop('steps')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}