Readmission_catboost&lgbm.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Readmission_catboost.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 2
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython2",
      "version": "2.7.6"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "VXwIonM6F7th",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/acmilannesta/Adult_readmission"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HM0UNy4eGZ-b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install catboost"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VxA0rxmizgdD",
        "colab_type": "text"
      },
      "source": [
        "#**Catboost model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mwws2tm2F5TD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os,  pandas as pd,  numpy as np, gc\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from catboost import CatBoostClassifier, Pool, cv\n",
        "from hyperopt import fmin, hp, tpe, STATUS_OK, Trials"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kcvTc_KrGgv1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "analysis = pd.read_sas('Adult_readmission//combined_vars_04092019.sas7bdat', encoding='latin1').drop(['TRR_ID', 'TRANSPLANT_DT', 'TRANSPLANT_DISCHARGE_DT', 'READMISSION_DT'], axis=1)\n",
        "biomarker = pd.read_csv('Adult_readmission/feature_extracted_365days.csv')\n",
        "X = analysis.merge(biomarker, on='PERSON_ID').drop(['CODE_REHOSP', 'PERSON_ID'], 1)\n",
        "# X = biomarker.drop('PERSON_ID', 1)\n",
        "y = analysis['CODE_REHOSP'].replace(2, 0)\n",
        "cat_colidx = [X.columns.get_loc(col) for col in X.columns if X[col].nunique() <= 10]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AeJ9HsYYGlxG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for col in cat_colidx:\n",
        "    if X[X.columns[col]].dtype == 'float64':\n",
        "        X[X.columns[col]] = X[X.columns[col]].fillna(-1).astype(int)\n",
        "    else:\n",
        "        X[X.columns[col]] = X[X.columns[col]].fillna('')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mSEO3eJlGn8M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cbc_params = {\n",
        "    'max_depth': hp.quniform('max_depth', 2, 11, 1),\n",
        "    'l2_leaf_reg': hp.uniform('l2_leaf_reg', 0, 500),\n",
        "    'eta': hp.uniform('eta', 0.1, 0.5),\n",
        "    'silent': True,\n",
        "    'eval_metric': 'AUC',\n",
        "    'objective': 'Logloss',\n",
        "    'task_type': 'GPU',\n",
        "    'one_hot_max_size': 5\n",
        "#     'colsample_bylevel': hp.uniform('colsample_bylevel', 0.1, 1),\n",
        "#     'subsample': hp.uniform('subsample', 0.1, 1),    \n",
        "#     'bootstrap_type': hp.choice('bootstrap_type', ['Bernoulli', 'Poisson', 'No']),\n",
        "#     'one_hot_max_size': hp.choice('one_hot_max_size', np.arange(2,6))\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-zKHmwGIGtcR",
        "colab_type": "code",
        "outputId": "bd7b31cd-d832-448b-e78e-4df952281301",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 271
        }
      },
      "source": [
        "def f_cbc(params):\n",
        "  data = Pool(X, y, cat_colidx)\n",
        "  out = cv(\n",
        "      pool = data, \n",
        "      params = params, \n",
        "      nfold = 3, \n",
        "      early_stopping_rounds = 50, \n",
        "      iterations = 500, \n",
        "      # metric_period = 20\n",
        "      )\n",
        "  # kfold = StratifiedKFold(5, True, 2019)\n",
        "  # auc = np.zeros(kfold.get_n_splits())\n",
        "  # cbc_pred = np.zeros(len(X))\n",
        "  # featureimp = np.zeros(X.shape[1])\n",
        "  # cbc = CatBoostClassifier(\n",
        "  #     **params,\n",
        "  #     n_estimators=2000,\n",
        "  #     random_state=2019,\n",
        "  #     eval_metric='AUC',\n",
        "  #     cat_features=cat_colidx,\n",
        "  #     silent=True,\n",
        "  #     one_hot_max_size=2,\n",
        "  #     bootstrap_type='Bernoulli',\n",
        "  #     boosting_type='Plain',\n",
        "  #     task_type='GPU',\n",
        "  # )\n",
        "  # for i, (tr_idx, val_idx) in enumerate(kfold.split(X, y)):\n",
        "  #     clf = cbc.fit(X.iloc[tr_idx], \n",
        "  #                   y[tr_idx], \n",
        "  #                   use_best_model=True,\n",
        "  #                   eval_set=(X.iloc[val_idx], y[val_idx]),\n",
        "  #                   early_stopping_rounds=200,\n",
        "  #                   verbose_eval=False)\n",
        "  #     cbc_pred[val_idx] = clf.predict_proba(X.iloc[val_idx])[:, 1]\n",
        "  #     featureimp += np.asarray(clf.get_feature_importance()) / kfold.n_splits\n",
        "  #     auc[i] = roc_auc_score(y[val_idx], cbc_pred[val_idx])\n",
        "  #     del clf\n",
        "  #     gc.collect()\n",
        "  #     print(\"Mean AUC(%g|%g): %.5f\" %(i, kfold.get_n_splits(), np.sum(auc)/i))  +\n",
        "  # return {'loss': -np.mean(auc).round(5), 'status': STATUS_OK, 'featureimp': featureimp}\n",
        "  return {'loss': -round(out['test-AUC-mean'].max(), 5), 'status': STATUS_OK}\n",
        "\n",
        "trials = Trials()\n",
        "cbc_best = fmin(f_cbc, cbc_params, algo=tpe.suggest, rstate=np.random.RandomState(2019), max_evals=10, trials=trials)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/10 [00:00<?, ?it/s, best loss: ?]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 10%|█         | 1/10 [00:54<08:09, 54.37s/it, best loss: -0.6758]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 20%|██        | 2/10 [03:35<11:30, 86.29s/it, best loss: -0.6758]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 30%|███       | 3/10 [06:43<13:38, 116.87s/it, best loss: -0.67986]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 40%|████      | 4/10 [07:55<10:20, 103.43s/it, best loss: -0.67986]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 50%|█████     | 5/10 [12:53<13:28, 161.73s/it, best loss: -0.67986]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 60%|██████    | 6/10 [15:47<11:01, 165.45s/it, best loss: -0.67986]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mvFdD5WOgd-7",
        "colab_type": "code",
        "outputId": "3aabaa61-b3af-4ba9-fa83-4197b08f86e2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 639
        }
      },
      "source": [
        "pd.DataFrame({'features': X.columns, 'importance': trials.best_trial['result']['featureimp']}).sort_values('importance', ascending=False).head(20)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>features</th>\n",
              "      <th>importance</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>hemoglobin_min_tx</td>\n",
              "      <td>7.698757</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>creatinine_min_discharge</td>\n",
              "      <td>7.001471</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>prior_tx</td>\n",
              "      <td>6.792803</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>hemoglobin_min_discharge</td>\n",
              "      <td>4.884014</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>albumin_min_discharge</td>\n",
              "      <td>4.404390</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>prograf_max_discharge</td>\n",
              "      <td>3.345072</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>a1c_tx</td>\n",
              "      <td>3.066204</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>133</th>\n",
              "      <td>GLUCOSE__skewness</td>\n",
              "      <td>2.426604</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>wbc_min_tx</td>\n",
              "      <td>2.226028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110</th>\n",
              "      <td>MAGNESIUM__energy_ratio_by_chunks__num_segment...</td>\n",
              "      <td>2.207351</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>wbc_min_discharge</td>\n",
              "      <td>2.126185</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58</th>\n",
              "      <td>wbc_max_discharge</td>\n",
              "      <td>1.964200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>76</th>\n",
              "      <td>smoke</td>\n",
              "      <td>1.502926</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>abo_recipient</td>\n",
              "      <td>1.453557</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>creatinine_discharge</td>\n",
              "      <td>1.348038</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>wbc_max_tx</td>\n",
              "      <td>1.347501</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>albumin_discharge</td>\n",
              "      <td>1.235206</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>albumin_min_tx</td>\n",
              "      <td>1.194102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>creatinine_min_pretx</td>\n",
              "      <td>1.095848</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>TRANSPLANT_LOS</td>\n",
              "      <td>1.017302</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              features  importance\n",
              "63                                   hemoglobin_min_tx    7.698757\n",
              "45                            creatinine_min_discharge    7.001471\n",
              "72                                            prior_tx    6.792803\n",
              "66                            hemoglobin_min_discharge    4.884014\n",
              "38                               albumin_min_discharge    4.404390\n",
              "71                               prograf_max_discharge    3.345072\n",
              "59                                              a1c_tx    3.066204\n",
              "133                                  GLUCOSE__skewness    2.426604\n",
              "54                                          wbc_min_tx    2.226028\n",
              "110  MAGNESIUM__energy_ratio_by_chunks__num_segment...    2.207351\n",
              "57                                   wbc_min_discharge    2.126185\n",
              "58                                   wbc_max_discharge    1.964200\n",
              "76                                               smoke    1.502926\n",
              "32                                       abo_recipient    1.453557\n",
              "44                                creatinine_discharge    1.348038\n",
              "55                                          wbc_max_tx    1.347501\n",
              "37                                   albumin_discharge    1.235206\n",
              "35                                      albumin_min_tx    1.194102\n",
              "42                                creatinine_min_pretx    1.095848\n",
              "24                                      TRANSPLANT_LOS    1.017302"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FEQEHc6CmYcA",
        "colab_type": "text"
      },
      "source": [
        "#**LGBM model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KiP8b_skKDO3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 354
        },
        "outputId": "44ffecda-528c-46e7-d898-9ce065fea759"
      },
      "source": [
        "!pip uninstall lightgbm -y\n",
        "!pip install lightgbm --install-option=--gpu"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Uninstalling lightgbm-2.2.3:\n",
            "  Would remove:\n",
            "    /usr/local/lib/python3.6/dist-packages/lightgbm-2.2.3.dist-info/*\n",
            "    /usr/local/lib/python3.6/dist-packages/lightgbm/*\n",
            "Proceed (y/n)? y\n",
            "  Successfully uninstalled lightgbm-2.2.3\n",
            "/usr/local/lib/python3.6/dist-packages/pip/_internal/commands/install.py:244: UserWarning: Disabling all use of wheels due to the use of --build-options / --global-options / --install-options.\n",
            "  cmdoptions.check_install_build_global(options)\n",
            "Collecting lightgbm\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c9/ce/3aff55e25e282383c19c5a5fb7387fd400e64b1a1036671aefa63ceeaaf4/lightgbm-2.2.3.tar.gz (649kB)\n",
            "\u001b[K     |████████████████████████████████| 655kB 2.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from lightgbm) (1.16.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from lightgbm) (1.3.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from lightgbm) (0.21.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->lightgbm) (0.13.2)\n",
            "Skipping bdist_wheel for lightgbm, due to binaries being disabled for it.\n",
            "Installing collected packages: lightgbm\n",
            "  Running setup.py install for lightgbm ... \u001b[?25l\u001b[?25hdone\n",
            "Successfully installed lightgbm-2.2.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VyQo_4OmzquN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os,  pandas as pd,  numpy as np, gc\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from hyperopt import fmin, hp, tpe, STATUS_OK, Trials\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "import lightgbm as lgbm\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9_tmQRamX9W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "analysis = pd.read_sas('Adult_readmission//combined_vars_04092019.sas7bdat', encoding='latin1').drop(['TRR_ID', 'TRANSPLANT_DT', 'TRANSPLANT_DISCHARGE_DT', 'READMISSION_DT'], axis=1)\n",
        "biomarker = pd.read_csv('Adult_readmission/feature_extracted_365days.csv')\n",
        "X = analysis.merge(biomarker, on='PERSON_ID').drop(['CODE_REHOSP', 'PERSON_ID'], 1)\n",
        "y = analysis['CODE_REHOSP'].replace(2, 0)\n",
        "cat_col = [X.columns.get_loc(col) for col in X.columns if 2 <= X[col].nunique() <= 10]\n",
        "for col in X.columns:\n",
        "    if X[col].dtype == 'O':\n",
        "        X[col] = LabelEncoder().fit_transform(X[col].fillna('Unknown'))\n",
        "    elif 2 < X[col].nunique() <= 10:\n",
        "        X[col] = LabelEncoder().fit_transform(X[col].fillna(99))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7exawZEqzVEn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lgbm_param = {\n",
        "        'num_leaves': hp.choice('num_leaves', np.arange(2, 21)),\n",
        "        'learning_rate': hp.uniform('learning_rate', 0.005, 0.1),\n",
        "        'feature_fraction': hp.uniform('feature_fraction', 0.01, 0.5),\n",
        "        'max_depth': hp.choice('max_depth', np.arange(2, 11)),\n",
        "        'objective': 'binary',\n",
        "        # 'boosting_type': 'dart',\n",
        "        'metric': 'auc',\n",
        "        'verbose': -1,\n",
        "        'device_type': 'gpu'\n",
        "    }\n",
        "\n",
        "def f_lgbm(params):\n",
        "    lgbm_pred = np.zeros((len(X), ))\n",
        "    auc = np.zeros(5)\n",
        "    for i, (tr_idx, te_idx) in enumerate(StratifiedKFold(5, True, 2019).split(X, y)):\n",
        "        tr_data = lgbm.Dataset(X.values[tr_idx], y.ravel()[tr_idx], categorical_feature=cat_col)\n",
        "        te_data = lgbm.Dataset(X.values[te_idx], y.ravel()[te_idx], categorical_feature=cat_col)\n",
        "        clf = lgbm.train(params,\n",
        "                         tr_data,\n",
        "                         num_boost_round=2000,\n",
        "                         verbose_eval=200,\n",
        "                         valid_sets=[tr_data, te_data],\n",
        "                         early_stopping_rounds=False,\n",
        "                    )\n",
        "        lgbm_pred[te_idx] = clf.predict(X.values[te_idx], num_iteration=clf.best_iteration)\n",
        "        auc[i] = roc_auc_score(y.ravel()[te_idx], lgbm_pred[te_idx])\n",
        "        del clf\n",
        "        gc.collect()\n",
        "    return {'loss': -np.mean(auc).round(5), 'status': STATUS_OK}\n",
        "\n",
        "trials = Trials()\n",
        "lgbm_best = fmin(f_lgbm, lgbm_param, algo=tpe.suggest, rstate=np.random.RandomState(2019), max_evals=20, trials=trials)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}