NLP_Machine_learning_model_gradient_boosting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body_len</th>\n",
       "      <th>punct%</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>8094</th>\n",
       "      <th>8095</th>\n",
       "      <th>8096</th>\n",
       "      <th>8097</th>\n",
       "      <th>8098</th>\n",
       "      <th>8099</th>\n",
       "      <th>8100</th>\n",
       "      <th>8101</th>\n",
       "      <th>8102</th>\n",
       "      <th>8103</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>4.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>4.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62</td>\n",
       "      <td>3.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>28</td>\n",
       "      <td>7.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>135</td>\n",
       "      <td>4.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 8106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   body_len  punct%    0    1    2    3    4    5    6    7  ...   8094  8095  \\\n",
       "0       128     4.7  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   0.0   \n",
       "1        49     4.1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   0.0   \n",
       "2        62     3.2  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   0.0   \n",
       "3        28     7.1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   0.0   \n",
       "4       135     4.4  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   0.0   \n",
       "\n",
       "   8096  8097  8098  8099  8100  8101  8102  8103  \n",
       "0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "1   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "3   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[5 rows x 8106 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "import pandas as pd\n",
    "import re\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import string\n",
    "\n",
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "ps = nltk.PorterStemmer()\n",
    "\n",
    "data = pd.read_csv('SMSSpamCollection.tsv', sep='\\t')\n",
    "data.columns = ['label', 'body_text']\n",
    "\n",
    "def count_punct(text):\n",
    "    count = sum((1 for char in text if char in string.punctuation))\n",
    "    return round(count/(len(text) - text.count(' ')), 3)*100\n",
    "\n",
    "data['body_len'] = data['body_text'].apply(lambda x: len(x) - x.count(\" \"))\n",
    "data['punct%'] = data['body_text'].apply(lambda x: count_punct(x))\n",
    "\n",
    "def clean_text(text):\n",
    "    text = ''.join([word.lower() for word in text if word not in string.punctuation])\n",
    "    tokens = re.split('\\W+', text)\n",
    "    text = [ps.stem(word) for word in tokens if word not in stopwords]\n",
    "    return text\n",
    "\n",
    "tfidf_vect = TfidfVectorizer(analyzer = clean_text)\n",
    "X_tfidf = tfidf_vect.fit_transform(data['body_text'])\n",
    "\n",
    "X_features = pd.concat([data['body_len'], data['punct%'], pd.DataFrame(X_tfidf.toarray())], axis = 1) \n",
    "X_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['_SUPPORTED_LOSS', '__abstractmethods__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_abc_cache', '_abc_negative_cache', '_abc_negative_cache_version', '_abc_registry', '_check_initialized', '_check_params', '_clear_state', '_decision_function', '_estimator_type', '_fit_stage', '_fit_stages', '_get_param_names', '_init_decision_function', '_init_state', '_is_initialized', '_make_estimator', '_resize_state', '_staged_decision_function', '_validate_estimator', '_validate_y', 'apply', 'decision_function', 'feature_importances_', 'fit', 'get_params', 'n_features', 'predict', 'predict_log_proba', 'predict_proba', 'score', 'set_params', 'staged_decision_function', 'staged_predict', 'staged_predict_proba']\n",
      "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
      "              learning_rate=0.1, loss='deviance', max_depth=3,\n",
      "              max_features=None, max_leaf_nodes=None,\n",
      "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "              min_samples_leaf=1, min_samples_split=2,\n",
      "              min_weight_fraction_leaf=0.0, n_estimators=100,\n",
      "              presort='auto', random_state=None, subsample=1.0, verbose=0,\n",
      "              warm_start=False)\n"
     ]
    }
   ],
   "source": [
    "print(dir(GradientBoostingClassifier))\n",
    "print(GradientBoostingClassifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support as score\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_features, data['label'], test_size= 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_GB(est, max_depth, lr):\n",
    "    gb = GradientBoostingClassifier(n_estimators=est, max_depth=max_depth, learning_rate=lr)\n",
    "    gb_model = gb.fit(X_train, y_train)\n",
    "    y_pred = gb_model.predict(X_test)\n",
    "    precision, recall, fscore, support = score(y_test, y_pred, pos_label= \"spam\", average = 'binary')\n",
    "    print('Est: {} / Depth: {} / LR: {} ------- precision: {} / recall: {} / Acurracy: {}'.format(\n",
    "        est, max_depth, lr, round(precision, 3), round(recall, 3),\n",
    "        round((y_pred==y_test).sum() / len(y_pred), 3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Est: 50 / Depth: 3 / LR: 0.01 ------- precision: 0.0 / recall: 0.0 / Acurracy: 0.868\n",
      "Est: 50 / Depth: 3 / LR: 0.1 ------- precision: 0.959 / recall: 0.796 / Acurracy: 0.969\n",
      "Est: 50 / Depth: 3 / LR: 1 ------- precision: 0.923 / recall: 0.816 / Acurracy: 0.967\n",
      "Est: 50 / Depth: 7 / LR: 0.01 ------- precision: 1.0 / recall: 0.014 / Acurracy: 0.87\n",
      "Est: 50 / Depth: 7 / LR: 0.1 ------- precision: 0.947 / recall: 0.857 / Acurracy: 0.975\n",
      "Est: 50 / Depth: 7 / LR: 1 ------- precision: 0.919 / recall: 0.85 / Acurracy: 0.97\n",
      "Est: 50 / Depth: 11 / LR: 0.01 ------- precision: 1.0 / recall: 0.034 / Acurracy: 0.873\n",
      "Est: 50 / Depth: 11 / LR: 0.1 ------- precision: 0.949 / recall: 0.884 / Acurracy: 0.978\n",
      "Est: 50 / Depth: 11 / LR: 1 ------- precision: 0.931 / recall: 0.912 / Acurracy: 0.979\n",
      "Est: 50 / Depth: 15 / LR: 0.01 ------- precision: 1.0 / recall: 0.014 / Acurracy: 0.87\n",
      "Est: 50 / Depth: 15 / LR: 0.1 ------- precision: 0.936 / recall: 0.891 / Acurracy: 0.978\n",
      "Est: 50 / Depth: 15 / LR: 1 ------- precision: 0.917 / recall: 0.905 / Acurracy: 0.977\n",
      "Est: 100 / Depth: 3 / LR: 0.01 ------- precision: 0.964 / recall: 0.544 / Acurracy: 0.937\n",
      "Est: 100 / Depth: 3 / LR: 0.1 ------- precision: 0.961 / recall: 0.83 / Acurracy: 0.973\n",
      "Est: 100 / Depth: 3 / LR: 1 ------- precision: 0.925 / recall: 0.837 / Acurracy: 0.969\n",
      "Est: 100 / Depth: 7 / LR: 0.01 ------- precision: 0.963 / recall: 0.701 / Acurracy: 0.957\n",
      "Est: 100 / Depth: 7 / LR: 0.1 ------- precision: 0.942 / recall: 0.891 / Acurracy: 0.978\n",
      "Est: 100 / Depth: 7 / LR: 1 ------- precision: 0.919 / recall: 0.85 / Acurracy: 0.97\n"
     ]
    }
   ],
   "source": [
    "for n_est in [50, 100, 150]:\n",
    "    for max_depth in [3, 7, 11, 15]:\n",
    "        for lr in [0.01, 0.1, 1]:\n",
    "            train_GB(n_est, max_depth, lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body_len</th>\n",
       "      <th>punct%</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>8094</th>\n",
       "      <th>8095</th>\n",
       "      <th>8096</th>\n",
       "      <th>8097</th>\n",
       "      <th>8098</th>\n",
       "      <th>8099</th>\n",
       "      <th>8100</th>\n",
       "      <th>8101</th>\n",
       "      <th>8102</th>\n",
       "      <th>8103</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>4.7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>4.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62</td>\n",
       "      <td>3.2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>28</td>\n",
       "      <td>7.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>135</td>\n",
       "      <td>4.4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 8106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   body_len  punct%  0  1  2  3  4  5  6  7  ...   8094  8095  8096  8097  \\\n",
       "0       128     4.7  0  0  0  0  0  0  0  0  ...      0     0     0     0   \n",
       "1        49     4.1  0  0  0  0  0  0  0  0  ...      0     0     0     0   \n",
       "2        62     3.2  0  0  0  0  0  0  0  0  ...      0     0     0     0   \n",
       "3        28     7.1  0  0  0  0  0  0  0  0  ...      0     0     0     0   \n",
       "4       135     4.4  0  0  0  0  0  0  0  0  ...      0     0     0     0   \n",
       "\n",
       "   8098  8099  8100  8101  8102  8103  \n",
       "0     0     0     0     0     0     0  \n",
       "1     0     0     0     0     0     0  \n",
       "2     0     0     0     0     0     0  \n",
       "3     0     0     0     0     0     0  \n",
       "4     0     0     0     0     0     0  \n",
       "\n",
       "[5 rows x 8106 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "import pandas as pd\n",
    "import re\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "import string\n",
    "\n",
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "ps = nltk.PorterStemmer()\n",
    "\n",
    "data = pd.read_csv('SMSSpamCollection.tsv', sep='\\t')\n",
    "data.columns = ['label', 'body_text']\n",
    "\n",
    "def count_punct(text):\n",
    "    count = sum((1 for char in text if char in string.punctuation))\n",
    "    return round(count/(len(text) - text.count(' ')), 3)*100\n",
    "\n",
    "data['body_len'] = data['body_text'].apply(lambda x: len(x) - x.count(\" \"))\n",
    "data['punct%'] = data['body_text'].apply(lambda x: count_punct(x))\n",
    "\n",
    "def clean_text(text):\n",
    "    text = ''.join([word.lower() for word in text if word not in string.punctuation])\n",
    "    tokens = re.split('\\W+', text)\n",
    "    text = [ps.stem(word) for word in tokens if word not in stopwords]\n",
    "    return text\n",
    "\n",
    "# TF-IDF\n",
    "tfidf_vect = TfidfVectorizer(analyzer = clean_text)\n",
    "X_tfidf = tfidf_vect.fit_transform(data['body_text'])\n",
    "X_tfidf_feat = pd.concat([data['body_len'], data['punct%'], pd.DataFrame(X_tfidf.toarray())], axis = 1)\n",
    "\n",
    "# CountVectorizer\n",
    "count_vect = CountVectorizer(analyzer = clean_text)\n",
    "X_count = count_vect.fit_transform(data['body_text'])\n",
    "X_count_feat = pd.concat([data['body_len'], data['punct%'], pd.DataFrame(X_count.toarray())], axis = 1) \n",
    "\n",
    "X_count_feat.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_train_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>split4_train_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>353.516538</td>\n",
       "      <td>3.132889</td>\n",
       "      <td>0.396316</td>\n",
       "      <td>0.043560</td>\n",
       "      <td>0.1</td>\n",
       "      <td>7</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 7, 'n_esti...</td>\n",
       "      <td>0.967713</td>\n",
       "      <td>0.981132</td>\n",
       "      <td>...</td>\n",
       "      <td>0.970361</td>\n",
       "      <td>0.005503</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>716.802505</td>\n",
       "      <td>101.432344</td>\n",
       "      <td>0.428775</td>\n",
       "      <td>0.163741</td>\n",
       "      <td>0.1</td>\n",
       "      <td>15</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 15, 'n_est...</td>\n",
       "      <td>0.965919</td>\n",
       "      <td>0.978437</td>\n",
       "      <td>...</td>\n",
       "      <td>0.970002</td>\n",
       "      <td>0.004338</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>544.022095</td>\n",
       "      <td>6.582808</td>\n",
       "      <td>0.432979</td>\n",
       "      <td>0.068480</td>\n",
       "      <td>0.1</td>\n",
       "      <td>11</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 11, 'n_est...</td>\n",
       "      <td>0.965022</td>\n",
       "      <td>0.979335</td>\n",
       "      <td>...</td>\n",
       "      <td>0.969643</td>\n",
       "      <td>0.005178</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>246.286867</td>\n",
       "      <td>12.384548</td>\n",
       "      <td>0.392709</td>\n",
       "      <td>0.024198</td>\n",
       "      <td>0.1</td>\n",
       "      <td>7</td>\n",
       "      <td>100</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 7, 'n_esti...</td>\n",
       "      <td>0.965022</td>\n",
       "      <td>0.979335</td>\n",
       "      <td>...</td>\n",
       "      <td>0.968385</td>\n",
       "      <td>0.005742</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>378.450409</td>\n",
       "      <td>2.540026</td>\n",
       "      <td>0.373989</td>\n",
       "      <td>0.050960</td>\n",
       "      <td>0.1</td>\n",
       "      <td>11</td>\n",
       "      <td>100</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 11, 'n_est...</td>\n",
       "      <td>0.964126</td>\n",
       "      <td>0.977538</td>\n",
       "      <td>...</td>\n",
       "      <td>0.968385</td>\n",
       "      <td>0.004923</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "1     353.516538      3.132889         0.396316        0.043560   \n",
       "5     716.802505    101.432344         0.428775        0.163741   \n",
       "3     544.022095      6.582808         0.432979        0.068480   \n",
       "0     246.286867     12.384548         0.392709        0.024198   \n",
       "2     378.450409      2.540026         0.373989        0.050960   \n",
       "\n",
       "  param_learning_rate param_max_depth param_n_estimators  \\\n",
       "1                 0.1               7                150   \n",
       "5                 0.1              15                150   \n",
       "3                 0.1              11                150   \n",
       "0                 0.1               7                100   \n",
       "2                 0.1              11                100   \n",
       "\n",
       "                                              params  split0_test_score  \\\n",
       "1  {'learning_rate': 0.1, 'max_depth': 7, 'n_esti...           0.967713   \n",
       "5  {'learning_rate': 0.1, 'max_depth': 15, 'n_est...           0.965919   \n",
       "3  {'learning_rate': 0.1, 'max_depth': 11, 'n_est...           0.965022   \n",
       "0  {'learning_rate': 0.1, 'max_depth': 7, 'n_esti...           0.965022   \n",
       "2  {'learning_rate': 0.1, 'max_depth': 11, 'n_est...           0.964126   \n",
       "\n",
       "   split1_test_score       ...         mean_test_score  std_test_score  \\\n",
       "1           0.981132       ...                0.970361        0.005503   \n",
       "5           0.978437       ...                0.970002        0.004338   \n",
       "3           0.979335       ...                0.969643        0.005178   \n",
       "0           0.979335       ...                0.968385        0.005742   \n",
       "2           0.977538       ...                0.968385        0.004923   \n",
       "\n",
       "   rank_test_score  split0_train_score  split1_train_score  \\\n",
       "1                1                 1.0                 1.0   \n",
       "5                2                 1.0                 1.0   \n",
       "3                3                 1.0                 1.0   \n",
       "0                4                 1.0                 1.0   \n",
       "2                4                 1.0                 1.0   \n",
       "\n",
       "   split2_train_score  split3_train_score  split4_train_score  \\\n",
       "1                 1.0                 1.0                 1.0   \n",
       "5                 1.0                 1.0                 1.0   \n",
       "3                 1.0                 1.0                 1.0   \n",
       "0                 1.0                 1.0                 1.0   \n",
       "2                 1.0                 1.0                 1.0   \n",
       "\n",
       "   mean_train_score  std_train_score  \n",
       "1               1.0              0.0  \n",
       "5               1.0              0.0  \n",
       "3               1.0              0.0  \n",
       "0               1.0              0.0  \n",
       "2               1.0              0.0  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb = GradientBoostingClassifier()\n",
    "param = {\n",
    "    'n_estimators':[100, 150],\n",
    "    'max_depth':[7,11,15],\n",
    "    'learning_rate':[0.1]\n",
    "}\n",
    "\n",
    "clf = GridSearchCV(gb, param, cv=5, n_jobs=-1)\n",
    "cv_fit = clf.fit(X_tfidf_feat, data['label'])\n",
    "pd.DataFrame(cv_fit.cv_results_).sort_values('mean_test_score', ascending=False)[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\dynamic_h\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_train_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>split4_train_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>584.067190</td>\n",
       "      <td>4.838024</td>\n",
       "      <td>0.458561</td>\n",
       "      <td>0.044244</td>\n",
       "      <td>0.1</td>\n",
       "      <td>11</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 11, 'n_est...</td>\n",
       "      <td>0.965022</td>\n",
       "      <td>0.978437</td>\n",
       "      <td>...</td>\n",
       "      <td>0.969643</td>\n",
       "      <td>0.004789</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11104.865202</td>\n",
       "      <td>8653.066880</td>\n",
       "      <td>0.307089</td>\n",
       "      <td>0.062728</td>\n",
       "      <td>0.1</td>\n",
       "      <td>15</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 15, 'n_est...</td>\n",
       "      <td>0.963229</td>\n",
       "      <td>0.978437</td>\n",
       "      <td>...</td>\n",
       "      <td>0.969643</td>\n",
       "      <td>0.005593</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>380.782241</td>\n",
       "      <td>25.809825</td>\n",
       "      <td>0.437204</td>\n",
       "      <td>0.057006</td>\n",
       "      <td>0.1</td>\n",
       "      <td>11</td>\n",
       "      <td>100</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 11, 'n_est...</td>\n",
       "      <td>0.964126</td>\n",
       "      <td>0.976640</td>\n",
       "      <td>...</td>\n",
       "      <td>0.968565</td>\n",
       "      <td>0.004604</td>\n",
       "      <td>3</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>210.436461</td>\n",
       "      <td>1.730359</td>\n",
       "      <td>0.416583</td>\n",
       "      <td>0.086502</td>\n",
       "      <td>0.1</td>\n",
       "      <td>7</td>\n",
       "      <td>100</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 7, 'n_esti...</td>\n",
       "      <td>0.963229</td>\n",
       "      <td>0.979335</td>\n",
       "      <td>...</td>\n",
       "      <td>0.968385</td>\n",
       "      <td>0.005794</td>\n",
       "      <td>4</td>\n",
       "      <td>0.999775</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999775</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999775</td>\n",
       "      <td>0.999865</td>\n",
       "      <td>0.00011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4025.652886</td>\n",
       "      <td>6975.556987</td>\n",
       "      <td>0.413123</td>\n",
       "      <td>0.124540</td>\n",
       "      <td>0.1</td>\n",
       "      <td>15</td>\n",
       "      <td>100</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 15, 'n_est...</td>\n",
       "      <td>0.963229</td>\n",
       "      <td>0.974843</td>\n",
       "      <td>...</td>\n",
       "      <td>0.968205</td>\n",
       "      <td>0.004026</td>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "3     584.067190      4.838024         0.458561        0.044244   \n",
       "5   11104.865202   8653.066880         0.307089        0.062728   \n",
       "2     380.782241     25.809825         0.437204        0.057006   \n",
       "0     210.436461      1.730359         0.416583        0.086502   \n",
       "4    4025.652886   6975.556987         0.413123        0.124540   \n",
       "\n",
       "  param_learning_rate param_max_depth param_n_estimators  \\\n",
       "3                 0.1              11                150   \n",
       "5                 0.1              15                150   \n",
       "2                 0.1              11                100   \n",
       "0                 0.1               7                100   \n",
       "4                 0.1              15                100   \n",
       "\n",
       "                                              params  split0_test_score  \\\n",
       "3  {'learning_rate': 0.1, 'max_depth': 11, 'n_est...           0.965022   \n",
       "5  {'learning_rate': 0.1, 'max_depth': 15, 'n_est...           0.963229   \n",
       "2  {'learning_rate': 0.1, 'max_depth': 11, 'n_est...           0.964126   \n",
       "0  {'learning_rate': 0.1, 'max_depth': 7, 'n_esti...           0.963229   \n",
       "4  {'learning_rate': 0.1, 'max_depth': 15, 'n_est...           0.963229   \n",
       "\n",
       "   split1_test_score       ...         mean_test_score  std_test_score  \\\n",
       "3           0.978437       ...                0.969643        0.004789   \n",
       "5           0.978437       ...                0.969643        0.005593   \n",
       "2           0.976640       ...                0.968565        0.004604   \n",
       "0           0.979335       ...                0.968385        0.005794   \n",
       "4           0.974843       ...                0.968205        0.004026   \n",
       "\n",
       "   rank_test_score  split0_train_score  split1_train_score  \\\n",
       "3                1            1.000000                 1.0   \n",
       "5                1            1.000000                 1.0   \n",
       "2                3            1.000000                 1.0   \n",
       "0                4            0.999775                 1.0   \n",
       "4                5            1.000000                 1.0   \n",
       "\n",
       "   split2_train_score  split3_train_score  split4_train_score  \\\n",
       "3            1.000000                 1.0            1.000000   \n",
       "5            1.000000                 1.0            1.000000   \n",
       "2            1.000000                 1.0            1.000000   \n",
       "0            0.999775                 1.0            0.999775   \n",
       "4            1.000000                 1.0            1.000000   \n",
       "\n",
       "   mean_train_score  std_train_score  \n",
       "3          1.000000          0.00000  \n",
       "5          1.000000          0.00000  \n",
       "2          1.000000          0.00000  \n",
       "0          0.999865          0.00011  \n",
       "4          1.000000          0.00000  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb = GradientBoostingClassifier()\n",
    "param = {\n",
    "    'n_estimators': [100, 150],\n",
    "    'max_depth': [7, 11, 15],\n",
    "    'learning_rate': [0.1]\n",
    "}\n",
    "\n",
    "gs = GridSearchCV(gb, param, cv=5, n_jobs=-1)\n",
    "cv_fit = gs.fit(X_count_feat, data['label'])\n",
    "pd.DataFrame(cv_fit.cv_results_).sort_values('mean_test_score', ascending=False)[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}