ensemble_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensembling on Digits dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import libraries\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1797, 64)\n"
     ]
    }
   ],
   "source": [
    "digits = load_digits()\n",
    "print(digits.data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(digits['data'], digits['target'], test_size = 0.25, random_state = 11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train : 1347\n",
      "X_test: 450\n",
      "y_train : 1347\n",
      "y_train : 450\n"
     ]
    }
   ],
   "source": [
    "#data details\n",
    "print(\"X_train :\", len(X_train))\n",
    "print(\"X_test:\", len(X_test))\n",
    "print(\"y_train :\", len(y_train))\n",
    "print(\"y_train :\", len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#create model objects\n",
    "log_clf = LogisticRegression()\n",
    "rnd_clf = RandomForestClassifier()\n",
    "svm_clf = SVC(probability=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#create an ensembled classifier\n",
    "voting_clf = VotingClassifier(\n",
    "estimators=[('lr', log_clf), ('rf', rnd_clf)],\n",
    "voting='soft'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time :  0.1565854549407959\n"
     ]
    }
   ],
   "source": [
    "#Train the model\n",
    "start = time.time()\n",
    "voting_clf.fit(X_train, y_train)\n",
    "print(\"Training time : \", time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#divide training data\n",
    "data_size = len(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression accuracy: 0.957777777778\n",
      "RandomForestClassifier accuracy: 0.935555555556\n",
      "VotingClassifier accuracy: 0.971111111111\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "clf_list = [log_clf, rnd_clf, voting_clf]\n",
    "for clf in clf_list:\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    accuracy = accuracy_score(y_pred, y_test)\n",
    "    print(clf.__class__.__name__ + \" accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging on Digits dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Set bootstrap = True for Bagging and false for Pasting\n",
    "bag_clf = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=1000, bootstrap=True, n_jobs=-1,random_state=11, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    2.4s remaining:    2.4s\n",
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    2.6s finished\n",
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    0.9s remaining:    0.9s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of bagging : 0.951111111111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    1.1s finished\n"
     ]
    }
   ],
   "source": [
    "bag_clf.fit(X_train, y_train)\n",
    "y_pred = bag_clf.predict(X_test)\n",
    "accuracy_bagging = accuracy_score(y_pred, y_test)\n",
    "print(\"Accuracy of bagging :\", accuracy_bagging)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Patches and Random Subspaces for Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag_clf_patch = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=1000, bootstrap=True, n_jobs=-1,random_state=11, verbose = 1, max_features=0.6, bootstrap_features=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    2.0s remaining:    2.0s\n",
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    2.2s finished\n",
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    1.0s remaining:    1.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of bagging with random patch: 0.973333333333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    1.2s finished\n"
     ]
    }
   ],
   "source": [
    "bag_clf_patch.fit(X_train, y_train)\n",
    "y_pred = bag_clf_patch.predict(X_test)\n",
    "accuracy_bagging = accuracy_score(y_pred, y_test)\n",
    "print(\"Accuracy of bagging with random patch:\", accuracy_bagging)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Set bootstrap = True for Bagging and false for Pasting\n",
    "pas_clf = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=1000, bootstrap=False, n_jobs=-1,random_state=11, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    3.0s remaining:    3.0s\n",
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    1.0s remaining:    1.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of pasting : 0.931111111111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    1.2s finished\n"
     ]
    }
   ],
   "source": [
    "pas_clf.fit(X_train, y_train)\n",
    "y_pred = pas_clf.predict(X_test)\n",
    "accuracy_bagging = accuracy_score(y_pred, y_test)\n",
    "print(\"Accuracy of pasting :\", accuracy_bagging)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble of Bagging and Pasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#create an ensembled classifier\n",
    "voting_clf_bagging = VotingClassifier(\n",
    "estimators=[('br', bag_clf), ('pr', pas_clf)],\n",
    "voting='soft'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    2.5s remaining:    2.5s\n",
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    2.7s finished\n",
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    3.0s remaining:    3.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time :  6.483613967895508\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    3.3s finished\n"
     ]
    }
   ],
   "source": [
    "#Train the model\n",
    "start = time.time()\n",
    "voting_clf_bagging.fit(X_train, y_train)\n",
    "print(\"Training time : \", time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    1.0s remaining:    1.0s\n",
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    1.0s remaining:    1.0s\n",
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    1.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VotingClassifier accuracy: 0.942222222222\n"
     ]
    }
   ],
   "source": [
    "y_pred = voting_clf_bagging.predict(X_test)\n",
    "accuracy = accuracy_score(y_pred, y_test)\n",
    "print(voting_clf_bagging.__class__.__name__ + \" accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GridSearch on Bagging "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    'n_estimators' : np.arange(500, 1000, 100).tolist(),\n",
    "    'max_samples': np.arange(100, 1000, 200).tolist(),\n",
    "    'bootstrap' : [True, False],\n",
    "    'max_features': np.arange(0.2, 1.0, 0.2),\n",
    "    'bootstrap_features': [True, False]\n",
    "}\n",
    "grid_voting_clf = GridSearchCV(bag_clf, param_grid=parameters, n_jobs=-1,verbose=10, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 400 candidates, totalling 2000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 tasks      | elapsed:    3.1s\n",
      "[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:    4.8s\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:    8.6s\n",
      "[Parallel(n_jobs=-1)]: Done  24 tasks      | elapsed:   12.0s\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:   15.7s\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   20.0s\n",
      "[Parallel(n_jobs=-1)]: Done  53 tasks      | elapsed:   26.4s\n",
      "[Parallel(n_jobs=-1)]: Done  64 tasks      | elapsed:   31.6s\n",
      "[Parallel(n_jobs=-1)]: Done  77 tasks      | elapsed:   40.6s\n",
      "[Parallel(n_jobs=-1)]: Done  90 tasks      | elapsed:   48.4s\n",
      "[Parallel(n_jobs=-1)]: Done 105 tasks      | elapsed:   58.0s\n",
      "[Parallel(n_jobs=-1)]: Done 120 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 137 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 173 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 213 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 234 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 257 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 280 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 305 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=-1)]: Done 330 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 357 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 384 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=-1)]: Done 413 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  5.7min\n",
      "[Parallel(n_jobs=-1)]: Done 473 tasks      | elapsed:  6.4min\n",
      "[Parallel(n_jobs=-1)]: Done 504 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=-1)]: Done 537 tasks      | elapsed:  7.4min\n",
      "[Parallel(n_jobs=-1)]: Done 570 tasks      | elapsed:  7.7min\n",
      "[Parallel(n_jobs=-1)]: Done 605 tasks      | elapsed:  8.0min\n",
      "[Parallel(n_jobs=-1)]: Done 640 tasks      | elapsed:  8.4min\n",
      "[Parallel(n_jobs=-1)]: Done 677 tasks      | elapsed:  8.8min\n",
      "[Parallel(n_jobs=-1)]: Done 714 tasks      | elapsed:  9.2min\n",
      "[Parallel(n_jobs=-1)]: Done 753 tasks      | elapsed:  9.8min\n",
      "[Parallel(n_jobs=-1)]: Done 792 tasks      | elapsed: 10.3min\n",
      "[Parallel(n_jobs=-1)]: Done 833 tasks      | elapsed: 10.9min\n",
      "[Parallel(n_jobs=-1)]: Done 874 tasks      | elapsed: 11.7min\n",
      "[Parallel(n_jobs=-1)]: Done 917 tasks      | elapsed: 12.2min\n",
      "[Parallel(n_jobs=-1)]: Done 960 tasks      | elapsed: 13.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1005 tasks      | elapsed: 14.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1050 tasks      | elapsed: 14.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1097 tasks      | elapsed: 15.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1144 tasks      | elapsed: 15.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1193 tasks      | elapsed: 16.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1242 tasks      | elapsed: 17.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1293 tasks      | elapsed: 17.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1344 tasks      | elapsed: 18.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1397 tasks      | elapsed: 19.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1450 tasks      | elapsed: 20.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1505 tasks      | elapsed: 22.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1560 tasks      | elapsed: 22.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1617 tasks      | elapsed: 23.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1674 tasks      | elapsed: 24.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1733 tasks      | elapsed: 25.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1792 tasks      | elapsed: 26.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1853 tasks      | elapsed: 27.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1914 tasks      | elapsed: 28.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1977 tasks      | elapsed: 30.2min\n",
      "[Parallel(n_jobs=-1)]: Done 2000 out of 2000 | elapsed: 31.1min finished\n",
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    3.3s remaining:    3.3s\n",
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    3.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=BaggingClassifier(base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "        ...n_estimators=500, n_jobs=-1, oob_score=False,\n",
       "         random_state=11, verbose=1, warm_start=False),\n",
       "       fit_params={}, iid=True, n_jobs=-1,\n",
       "       param_grid={'n_estimators': [500, 600, 700, 800, 900], 'max_samples': [100, 300, 500, 700, 900], 'bootstrap': [True, False], 'max_features': array([ 0.2,  0.4,  0.6,  0.8]), 'bootstrap_features': [True, False]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=10)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_voting_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': False, 'bootstrap_features': True, 'max_features': 0.60000000000000009, 'max_samples': 900, 'n_estimators': 900}\n"
     ]
    }
   ],
   "source": [
    "#best parameters\n",
    "print (grid_voting_clf.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   2 out of   4 | elapsed:    1.2s remaining:    1.2s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for grid serach is : 0.973333333333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   4 out of   4 | elapsed:    1.4s finished\n"
     ]
    }
   ],
   "source": [
    "predicted_grid_voting_clf = grid_voting_clf.predict(X_test)\n",
    "grid_accuracy = accuracy_score(predicted_grid_voting_clf, y_test)\n",
    "print(\"Accuracy for grid serach is :\", grid_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Accuracy using ensemble of Logistic Regression and Random Forest =  0.9711\n",
    "# Bagging Accuracy = 0.9511\n",
    "# Bagging with random patches = 0.9733\n",
    "# Pasting Accuracy = 0.9311\n",
    "# Ensemble of Bagging and Pasting = 0.9422\n",
    "# Grid Search on Bagging with random patches = 0.9733"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}