EnsembleModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import load_data\n",
    "import numpy as np\n",
    "import keras\n",
    "from keras.layers import Dense, Conv1D, Flatten, Dropout\n",
    "from keras.layers import MaxPooling1D, AveragePooling1D\n",
    "from keras.models import Sequential, Model\n",
    "from keras.optimizers import Adam, SGD, RMSprop\n",
    "from sklearn.preprocessing import LabelBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84420, 310)\n",
      "(84420,)\n",
      "(84420, 3)\n",
      "\n",
      "(58128, 310)\n",
      "(58128,)\n",
      "(58128, 3)\n"
     ]
    }
   ],
   "source": [
    "data = load_data.read_data_sets(one_hot=False)\n",
    "\n",
    "x_train = data.train.data\n",
    "y_train = data.train.labels\n",
    "x_test = data.test.data\n",
    "y_test = data.test.labels\n",
    "\n",
    "lbin = LabelBinarizer()\n",
    "y_train_oh = lbin.fit_transform(y_train)\n",
    "y_test_oh = lbin.transform(y_test)\n",
    "\n",
    "print(x_train.shape)\n",
    "print(y_train.shape)\n",
    "print(y_train_oh.shape)\n",
    "print()\n",
    "print(x_test.shape)\n",
    "print(y_test.shape)\n",
    "print(y_test_oh.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statistics\n",
    "from statistics import mode\n",
    "\n",
    "def most_common(mylist):\n",
    "    my_dict = {}\n",
    "    for n in mylist:\n",
    "        if (n in my_dict):\n",
    "            my_dict[n] += 1\n",
    "        else:\n",
    "            my_dict[n] = 1\n",
    "    return int(max(my_dict, key=my_dict.get))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.svm import SVC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pca1 = PCA(180, svd_solver='full')\n",
    "# trx1 = pca1.fit_transform(x_train)\n",
    "# tsx1 = pca1.transform(x_test)\n",
    "\n",
    "# pca2 = PCA(220, svd_solver='full')\n",
    "# trx2 = pca2.fit_transform(x_train)\n",
    "# tsx2 = pca2.transform(x_test)\n",
    "\n",
    "# pca3 = PCA(250, svd_solver='full')\n",
    "# trx3 = pca3.fit_transform(x_train)\n",
    "# tsx3 = pca3.transform(x_test)\n",
    "\n",
    "# pca4 = PCA(225, svd_solver='full')\n",
    "# trx4 = pca4.fit_transform(x_train)\n",
    "# tsx4 = pca4.transform(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import RandomForestClassifier as RF\n",
    "\n",
    "# std1 = StandardScaler()\n",
    "# trx_1 = std1.fit_transform(x_train)\n",
    "# tsx_1 = std1.transform(x_test)\n",
    "\n",
    "# std2 = StandardScaler()\n",
    "# trx_2 = std2.fit_transform(trx1)\n",
    "# tsx_2 = std2.transform(tsx1)\n",
    "\n",
    "# std3 = StandardScaler()\n",
    "# trx_3 = std3.fit_transform(trx2)\n",
    "# tsx_3 = std3.transform(tsx2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tRF Accuracy (100 estimators) = 77.22% ..... 22.54 seconds\n",
      "\n",
      "\tRF Accuracy (150 estimators) = 78.52% ..... 33.58 seconds\n",
      "\n",
      "\tRF Accuracy (200 estimators) = 77.64% ..... 44.42 seconds\n",
      "\n",
      "\tRF Accuracy (250 estimators) = 77.47% ..... 54.86 seconds\n",
      "\n",
      "\tRF Accuracy (300 estimators) = 78.14% ..... 65.39 seconds\n",
      "\n",
      "\tRF Accuracy (350 estimators) = 78.06% ..... 75.42 seconds\n",
      "\n",
      "\tRF Accuracy (400 estimators) = 77.98% ..... 85.85 seconds\n",
      "\n",
      "\tRF Accuracy (450 estimators) = 78.14% ..... 95.69 seconds\n",
      "\n",
      "\tRF Accuracy (500 estimators) = 78.04% ..... 106.23 seconds\n",
      "\n",
      "\tRF Accuracy (550 estimators) = 78.1% ..... 117.16 seconds\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# rfs = []\n",
    "# for i in range(10):\n",
    "#     start = time.time()\n",
    "#     nest = 100 + (50 * i)\n",
    "#     rf = RF(n_estimators=nest, max_depth=5000, verbose=0, n_jobs=-1)\n",
    "#     rf.fit(x_train, y_train)\n",
    "#     stop = time.time()\n",
    "#     prd = rf.predict(x_test)\n",
    "#     rfs.append(prd)\n",
    "#     tm = round(stop - start, 2)\n",
    "#     rf_score = round(rf.score(x_test, y_test) * 100, 2)\n",
    "#     print('\\tRF Accuracy ({} estimators) = {}% ..... {} seconds\\n'.format(nest, rf_score, tm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(58128, 5)\n"
     ]
    }
   ],
   "source": [
    "# rfs = np.transpose(rfs)\n",
    "# preds_svms = np.transpose(preds_svms)\n",
    "\n",
    "# all_preds = np.transpose(np.array([*preds_svms]))\n",
    "# print(all_preds.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy after ensemble learning = 81.53%\n"
     ]
    }
   ],
   "source": [
    "# final_preds = []\n",
    "\n",
    "# for i in range(all_preds.shape[0]):\n",
    "#     f = most_common(all_preds[i])\n",
    "#     final_preds.append(f)\n",
    "\n",
    "# final_preds = np.array(final_preds)\n",
    "# y_trues = y_test\n",
    "\n",
    "# from sklearn.metrics import accuracy_score\n",
    "\n",
    "# acc = accuracy_score(y_trues, final_preds) * 100\n",
    "# print('Accuracy after ensemble learning = {}%'.format(round(acc, 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_svms(num, x_trains, y_trains, x_tests, y_tests):\n",
    "    print('='*50)\n",
    "    print('\\tMaking SVM Models')\n",
    "    print('='*50)\n",
    "    svm_predictions = []\n",
    "    for i in range(num):\n",
    "        x_train = x_trains[i]\n",
    "        y_train = y_trains[i]\n",
    "        x_test = x_tests[i]\n",
    "        y_test = y_tests[i]\n",
    "        \n",
    "        start = time.time()\n",
    "        \n",
    "        svm_model = SVC(C=0.2, verbose=2, gamma='auto')\n",
    "        svm_model.fit(x_train, y_train)\n",
    "        stop1 = time.time()\n",
    "        print('SVM Model-{} Fitted ... {} seconds'.format(i+1, round(stop1 - start, 2)))\n",
    "        svm_preds = svm_model.predict(x_test)\n",
    "        svm_predictions.append(svm_preds)\n",
    "        stop2 = time.time()\n",
    "        print('SVM-{} Labels Predicted ... {} seconds'.format(i+1, round(stop2 - stop1, 2)))\n",
    "        \n",
    "    return svm_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_randomforests(num, x_trains, y_trains, x_tests, y_tests):\n",
    "    print('='*50)\n",
    "    print('\\tBuilding Random Forests')\n",
    "    print('='*50)\n",
    "    rf_predictions = []\n",
    "    for i in range(num):\n",
    "        x_train = x_trains[i]\n",
    "        y_train = y_trains[i]\n",
    "        x_test = x_tests[i]\n",
    "        y_test = y_tests[i]\n",
    "        \n",
    "        start = time.time()\n",
    "        \n",
    "        rf_model = RF(n_estimators=200, verbose=1, n_jobs=-1)\n",
    "        rf_model.fit(x_train, y_train)\n",
    "        stop1 = time.time()\n",
    "        print('RandomForest-{} Fitted ... {} seconds'.format(i+1, round(stop1 - start, 2)))\n",
    "        rf_preds = rf_model.predict(x_test)\n",
    "        rf_predictions.append(rf_preds)\n",
    "        stop2 = time.time()\n",
    "        print('RandomForest-{} Labels Predicted ... {} seconds'.format(i+1, round(stop2 - stop1, 2)))\n",
    "    return rf_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_neuralnets(num, x_trains, y_trains, x_tests, y_tests):\n",
    "    print('='*50)\n",
    "    print('\\tMaking Neural Network Models')\n",
    "    print('='*50)\n",
    "    nn_predictions = []\n",
    "    for i in range(num):\n",
    "        lbin = LabelBinarizer()\n",
    "        x_train = x_trains[i]\n",
    "        y_train = y_trains[i]\n",
    "        y_train = lbin.fit_transform(y_train)\n",
    "        x_test = x_tests[i]\n",
    "        y_test = y_tests[i]\n",
    "        y_test = lbin.transform(y_test)\n",
    "        \n",
    "        model = Sequential()\n",
    "        \n",
    "#         model.add(Dense(512, name='fc1', activation='relu', input_shape=(x_train.shape[1],)))\n",
    "#         model.add(Dense(64, name='fc2', activation='relu'))\n",
    "# #         model.add(Dense(32, name='fc3', activation='relu'))\n",
    "#         model.add(Dense(3, name='classifier', activation='softmax'))\n",
    "#         model.compile(loss='categorical_crossentropy', metrics=['acc'], optimizer='adam')\n",
    "        \n",
    "        model.add(Dropout(0.1, input_shape=(x_train.shape[1],)))\n",
    "        model.add(Dense(512, name='fc1', activation='relu'))\n",
    "        model.add(Dropout(0.1))\n",
    "        model.add(Dense(256, name='fc2', activation='relu'))\n",
    "        model.add(Dropout(0.1))\n",
    "        model.add(Dense(128, name='fc3', activation='relu'))\n",
    "        model.add(Dropout(0.1))\n",
    "        model.add(Dense(64, name='fc4', activation='relu'))\n",
    "        # model.add(Dense(32, name='fc3', activation='relu'))\n",
    "        model.add(Dense(3, name='classifier', activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy', metrics=['acc'], optimizer='adam')\n",
    "        \n",
    "        print('NeuralNetwork-{} fitting now ...\\n'.format(i+1))\n",
    "        \n",
    "        h = model.fit(x_train, y_train, \n",
    "                      batch_size=1024, \n",
    "                      epochs=200, \n",
    "                      verbose=0,\n",
    "                      validation_data=(x_test, y_test))\n",
    "        \n",
    "        temp_preds = model.predict(x_test)\n",
    "        y_pred = np.argmax(temp_preds, axis=1)\n",
    "        \n",
    "        nn_predictions.append(y_pred)\n",
    "        \n",
    "        print('Test Accuracy = {}%'.format(round(h.history['val_acc'][-1] * 100, 2)))\n",
    "        \n",
    "    return nn_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_voting(all_predictions, y_true):\n",
    "    final_preds = []\n",
    "    for i in range(all_predictions.shape[0]):\n",
    "        f = most_common(all_predictions[i])\n",
    "        final_preds.append(f)\n",
    "\n",
    "    final_preds = np.array(final_preds)\n",
    "\n",
    "    acc = accuracy_score(y_true, final_preds) * 100\n",
    "    print()\n",
    "    print()\n",
    "    print('Accuracy after ensemble learning = {}%'.format(round(acc, 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "\tPCA Feature Extraction\n",
      "==================================================\n",
      "PCA 180 shape = (84420, 180)\n",
      "PCA 220 shape = (84420, 220)\n",
      "PCA 225 shape = (84420, 225)\n",
      "PCA 250 shape = (84420, 250)\n",
      "\n",
      "Data Generated ...\n"
     ]
    }
   ],
   "source": [
    "print('='*50)\n",
    "print('\\tPCA Feature Extraction')\n",
    "print('='*50)\n",
    "\n",
    "pca180 = PCA(180, svd_solver='full')\n",
    "trx180 = pca180.fit_transform(x_train)\n",
    "tsx180 = pca180.transform(x_test)\n",
    "print('PCA 180 shape = {}'.format(trx180.shape))\n",
    "\n",
    "pca220 = PCA(220, svd_solver='full')\n",
    "trx220 = pca220.fit_transform(x_train)\n",
    "tsx220 = pca220.transform(x_test)\n",
    "print('PCA 220 shape = {}'.format(trx220.shape))\n",
    "\n",
    "pca225 = PCA(225, svd_solver='full')\n",
    "trx225 = pca225.fit_transform(x_train)\n",
    "tsx225 = pca225.transform(x_test)\n",
    "print('PCA 225 shape = {}'.format(trx225.shape))\n",
    "\n",
    "pca250 = PCA(250, svd_solver='full')\n",
    "trx250 = pca250.fit_transform(x_train)\n",
    "tsx250 = pca250.transform(x_test)\n",
    "print('PCA 250 shape = {}'.format(trx250.shape))\n",
    "\n",
    "print()\n",
    "print('Data Generated ...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaling Data ...\n",
      "\n",
      "Data Scaling Complete ...\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "scaler = StandardScaler()\n",
    "print('Scaling Data ...')\n",
    "\n",
    "x_train_sc = scaler.fit_transform(x_train)\n",
    "x_test_sc = scaler.transform(x_test)\n",
    "\n",
    "trx180_sc = scaler.fit_transform(trx180)\n",
    "tsx180_sc = scaler.transform(tsx180)\n",
    "\n",
    "trx220_sc = scaler.fit_transform(trx220)\n",
    "tsx220_sc = scaler.transform(tsx220)\n",
    "\n",
    "trx225_sc = scaler.fit_transform(trx225)\n",
    "tsx225_sc = scaler.transform(tsx225)\n",
    "\n",
    "trx250_sc = scaler.fit_transform(trx250)\n",
    "tsx250_sc = scaler.transform(tsx250)\n",
    "\n",
    "print('\\nData Scaling Complete ...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_rf = 5\n",
    "rf_x_train = [x_train_sc, trx180_sc, trx220_sc, trx225_sc, trx250_sc]\n",
    "rf_y_train = [y_train, y_train, y_train, y_train, y_train]\n",
    "rf_x_test = [x_test_sc, tsx180_sc, tsx220_sc, tsx225_sc, tsx250_sc]\n",
    "rf_y_test = [y_test, y_test, y_test, y_test, y_test]\n",
    "\n",
    "num_svm = 4\n",
    "svm_x_train = [x_train, trx180, trx220, trx250]\n",
    "svm_y_train = [y_train, y_train, y_train, y_train]\n",
    "svm_x_test = [x_test, tsx180, tsx220, tsx250]\n",
    "svm_y_test = [y_test, y_test, y_test, y_test]\n",
    "\n",
    "num_nn = 5\n",
    "nn_x_train = [x_train_sc, trx180_sc, trx220_sc, trx225_sc, trx250_sc]\n",
    "nn_y_train = [y_train, y_train, y_train, y_train, y_train]\n",
    "nn_x_test = [x_test_sc, tsx180_sc, tsx220_sc, tsx225_sc, tsx250_sc]\n",
    "nn_y_test = [y_test, y_test, y_test, y_test, y_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "\tBuilding Random Forests\n",
      "==================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    8.3s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   32.2s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-1 Fitted ... 32.38 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-1 Labels Predicted ... 0.35 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    6.6s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   25.5s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-2 Fitted ... 25.63 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-2 Labels Predicted ... 0.22 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.2s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   28.2s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-3 Fitted ... 28.35 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-3 Labels Predicted ... 0.23 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.6s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   30.0s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-4 Fitted ... 30.15 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-4 Labels Predicted ... 0.23 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.9s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   30.4s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-5 Fitted ... 30.55 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-5 Labels Predicted ... 0.24 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    8.1s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   31.7s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-6 Fitted ... 31.87 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-6 Labels Predicted ... 0.35 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    6.8s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   25.8s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-7 Fitted ... 25.93 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-7 Labels Predicted ... 0.22 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.3s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   28.1s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-8 Fitted ... 28.24 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-8 Labels Predicted ... 0.23 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.8s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   29.8s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-9 Fitted ... 29.95 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-9 Labels Predicted ... 0.23 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.8s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   30.1s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-10 Fitted ... 30.26 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-10 Labels Predicted ... 0.24 seconds\n",
      "==================================================\n",
      "\tMaking Neural Network Models\n",
      "==================================================\n",
      "NeuralNetwork-1 fitting now ...\n",
      "\n",
      "\n",
      "\n",
      "NeuralNetwork-2 fitting now ...\n",
      "\n",
      "\n",
      "\n",
      "NeuralNetwork-3 fitting now ...\n",
      "\n",
      "\n",
      "\n",
      "==================================================\n",
      "\tMaking SVM Models\n",
      "==================================================\n",
      "[LibSVM]SVM Model-1 Fitted ... 641.23 seconds\n",
      "SVM-1 Labels Predicted ... 406.67 seconds\n",
      "[LibSVM]SVM Model-2 Fitted ... 315.63 seconds\n",
      "SVM-2 Labels Predicted ... 183.95 seconds\n",
      "[LibSVM]SVM Model-3 Fitted ... 415.39 seconds\n",
      "SVM-3 Labels Predicted ... 247.22 seconds\n",
      "[LibSVM]SVM Model-4 Fitted ... 487.85 seconds\n",
      "SVM-4 Labels Predicted ... 287.77 seconds\n",
      "Accuracy after ensemble learning = 81.47%\n"
     ]
    }
   ],
   "source": [
    "# Random forest predictions\n",
    "rf_preds = make_randomforests(num_rf, rf_x_train, rf_y_train, rf_x_test, rf_y_test)\n",
    "\n",
    "# Neural network predictions\n",
    "nn_preds = make_neuralnets(num_nn, nn_x_train, nn_y_train, nn_x_test, nn_y_test)\n",
    "\n",
    "# SVM predictions\n",
    "svm_preds = make_svms(num_svm, svm_x_train, svm_y_train, svm_x_test, svm_y_test)\n",
    "\n",
    "# Combining all predictions for majority-voting\n",
    "preds = np.array([*rf_preds, *svm_preds, *nn_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "\tBuilding Random Forests\n",
      "==================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    8.2s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   32.3s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-1 Fitted ... 32.48 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-1 Labels Predicted ... 0.35 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    6.8s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   25.8s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-2 Fitted ... 25.94 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-2 Labels Predicted ... 0.23 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.3s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   28.1s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-3 Fitted ... 28.24 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-3 Labels Predicted ... 0.23 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.9s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   30.1s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-4 Fitted ... 30.25 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-4 Labels Predicted ... 0.23 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.8s\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   30.1s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-5 Fitted ... 30.25 seconds\n",
      "RandomForest-5 Labels Predicted ... 0.24 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 150 out of 150 | elapsed:    0.2s finished\n"
     ]
    }
   ],
   "source": [
    "# Random forest predictions\n",
    "rf_preds = make_randomforests(num_rf, rf_x_train, rf_y_train, rf_x_test, rf_y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy after ensemble learning = 81.83%\n"
     ]
    }
   ],
   "source": [
    "# Neural network predictions\n",
    "# nn_preds = make_neuralnets(num_nn, nn_x_train, nn_y_train, nn_x_test, nn_y_test)\n",
    "\n",
    "# Combining all predictions for majority-voting\n",
    "preds = np.array([*rf_preds, *nn_preds, *svm_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy after ensemble learning = 82.34%\n"
     ]
    }
   ],
   "source": [
    "# Combining all predictions for majority-voting\n",
    "# preds = np.array([svm_preds[1], svm_preds[2], svm_preds[3], *rf_preds])\n",
    "preds = np.array([*rf_preds, *svm_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy after ensemble learning = 79.77%\n"
     ]
    }
   ],
   "source": [
    "preds = np.array([*rf_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy after ensemble learning = 81.06%\n"
     ]
    }
   ],
   "source": [
    "preds = np.array([*svm_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "\tMaking Neural Network Models\n",
      "==================================================\n",
      "NeuralNetwork-1 fitting now ...\n",
      "\n",
      "Test Accuracy = 81.7%\n",
      "NeuralNetwork-2 fitting now ...\n",
      "\n",
      "Test Accuracy = 72.3%\n",
      "NeuralNetwork-3 fitting now ...\n",
      "\n",
      "Test Accuracy = 71.93%\n",
      "NeuralNetwork-4 fitting now ...\n",
      "\n",
      "Test Accuracy = 72.58%\n",
      "NeuralNetwork-5 fitting now ...\n",
      "\n",
      "Test Accuracy = 73.7%\n",
      "Accuracy after ensemble learning = 78.84%\n"
     ]
    }
   ],
   "source": [
    "nn_preds = make_neuralnets(num_nn, nn_x_train, nn_y_train, nn_x_test, nn_y_test)\n",
    "\n",
    "preds = np.array([*nn_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy after ensemble learning = 82.61%\n"
     ]
    }
   ],
   "source": [
    "preds = np.array([*nn_preds, *svm_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "\tBuilding Random Forests\n",
      "==================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    8.4s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   39.9s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   43.3s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-1 Fitted ... 43.52 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-1 Labels Predicted ... 0.35 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    6.6s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   31.6s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   34.3s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-2 Fitted ... 34.37 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-2 Labels Predicted ... 0.33 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.2s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   34.4s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   37.2s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-3 Fitted ... 37.38 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-3 Labels Predicted ... 0.33 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.6s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   37.4s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   40.3s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-4 Fitted ... 40.38 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-4 Labels Predicted ... 0.33 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.8s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   37.6s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   40.5s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-5 Fitted ... 40.59 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-5 Labels Predicted ... 0.34 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    8.3s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   39.4s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   42.5s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-6 Fitted ... 42.61 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-6 Labels Predicted ... 0.35 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    6.7s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   31.6s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   34.2s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-7 Fitted ... 34.37 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-7 Labels Predicted ... 0.33 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.3s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   34.7s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   37.4s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-8 Fitted ... 37.58 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-8 Labels Predicted ... 0.34 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.7s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   37.1s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   40.0s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-9 Fitted ... 40.18 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-9 Labels Predicted ... 0.33 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    7.9s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   37.7s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   40.7s finished\n",
      "[Parallel(n_jobs=8)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    0.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-10 Fitted ... 40.79 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 184 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 200 out of 200 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest-10 Labels Predicted ... 0.34 seconds\n",
      "\n",
      "\n",
      "Accuracy after ensemble learning = 80.21%\n"
     ]
    }
   ],
   "source": [
    "numrf = 10\n",
    "rf_xtr = [x_train, trx180, trx220, trx225, trx250, x_train_sc, trx180_sc, trx220_sc, trx225_sc, trx250_sc]\n",
    "rf_ytr = [y_train, y_train, y_train, y_train, y_train, y_train, y_train, y_train, y_train, y_train]\n",
    "rf_xts = [x_test, tsx180, tsx220, tsx225, tsx250, x_test_sc, tsx180_sc, tsx220_sc, tsx225_sc, tsx250_sc]\n",
    "rf_yts = [y_test, y_test, y_test, y_test, y_test, y_test, y_test, y_test, y_test, y_test]\n",
    "\n",
    "# Random forest predictions\n",
    "rf_preds = make_randomforests(numrf, rf_xtr, rf_ytr, rf_xts, rf_yts)\n",
    "\n",
    "print()\n",
    "print()\n",
    "\n",
    "preds = np.array([*rf_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy after ensemble learning = 82.61%\n"
     ]
    }
   ],
   "source": [
    "## --- Final results for the project ---\n",
    "## All neural network models and all SVM models are majority-voted\n",
    "\n",
    "preds = np.array([*nn_preds, *svm_preds])\n",
    "preds = np.transpose(preds)\n",
    "\n",
    "# Calculating final predictions and final Accuracy\n",
    "make_voting(preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}