Binary Workflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "import pandas as pd\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras import initializers\n",
    "from keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout\n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn import preprocessing\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import optimizers\n",
    "from sklearn.model_selection import train_test_split\n",
    "import cv2\n",
    "import os\n",
    "import random\n",
    "from glob import glob\n",
    "import seaborn as sns\n",
    "from sklearn.utils import class_weight\n",
    "from keras import regularizers\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from sklearn.externals import joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup (Data Preparation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4626\n",
      "4626\n"
     ]
    }
   ],
   "source": [
    "# Load npz file containing image arrays\n",
    "x_npz = np.load(\"x_images_arrays_bin.npz\")\n",
    "x = x_npz['arr_0']\n",
    "# Load disease labels\n",
    "y_npz = np.load(\"y_disease_labels_bin.npz\")\n",
    "y = y_npz['arr_0']\n",
    "print(len(y))\n",
    "print(len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(x,y, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3238, 128, 128, 3)\n",
      "(1388, 128, 128, 3)\n",
      "(3238,)\n",
      "(1388,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_reshape = X_train.reshape(X_train.shape[0], -1).astype('float32')\n",
    "X_test_reshape = X_test.reshape(X_test.shape[0], -1).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#one-hot encode y_train & y_test\n",
    "y_trainHot = keras.utils.to_categorical(y_train, 2)\n",
    "y_testHot = keras.utils.to_categorical(y_test, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs=5\n",
    "# optimizer = optimizers.Adam(lr=0.00003, decay=0.0001)\n",
    "optimizer = optimizers.SGD(lr=0.003, decay=0.0001, momentum=0.9)\n",
    "# optimizer= adam\n",
    "batch_size=100\n",
    "loss='categorical_crossentropy'\n",
    "activation='relu'\n",
    "out_activation='softmax'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tbcallback = keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, \n",
    "                                         write_graph=True, write_images=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Balanced Class Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.7598554533508541, 1: 1.4620733249051834}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_weights = class_weight.compute_class_weight('balanced', np.unique(y), y)\n",
    "class_weights = dict(enumerate(class_weights))\n",
    "class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3238 samples, validate on 1388 samples\n",
      "Epoch 1/5\n",
      "3238/3238 [==============================] - 55s 17ms/step - loss: 8.0433 - acc: 0.4395 - val_loss: 10.7880 - val_acc: 0.3307\n",
      "Epoch 2/5\n",
      "3238/3238 [==============================] - 53s 16ms/step - loss: 8.1433 - acc: 0.4185 - val_loss: 10.7880 - val_acc: 0.3307\n",
      "Epoch 3/5\n",
      "3238/3238 [==============================] - 52s 16ms/step - loss: 8.0761 - acc: 0.4234 - val_loss: 10.7880 - val_acc: 0.3307\n",
      "Epoch 4/5\n",
      "3238/3238 [==============================] - 53s 16ms/step - loss: 7.9218 - acc: 0.4268 - val_loss: 10.7880 - val_acc: 0.3307\n",
      "Epoch 5/5\n",
      "3238/3238 [==============================] - 52s 16ms/step - loss: 7.9409 - acc: 0.4240 - val_loss: 10.7880 - val_acc: 0.3307\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(100, input_dim=49152, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(75, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(25, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(25, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(10, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(2, activation=out_activation))\n",
    "model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) \n",
    "history = model.fit(X_train_reshape, y_trainHot, epochs=epochs, batch_size=batch_size, \n",
    "                    class_weight = class_weights, verbose=1, \n",
    "                    validation_data=[X_test_reshape, y_testHot], callbacks=[tbcallback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3168 samples, validate on 1359 samples\n",
      "Epoch 1/5\n",
      "3168/3168 [==============================] - 69s 22ms/step - loss: 13.8794 - acc: 0.6398 - val_loss: 5.4557 - val_acc: 0.6615\n",
      "Epoch 2/5\n",
      "3168/3168 [==============================] - 72s 23ms/step - loss: 13.8793 - acc: 0.6761 - val_loss: 5.4557 - val_acc: 0.6615\n",
      "Epoch 3/5\n",
      "3168/3168 [==============================] - 70s 22ms/step - loss: 13.8822 - acc: 0.6752 - val_loss: 5.4557 - val_acc: 0.6615\n",
      "Epoch 4/5\n",
      "3168/3168 [==============================] - 75s 24ms/step - loss: 13.8793 - acc: 0.6761 - val_loss: 5.4557 - val_acc: 0.6615\n",
      "Epoch 5/5\n",
      "3168/3168 [==============================] - 74s 23ms/step - loss: 13.8793 - acc: 0.6761 - val_loss: 5.4557 - val_acc: 0.6615\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(150, input_dim=49152, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(125, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(100, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(75, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(50, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(8, activation=out_activation))\n",
    "model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) \n",
    "history = model.fit(X_train_reshape, y_trainHot, epochs=epochs, batch_size=batch_size, \n",
    "                    class_weight = class_weights, verbose=1, \n",
    "                    validation_data=[X_test_reshape, y_testHot], callbacks=[tbcallback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can't pickle _thread.RLock objects",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-68-b41f9304db01>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'my_model_weights.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mjoblib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'mlp_balance.pickle'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(value, filename, compress, protocol, cache_size)\u001b[0m\n\u001b[1;32m    482\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mis_filename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    483\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 484\u001b[0;31m             \u001b[0mNumpyPickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    485\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m         \u001b[0mNumpyPickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    407\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_framing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 409\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    410\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSTOP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    411\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend_framing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    519\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m         \u001b[0;31m# Save the reduce() output and finally memoize the object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_reduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_reduce\u001b[0;34m(self, func, args, state, listitems, dictitems, obj)\u001b[0m\n\u001b[1;32m    632\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 634\u001b[0;31m             \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    635\u001b[0m             \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUILD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    477\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_dict\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    819\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    820\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 821\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_setitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    822\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    823\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36m_batch_setitems\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    845\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSETITEMS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    519\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m         \u001b[0;31m# Save the reduce() output and finally memoize the object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_reduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_reduce\u001b[0;34m(self, func, args, state, listitems, dictitems, obj)\u001b[0m\n\u001b[1;32m    632\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 634\u001b[0;31m             \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    635\u001b[0m             \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUILD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    477\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_dict\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    819\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    820\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 821\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_setitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    822\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    823\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36m_batch_setitems\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    845\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSETITEMS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    477\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_list\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    779\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 781\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_appends\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    782\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    783\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_list\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36m_batch_appends\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    803\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mMARK\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    804\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 805\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    806\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAPPENDS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    807\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    519\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m         \u001b[0;31m# Save the reduce() output and finally memoize the object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_reduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_reduce\u001b[0;34m(self, func, args, state, listitems, dictitems, obj)\u001b[0m\n\u001b[1;32m    632\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 634\u001b[0;31m             \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    635\u001b[0m             \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUILD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    477\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_dict\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    819\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    820\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 821\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_setitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    822\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    823\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36m_batch_setitems\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    845\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSETITEMS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    519\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m         \u001b[0;31m# Save the reduce() output and finally memoize the object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_reduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_reduce\u001b[0;34m(self, func, args, state, listitems, dictitems, obj)\u001b[0m\n\u001b[1;32m    632\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 634\u001b[0;31m             \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    635\u001b[0m             \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUILD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    477\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_dict\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    819\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    820\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 821\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_setitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    822\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    823\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36m_batch_setitems\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    845\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSETITEMS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    519\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m         \u001b[0;31m# Save the reduce() output and finally memoize the object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_reduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_reduce\u001b[0;34m(self, func, args, state, listitems, dictitems, obj)\u001b[0m\n\u001b[1;32m    632\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 634\u001b[0;31m             \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    635\u001b[0m             \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUILD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    477\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave_dict\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    819\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    820\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 821\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_setitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    822\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    823\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36m_batch_setitems\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    845\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSETITEMS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    494\u001b[0m             \u001b[0mreduce\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"__reduce_ex__\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mreduce\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 496\u001b[0;31m                 \u001b[0mrv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproto\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    497\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    498\u001b[0m                 \u001b[0mreduce\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"__reduce__\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: can't pickle _thread.RLock objects"
     ]
    }
   ],
   "source": [
    "model.save_weights('my_model_weights.h5')\n",
    "joblib.dump(model, 'mlp_balance.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Oversampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "ros = RandomOverSampler(ratio='auto')\n",
    "X_trainRos, y_trainRos = ros.fit_sample(X_train_reshape, y_train)\n",
    "X_testRos, y_testRos = ros.fit_sample(X_test_reshape, y_test)\n",
    "\n",
    "y_trainRosHot = keras.utils.to_categorical(y_trainRos, num_classes = 8)\n",
    "y_testRosHot = keras.utils.to_categorical(y_testRos, num_classes = 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train:  (3238, 128, 128, 3)\n",
      "X_trainFlat:  (3238, 49152)\n",
      "X_trainRos Shape:  (4230, 49152)\n",
      "X_testRos Shape:  (1858, 49152)\n",
      "Y_trainRosHot Shape:  (4230, 8)\n",
      "Y_testRosHot Shape:  (1858, 8)\n"
     ]
    }
   ],
   "source": [
    "print(\"X_train: \", X_train.shape)\n",
    "print(\"X_trainFlat: \", X_train_reshape.shape)\n",
    "print(\"X_trainRos Shape: \",X_trainRos.shape)\n",
    "print(\"X_testRos Shape: \",X_testRos.shape)\n",
    "print(\"Y_trainRosHot Shape: \",y_trainRosHot.shape)\n",
    "print(\"Y_testRosHot Shape: \",y_testRosHot.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 4230 samples, validate on 1858 samples\n",
      "Epoch 1/15\n",
      "4230/4230 [==============================] - 72s 17ms/step - loss: 9.0433 - acc: 0.4113 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 2/15\n",
      "4230/4230 [==============================] - 71s 17ms/step - loss: 8.1892 - acc: 0.4522 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 3/15\n",
      "4230/4230 [==============================] - 69s 16ms/step - loss: 8.2518 - acc: 0.4478 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 4/15\n",
      "4230/4230 [==============================] - 67s 16ms/step - loss: 8.3768 - acc: 0.4378 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 5/15\n",
      "4230/4230 [==============================] - 74s 17ms/step - loss: 8.2183 - acc: 0.4499 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 6/15\n",
      "4230/4230 [==============================] - 71s 17ms/step - loss: 8.3832 - acc: 0.4423 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 7/15\n",
      "4230/4230 [==============================] - 72s 17ms/step - loss: 8.2251 - acc: 0.4489 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 8/15\n",
      "4230/4230 [==============================] - 68s 16ms/step - loss: 7.8401 - acc: 0.4641 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 9/15\n",
      "4230/4230 [==============================] - 68s 16ms/step - loss: 7.2529 - acc: 0.5002 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 10/15\n",
      "4230/4230 [==============================] - 71s 17ms/step - loss: 7.3448 - acc: 0.4950 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 11/15\n",
      "4230/4230 [==============================] - 73s 17ms/step - loss: 7.1268 - acc: 0.5014 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 12/15\n",
      "4230/4230 [==============================] - 80s 19ms/step - loss: 7.0611 - acc: 0.5000 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 13/15\n",
      "4230/4230 [==============================] - 72s 17ms/step - loss: 7.1337 - acc: 0.4939 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 14/15\n",
      "4230/4230 [==============================] - 67s 16ms/step - loss: 7.0572 - acc: 0.5021 - val_loss: 8.0590 - val_acc: 0.5000\n",
      "Epoch 15/15\n",
      "4230/4230 [==============================] - 66s 16ms/step - loss: 6.9748 - acc: 0.5017 - val_loss: 8.0590 - val_acc: 0.5000\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(100, input_dim=49152, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(25, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(25, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(25, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(75, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(10, activation=activation))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(8, activation=out_activation))\n",
    "model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) \n",
    "history = model.fit(X_trainRos, y_trainRosHot, epochs=15, batch_size=batch_size, verbose=1, \n",
    "                    validation_data=(X_testRos, y_testRosHot), callbacks=[tbcallback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights('my_model_weights_ros.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}