ML Pipeline Preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Pipeline Preparation\n",
    "Follow the instructions below to help you create your ML pipeline.\n",
    "### 1. Import libraries and load data from database.\n",
    "- Import Python libraries\n",
    "- Load dataset from database with [`read_sql_table`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_sql_table.html)\n",
    "- Define feature and target variables X and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/freemo/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/freemo/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# import libraries\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sqlalchemy import create_engine\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import nltk\n",
    "nltk.download(['punkt', 'wordnet'])\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import multilabel_confusion_matrix\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.datasets import make_multilabel_classification\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer\n",
    "from sklearn.metrics import make_scorer, accuracy_score, f1_score, classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "engine = create_engine('sqlite:///data/DisasterResponse.db')\n",
    "df = pd.read_sql(\"DisasterResponse\", engine)\n",
    "X = df['message']\n",
    "#drop also \"related\" column as there were records with 3 choices\n",
    "y = df.drop(['id', 'message', 'original', 'genre', 'related'], axis = 1)\n",
    "categories = y.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.isnull().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>request</th>\n",
       "      <th>offer</th>\n",
       "      <th>aid_related</th>\n",
       "      <th>medical_help</th>\n",
       "      <th>medical_products</th>\n",
       "      <th>search_and_rescue</th>\n",
       "      <th>security</th>\n",
       "      <th>military</th>\n",
       "      <th>child_alone</th>\n",
       "      <th>water</th>\n",
       "      <th>...</th>\n",
       "      <th>aid_centers</th>\n",
       "      <th>other_infrastructure</th>\n",
       "      <th>weather_related</th>\n",
       "      <th>floods</th>\n",
       "      <th>storm</th>\n",
       "      <th>fire</th>\n",
       "      <th>earthquake</th>\n",
       "      <th>cold</th>\n",
       "      <th>other_weather</th>\n",
       "      <th>direct_report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30178</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30179</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30180</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30181</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30182</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30183 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       request  offer  aid_related  medical_help  medical_products  \\\n",
       "0          NaN    NaN          NaN           NaN               NaN   \n",
       "1          NaN    NaN          NaN           NaN               NaN   \n",
       "2          NaN    NaN          NaN           NaN               NaN   \n",
       "3          NaN    NaN          NaN           NaN               NaN   \n",
       "4          NaN    NaN          NaN           NaN               NaN   \n",
       "...        ...    ...          ...           ...               ...   \n",
       "30178      0.0    0.0          1.0           0.0               0.0   \n",
       "30179      0.0    0.0          1.0           1.0               0.0   \n",
       "30180      0.0    0.0          1.0           0.0               0.0   \n",
       "30181      0.0    0.0          0.0           0.0               0.0   \n",
       "30182      0.0    0.0          1.0           0.0               0.0   \n",
       "\n",
       "       search_and_rescue  security  military  child_alone  water  ...  \\\n",
       "0                    NaN       NaN       NaN          NaN    NaN  ...   \n",
       "1                    NaN       NaN       NaN          NaN    NaN  ...   \n",
       "2                    NaN       NaN       NaN          NaN    NaN  ...   \n",
       "3                    NaN       NaN       NaN          NaN    NaN  ...   \n",
       "4                    NaN       NaN       NaN          NaN    NaN  ...   \n",
       "...                  ...       ...       ...          ...    ...  ...   \n",
       "30178                0.0       0.0       0.0          0.0    0.0  ...   \n",
       "30179                0.0       0.0       0.0          0.0    0.0  ...   \n",
       "30180                0.0       0.0       1.0          0.0    0.0  ...   \n",
       "30181                0.0       0.0       0.0          0.0    0.0  ...   \n",
       "30182                1.0       0.0       0.0          0.0    1.0  ...   \n",
       "\n",
       "       aid_centers  other_infrastructure  weather_related  floods  storm  \\\n",
       "0              NaN                   NaN              NaN     NaN    NaN   \n",
       "1              NaN                   NaN              NaN     NaN    NaN   \n",
       "2              NaN                   NaN              NaN     NaN    NaN   \n",
       "3              NaN                   NaN              NaN     NaN    NaN   \n",
       "4              NaN                   NaN              NaN     NaN    NaN   \n",
       "...            ...                   ...              ...     ...    ...   \n",
       "30178          0.0                   0.0              1.0     0.0    0.0   \n",
       "30179          0.0                   0.0              0.0     0.0    0.0   \n",
       "30180          0.0                   0.0              0.0     0.0    0.0   \n",
       "30181          0.0                   0.0              1.0     1.0    0.0   \n",
       "30182          0.0                   0.0              0.0     0.0    0.0   \n",
       "\n",
       "       fire  earthquake  cold  other_weather  direct_report  \n",
       "0       NaN         NaN   NaN            NaN            NaN  \n",
       "1       NaN         NaN   NaN            NaN            NaN  \n",
       "2       NaN         NaN   NaN            NaN            NaN  \n",
       "3       NaN         NaN   NaN            NaN            NaN  \n",
       "4       NaN         NaN   NaN            NaN            NaN  \n",
       "...     ...         ...   ...            ...            ...  \n",
       "30178   0.0         0.0   0.0            1.0            0.0  \n",
       "30179   0.0         0.0   0.0            0.0            0.0  \n",
       "30180   0.0         0.0   0.0            0.0            0.0  \n",
       "30181   1.0         0.0   0.0            1.0            0.0  \n",
       "30182   0.0         0.0   0.0            0.0            1.0  \n",
       "\n",
       "[30183 rows x 35 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories.value_counts().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26216, 35)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the shape\n",
    "X.shape\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Weather update - a cold front from Cuba that c...\n",
       "1              Is the Hurricane over or is it not over\n",
       "2                      Looking for someone but no name\n",
       "3    UN reports Leogane 80-90 destroyed. Only Hospi...\n",
       "4    says: west side of Haiti, rest of the country ...\n",
       "Name: message, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check randomly the two arrays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Write a tokenization function to process your text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a tokenization function\n",
    "def tokenize(text):\n",
    "    \"\"\"Tokenize and clean text.\n",
    "    \n",
    "    Args: message text\n",
    "    \n",
    "    Returns: normalized tokens\n",
    "    \"\"\"\n",
    "    \n",
    "    # remove punctuation\n",
    "    text = re.sub(r'[^a-zA-Z0-9]', ' ',text)\n",
    "    \n",
    "    # tokenize text\n",
    "    tokens = word_tokenize(text)\n",
    "    \n",
    "    # initiate lemmatizer\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    \n",
    "    # iterate through each token\n",
    "    clean_tokens = []\n",
    "    for tok in tokens:\n",
    "        \n",
    "        # lemmatize, normalize case, and remove leading/trailing white space\n",
    "        clean_tok = lemmatizer.lemmatize(tok).lower().strip()\n",
    "        clean_tokens.append(clean_tok)\n",
    "        \n",
    "    return clean_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weather update - a cold front from Cuba that could pass over Haiti\n",
      "['weather', 'update', 'a', 'cold', 'front', 'from', 'cuba', 'that', 'could', 'pas', 'over', 'haiti'] \n",
      "\n",
      "Is the Hurricane over or is it not over\n",
      "['is', 'the', 'hurricane', 'over', 'or', 'is', 'it', 'not', 'over'] \n",
      "\n",
      "Looking for someone but no name\n",
      "['looking', 'for', 'someone', 'but', 'no', 'name'] \n",
      "\n",
      "UN reports Leogane 80-90 destroyed. Only Hospital St. Croix functioning. Needs supplies desperately.\n",
      "['un', 'report', 'leogane', '80', '90', 'destroyed', 'only', 'hospital', 'st', 'croix', 'functioning', 'needs', 'supply', 'desperately'] \n",
      "\n",
      "says: west side of Haiti, rest of the country today and tonight\n",
      "['say', 'west', 'side', 'of', 'haiti', 'rest', 'of', 'the', 'country', 'today', 'and', 'tonight'] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# test function\n",
    "for message in X[:5]:\n",
    "    tokens = tokenize(message)\n",
    "    print(message)\n",
    "    print(tokens, '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Build a machine learning pipeline\n",
    "This machine pipeline should take in the `message` column as input and output classification results on the other 36 categories in the dataset. You may find the [MultiOutputClassifier](http://scikit-learn.org/stable/modules/generated/sklearn.multioutput.MultiOutputClassifier.html) helpful for predicting multiple target variables."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.analyticsvidhya.com/blog/2015/06/tuning-random-forest-model/\n",
    "\n",
    "https://medium.com/all-things-ai/in-depth-parameter-tuning-for-random-forest-d67bb7e920d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build the pipeline, parameters chosen after several attempts to maximize processing speed\n",
    "pipeline = Pipeline([\n",
    "    ('vect', CountVectorizer(tokenizer = tokenize)),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('clf', MultiOutputClassifier(RandomForestClassifier(n_estimators=100, oob_score=True, \n",
    "                                                         n_jobs=-1,random_state=50, max_features=\"auto\", \n",
    "                                                         min_samples_leaf=50)))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RandomForestRegressor(n_estimator = 100, oob_score = TRUE, n_jobs = -1,random_state =50, \n",
    "# max_features = \"auto\", min_samples_leaf = 50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Train pipeline\n",
    "- Split data into train and test sets\n",
    "- Train pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stratify=y,test_size=0.25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('vect',\n",
       "                 CountVectorizer(tokenizer=<function tokenize at 0x7fb57b26f7a0>)),\n",
       "                ('tfidf', TfidfTransformer()),\n",
       "                ('clf',\n",
       "                 MultiOutputClassifier(estimator=RandomForestClassifier(min_samples_leaf=50,\n",
       "                                                                        n_jobs=-1,\n",
       "                                                                        oob_score=True,\n",
       "                                                                        random_state=50)))])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train classifier, around 8 mins to complete / it took just 3mins with the new settings\n",
    "pipeline.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Test your model\n",
    "Report the f1 score, precision and recall for each output category of the dataset. You can do this by iterating through the columns and calling sklearn's `classification_report` on each."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict on test data\n",
    "y_pred = pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: request\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      1.00      0.91      6515\n",
      "           1       0.00      0.00      0.00      1350\n",
      "\n",
      "    accuracy                           0.83      7865\n",
      "   macro avg       0.41      0.50      0.45      7865\n",
      "weighted avg       0.69      0.83      0.75      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: offer\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      7827\n",
      "           1       0.00      0.00      0.00        38\n",
      "\n",
      "    accuracy                           1.00      7865\n",
      "   macro avg       0.50      0.50      0.50      7865\n",
      "weighted avg       0.99      1.00      0.99      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: aid_related\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.99      0.78      4615\n",
      "           1       0.91      0.21      0.34      3250\n",
      "\n",
      "    accuracy                           0.66      7865\n",
      "   macro avg       0.77      0.60      0.56      7865\n",
      "weighted avg       0.75      0.66      0.59      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: medical_help\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      7248\n",
      "           1       0.00      0.00      0.00       617\n",
      "\n",
      "    accuracy                           0.92      7865\n",
      "   macro avg       0.46      0.50      0.48      7865\n",
      "weighted avg       0.85      0.92      0.88      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: medical_products\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7479\n",
      "           1       0.00      0.00      0.00       386\n",
      "\n",
      "    accuracy                           0.95      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.90      0.95      0.93      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: search_and_rescue\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.99      7649\n",
      "           1       0.00      0.00      0.00       216\n",
      "\n",
      "    accuracy                           0.97      7865\n",
      "   macro avg       0.49      0.50      0.49      7865\n",
      "weighted avg       0.95      0.97      0.96      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: security\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      7724\n",
      "           1       0.00      0.00      0.00       141\n",
      "\n",
      "    accuracy                           0.98      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.96      0.98      0.97      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: military\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      7590\n",
      "           1       0.00      0.00      0.00       275\n",
      "\n",
      "    accuracy                           0.97      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.93      0.97      0.95      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: child_alone\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      7865\n",
      "\n",
      "    accuracy                           1.00      7865\n",
      "   macro avg       1.00      1.00      1.00      7865\n",
      "weighted avg       1.00      1.00      1.00      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: water\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      7381\n",
      "           1       0.00      0.00      0.00       484\n",
      "\n",
      "    accuracy                           0.94      7865\n",
      "   macro avg       0.47      0.50      0.48      7865\n",
      "weighted avg       0.88      0.94      0.91      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: food\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      1.00      0.94      6984\n",
      "           1       0.00      0.00      0.00       881\n",
      "\n",
      "    accuracy                           0.89      7865\n",
      "   macro avg       0.44      0.50      0.47      7865\n",
      "weighted avg       0.79      0.89      0.84      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: shelter\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.95      7155\n",
      "           1       0.00      0.00      0.00       710\n",
      "\n",
      "    accuracy                           0.91      7865\n",
      "   macro avg       0.45      0.50      0.48      7865\n",
      "weighted avg       0.83      0.91      0.87      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: clothing\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/freemo/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      7752\n",
      "           1       0.00      0.00      0.00       113\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.97      0.99      0.98      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: money\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      7694\n",
      "           1       0.00      0.00      0.00       171\n",
      "\n",
      "    accuracy                           0.98      7865\n",
      "   macro avg       0.49      0.50      0.49      7865\n",
      "weighted avg       0.96      0.98      0.97      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: missing_people\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      7771\n",
      "           1       0.00      0.00      0.00        94\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.98      0.99      0.98      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: refugees\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      7598\n",
      "           1       0.00      0.00      0.00       267\n",
      "\n",
      "    accuracy                           0.97      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.93      0.97      0.95      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: death\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7480\n",
      "           1       0.00      0.00      0.00       385\n",
      "\n",
      "    accuracy                           0.95      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.90      0.95      0.93      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: other_aid\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      1.00      0.93      6826\n",
      "           1       0.00      0.00      0.00      1039\n",
      "\n",
      "    accuracy                           0.87      7865\n",
      "   macro avg       0.43      0.50      0.46      7865\n",
      "weighted avg       0.75      0.87      0.81      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: infrastructure_related\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      7366\n",
      "           1       0.00      0.00      0.00       499\n",
      "\n",
      "    accuracy                           0.94      7865\n",
      "   macro avg       0.47      0.50      0.48      7865\n",
      "weighted avg       0.88      0.94      0.91      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: transport\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      7517\n",
      "           1       0.00      0.00      0.00       348\n",
      "\n",
      "    accuracy                           0.96      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.91      0.96      0.93      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: buildings\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7449\n",
      "           1       0.00      0.00      0.00       416\n",
      "\n",
      "    accuracy                           0.95      7865\n",
      "   macro avg       0.47      0.50      0.49      7865\n",
      "weighted avg       0.90      0.95      0.92      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: electricity\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      7710\n",
      "           1       0.00      0.00      0.00       155\n",
      "\n",
      "    accuracy                           0.98      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.96      0.98      0.97      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: tools\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      7818\n",
      "           1       0.00      0.00      0.00        47\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.50      0.50      0.50      7865\n",
      "weighted avg       0.99      0.99      0.99      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: hospitals\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      7787\n",
      "           1       0.00      0.00      0.00        78\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.50      0.50      0.50      7865\n",
      "weighted avg       0.98      0.99      0.99      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: shops\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      7831\n",
      "           1       0.00      0.00      0.00        34\n",
      "\n",
      "    accuracy                           1.00      7865\n",
      "   macro avg       0.50      0.50      0.50      7865\n",
      "weighted avg       0.99      1.00      0.99      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: aid_centers\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      7766\n",
      "           1       0.00      0.00      0.00        99\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.97      0.99      0.98      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: other_infrastructure\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      7525\n",
      "           1       0.00      0.00      0.00       340\n",
      "\n",
      "    accuracy                           0.96      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.92      0.96      0.94      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: weather_related\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      1.00      0.85      5734\n",
      "           1       0.93      0.04      0.08      2131\n",
      "\n",
      "    accuracy                           0.74      7865\n",
      "   macro avg       0.83      0.52      0.46      7865\n",
      "weighted avg       0.79      0.74      0.64      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: floods\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      7236\n",
      "           1       0.00      0.00      0.00       629\n",
      "\n",
      "    accuracy                           0.92      7865\n",
      "   macro avg       0.46      0.50      0.48      7865\n",
      "weighted avg       0.85      0.92      0.88      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: storm\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.95      7179\n",
      "           1       0.00      0.00      0.00       686\n",
      "\n",
      "    accuracy                           0.91      7865\n",
      "   macro avg       0.46      0.50      0.48      7865\n",
      "weighted avg       0.83      0.91      0.87      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: fire\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      7778\n",
      "           1       0.00      0.00      0.00        87\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.98      0.99      0.98      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: earthquake\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.95      7119\n",
      "           1       0.00      0.00      0.00       746\n",
      "\n",
      "    accuracy                           0.91      7865\n",
      "   macro avg       0.45      0.50      0.48      7865\n",
      "weighted avg       0.82      0.91      0.86      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: cold\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      7711\n",
      "           1       0.00      0.00      0.00       154\n",
      "\n",
      "    accuracy                           0.98      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.96      0.98      0.97      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: other_weather\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7467\n",
      "           1       0.00      0.00      0.00       398\n",
      "\n",
      "    accuracy                           0.95      7865\n",
      "   macro avg       0.47      0.50      0.49      7865\n",
      "weighted avg       0.90      0.95      0.92      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: direct_report\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      1.00      0.89      6357\n",
      "           1       0.00      0.00      0.00      1508\n",
      "\n",
      "    accuracy                           0.81      7865\n",
      "   macro avg       0.40      0.50      0.45      7865\n",
      "weighted avg       0.65      0.81      0.72      7865\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# check metrics\n",
    "y_pred_pd = pd.DataFrame(y_pred, columns = y_test.columns)\n",
    "for column in y_test.columns:\n",
    "    print('------------------------------------------------------\\n')\n",
    "    print('FEATURE: {}\\n'.format(column))\n",
    "    print(classification_report(y_test[column],y_pred_pd[column]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#print(classification_report(y_test, y_pred, target_names=y.columns))\n",
    "# multilabel_confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'request': 1,\n",
       "         'offer': 1,\n",
       "         'aid_related': 1,\n",
       "         'medical_help': 1,\n",
       "         'medical_products': 1,\n",
       "         'search_and_rescue': 1,\n",
       "         'security': 1,\n",
       "         'military': 1,\n",
       "         'child_alone': 1,\n",
       "         'water': 1,\n",
       "         'food': 1,\n",
       "         'shelter': 1,\n",
       "         'clothing': 1,\n",
       "         'money': 1,\n",
       "         'missing_people': 1,\n",
       "         'refugees': 1,\n",
       "         'death': 1,\n",
       "         'other_aid': 1,\n",
       "         'infrastructure_related': 1,\n",
       "         'transport': 1,\n",
       "         'buildings': 1,\n",
       "         'electricity': 1,\n",
       "         'tools': 1,\n",
       "         'hospitals': 1,\n",
       "         'shops': 1,\n",
       "         'aid_centers': 1,\n",
       "         'other_infrastructure': 1,\n",
       "         'weather_related': 1,\n",
       "         'floods': 1,\n",
       "         'storm': 1,\n",
       "         'fire': 1,\n",
       "         'earthquake': 1,\n",
       "         'cold': 1,\n",
       "         'other_weather': 1,\n",
       "         'direct_report': 1})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check data\n",
    "from collections import Counter\n",
    "Counter(y_train)\n",
    "Counter(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Improve your model\n",
    "Use grid search to find better parameters. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['memory', 'steps', 'verbose', 'vect', 'tfidf', 'clf', 'vect__analyzer', 'vect__binary', 'vect__decode_error', 'vect__dtype', 'vect__encoding', 'vect__input', 'vect__lowercase', 'vect__max_df', 'vect__max_features', 'vect__min_df', 'vect__ngram_range', 'vect__preprocessor', 'vect__stop_words', 'vect__strip_accents', 'vect__token_pattern', 'vect__tokenizer', 'vect__vocabulary', 'tfidf__norm', 'tfidf__smooth_idf', 'tfidf__sublinear_tf', 'tfidf__use_idf', 'clf__estimator__bootstrap', 'clf__estimator__ccp_alpha', 'clf__estimator__class_weight', 'clf__estimator__criterion', 'clf__estimator__max_depth', 'clf__estimator__max_features', 'clf__estimator__max_leaf_nodes', 'clf__estimator__max_samples', 'clf__estimator__min_impurity_decrease', 'clf__estimator__min_impurity_split', 'clf__estimator__min_samples_leaf', 'clf__estimator__min_samples_split', 'clf__estimator__min_weight_fraction_leaf', 'clf__estimator__n_estimators', 'clf__estimator__n_jobs', 'clf__estimator__oob_score', 'clf__estimator__random_state', 'clf__estimator__verbose', 'clf__estimator__warm_start', 'clf__estimator', 'clf__n_jobs'])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get params \n",
    "pipeline.get_params().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'memory': None,\n",
       " 'steps': [('vect',\n",
       "   CountVectorizer(tokenizer=<function tokenize at 0x7fb57b26f7a0>)),\n",
       "  ('tfidf', TfidfTransformer()),\n",
       "  ('clf',\n",
       "   MultiOutputClassifier(estimator=RandomForestClassifier(min_samples_leaf=50,\n",
       "                                                          n_jobs=-1,\n",
       "                                                          oob_score=True,\n",
       "                                                          random_state=50)))],\n",
       " 'verbose': False,\n",
       " 'vect': CountVectorizer(tokenizer=<function tokenize at 0x7fb57b26f7a0>),\n",
       " 'tfidf': TfidfTransformer(),\n",
       " 'clf': MultiOutputClassifier(estimator=RandomForestClassifier(min_samples_leaf=50,\n",
       "                                                        n_jobs=-1,\n",
       "                                                        oob_score=True,\n",
       "                                                        random_state=50)),\n",
       " 'vect__analyzer': 'word',\n",
       " 'vect__binary': False,\n",
       " 'vect__decode_error': 'strict',\n",
       " 'vect__dtype': numpy.int64,\n",
       " 'vect__encoding': 'utf-8',\n",
       " 'vect__input': 'content',\n",
       " 'vect__lowercase': True,\n",
       " 'vect__max_df': 1.0,\n",
       " 'vect__max_features': None,\n",
       " 'vect__min_df': 1,\n",
       " 'vect__ngram_range': (1, 1),\n",
       " 'vect__preprocessor': None,\n",
       " 'vect__stop_words': None,\n",
       " 'vect__strip_accents': None,\n",
       " 'vect__token_pattern': '(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       " 'vect__tokenizer': <function __main__.tokenize(text)>,\n",
       " 'vect__vocabulary': None,\n",
       " 'tfidf__norm': 'l2',\n",
       " 'tfidf__smooth_idf': True,\n",
       " 'tfidf__sublinear_tf': False,\n",
       " 'tfidf__use_idf': True,\n",
       " 'clf__estimator__bootstrap': True,\n",
       " 'clf__estimator__ccp_alpha': 0.0,\n",
       " 'clf__estimator__class_weight': None,\n",
       " 'clf__estimator__criterion': 'gini',\n",
       " 'clf__estimator__max_depth': None,\n",
       " 'clf__estimator__max_features': 'auto',\n",
       " 'clf__estimator__max_leaf_nodes': None,\n",
       " 'clf__estimator__max_samples': None,\n",
       " 'clf__estimator__min_impurity_decrease': 0.0,\n",
       " 'clf__estimator__min_impurity_split': None,\n",
       " 'clf__estimator__min_samples_leaf': 50,\n",
       " 'clf__estimator__min_samples_split': 2,\n",
       " 'clf__estimator__min_weight_fraction_leaf': 0.0,\n",
       " 'clf__estimator__n_estimators': 100,\n",
       " 'clf__estimator__n_jobs': -1,\n",
       " 'clf__estimator__oob_score': True,\n",
       " 'clf__estimator__random_state': 50,\n",
       " 'clf__estimator__verbose': 0,\n",
       " 'clf__estimator__warm_start': False,\n",
       " 'clf__estimator': RandomForestClassifier(min_samples_leaf=50, n_jobs=-1, oob_score=True,\n",
       "                        random_state=50),\n",
       " 'clf__n_jobs': None}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check all features\n",
    "pipeline.get_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "parameters = {'tfidf__norm': ['l1','l2'],\n",
    "              'clf__estimator__criterion': [\"gini\", \"entropy\"]\n",
    "    \n",
    "             }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# working model\n",
    "# parameters = {'clf__estimator__n_estimators': [100, 200]}\n",
    "\n",
    "# rf_Grid.best_params_\n",
    "\n",
    "#parameters = {'tfidf__norm': ['l1','l2'],'clf__estimator__criterion': [\"gini\", \"entropy\"]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "parameters = {\n",
    "        'features__text_pipeline__vect__ngram_range': ((1, 1), (1, 2)),\n",
    "        'features__text_pipeline__vect__max_df': (0.5, 0.75, 1.0),\n",
    "        'features__text_pipeline__vect__max_features': (None, 5000, 10000),\n",
    "        'features__text_pipeline__tfidf__use_idf': (True, False),\n",
    "        'clf__n_estimators': [50, 100, 200],\n",
    "        'clf__min_samples_split': [2, 3, 4],\n",
    "        'features__transformer_weights': (\n",
    "            {'text_pipeline': 1, 'starting_verb': 0.5},\n",
    "            {'text_pipeline': 0.5, 'starting_verb': 1},\n",
    "            {'text_pipeline': 0.8, 'starting_verb': 1},\n",
    "        )\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build GridSearch with parameters\n",
    "parameters = {'tfidf__norm': ['l1','l2'],\n",
    "              'clf__estimator__criterion': [\"gini\", \"entropy\"]\n",
    "    \n",
    "             }\n",
    "\n",
    "cv = GridSearchCV(pipeline, param_grid=parameters, cv=2, n_jobs=-1, verbose=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed:  1.3min remaining:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed:  1.3min remaining:   45.8s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  1.3min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  1.3min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2,\n",
       "             estimator=Pipeline(steps=[('vect',\n",
       "                                        CountVectorizer(tokenizer=<function tokenize at 0x7fb57b26f7a0>)),\n",
       "                                       ('tfidf', TfidfTransformer()),\n",
       "                                       ('clf',\n",
       "                                        MultiOutputClassifier(estimator=RandomForestClassifier(min_samples_leaf=50,\n",
       "                                                                                               n_jobs=-1,\n",
       "                                                                                               oob_score=True,\n",
       "                                                                                               random_state=50)))]),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'clf__estimator__criterion': ['gini', 'entropy'],\n",
       "                         'tfidf__norm': ['l1', 'l2']},\n",
       "             verbose=3)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# nothing for 15min, with improved settings it took just 2mins\n",
    "cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__estimator__criterion': 'gini', 'tfidf__norm': 'l2'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the best parameters\n",
    "cv.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Test your model\n",
    "Show the accuracy, precision, and recall of the tuned model.  \n",
    "\n",
    "Since this project focuses on code quality, process, and  pipelines, there is no minimum performance metric needed to pass. However, make sure to fine tune your models for accuracy, precision and recall to make your project stand out - especially for your portfolio!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test model\n",
    "y_pred = cv.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: request\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      1.00      0.91      6515\n",
      "           1       0.00      0.00      0.00      1350\n",
      "\n",
      "    accuracy                           0.83      7865\n",
      "   macro avg       0.41      0.50      0.45      7865\n",
      "weighted avg       0.69      0.83      0.75      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: offer\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      7827\n",
      "           1       0.00      0.00      0.00        38\n",
      "\n",
      "    accuracy                           1.00      7865\n",
      "   macro avg       0.50      0.50      0.50      7865\n",
      "weighted avg       0.99      1.00      0.99      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: aid_related\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.99      0.78      4615\n",
      "           1       0.91      0.21      0.34      3250\n",
      "\n",
      "    accuracy                           0.66      7865\n",
      "   macro avg       0.77      0.60      0.56      7865\n",
      "weighted avg       0.75      0.66      0.59      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: medical_help\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      7248\n",
      "           1       0.00      0.00      0.00       617\n",
      "\n",
      "    accuracy                           0.92      7865\n",
      "   macro avg       0.46      0.50      0.48      7865\n",
      "weighted avg       0.85      0.92      0.88      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: medical_products\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7479\n",
      "           1       0.00      0.00      0.00       386\n",
      "\n",
      "    accuracy                           0.95      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.90      0.95      0.93      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: search_and_rescue\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.99      7649\n",
      "           1       0.00      0.00      0.00       216\n",
      "\n",
      "    accuracy                           0.97      7865\n",
      "   macro avg       0.49      0.50      0.49      7865\n",
      "weighted avg       0.95      0.97      0.96      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: security\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      7724\n",
      "           1       0.00      0.00      0.00       141\n",
      "\n",
      "    accuracy                           0.98      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.96      0.98      0.97      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: military\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      7590\n",
      "           1       0.00      0.00      0.00       275\n",
      "\n",
      "    accuracy                           0.97      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.93      0.97      0.95      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: child_alone\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      7865\n",
      "\n",
      "    accuracy                           1.00      7865\n",
      "   macro avg       1.00      1.00      1.00      7865\n",
      "weighted avg       1.00      1.00      1.00      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: water\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      7381\n",
      "           1       0.00      0.00      0.00       484\n",
      "\n",
      "    accuracy                           0.94      7865\n",
      "   macro avg       0.47      0.50      0.48      7865\n",
      "weighted avg       0.88      0.94      0.91      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: food\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      1.00      0.94      6984\n",
      "           1       0.00      0.00      0.00       881\n",
      "\n",
      "    accuracy                           0.89      7865\n",
      "   macro avg       0.44      0.50      0.47      7865\n",
      "weighted avg       0.79      0.89      0.84      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: shelter\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.95      7155\n",
      "           1       0.00      0.00      0.00       710\n",
      "\n",
      "    accuracy                           0.91      7865\n",
      "   macro avg       0.45      0.50      0.48      7865\n",
      "weighted avg       0.83      0.91      0.87      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: clothing\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      7752\n",
      "           1       0.00      0.00      0.00       113\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.97      0.99      0.98      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: money\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      7694\n",
      "           1       0.00      0.00      0.00       171\n",
      "\n",
      "    accuracy                           0.98      7865\n",
      "   macro avg       0.49      0.50      0.49      7865\n",
      "weighted avg       0.96      0.98      0.97      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: missing_people\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      7771\n",
      "           1       0.00      0.00      0.00        94\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.98      0.99      0.98      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: refugees\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      7598\n",
      "           1       0.00      0.00      0.00       267\n",
      "\n",
      "    accuracy                           0.97      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.93      0.97      0.95      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: death\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7480\n",
      "           1       0.00      0.00      0.00       385\n",
      "\n",
      "    accuracy                           0.95      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.90      0.95      0.93      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: other_aid\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      1.00      0.93      6826\n",
      "           1       0.00      0.00      0.00      1039\n",
      "\n",
      "    accuracy                           0.87      7865\n",
      "   macro avg       0.43      0.50      0.46      7865\n",
      "weighted avg       0.75      0.87      0.81      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: infrastructure_related\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      7366\n",
      "           1       0.00      0.00      0.00       499\n",
      "\n",
      "    accuracy                           0.94      7865\n",
      "   macro avg       0.47      0.50      0.48      7865\n",
      "weighted avg       0.88      0.94      0.91      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: transport\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      7517\n",
      "           1       0.00      0.00      0.00       348\n",
      "\n",
      "    accuracy                           0.96      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.91      0.96      0.93      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: buildings\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7449\n",
      "           1       0.00      0.00      0.00       416\n",
      "\n",
      "    accuracy                           0.95      7865\n",
      "   macro avg       0.47      0.50      0.49      7865\n",
      "weighted avg       0.90      0.95      0.92      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: electricity\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      7710\n",
      "           1       0.00      0.00      0.00       155\n",
      "\n",
      "    accuracy                           0.98      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.96      0.98      0.97      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: tools\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      7818\n",
      "           1       0.00      0.00      0.00        47\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.50      0.50      0.50      7865\n",
      "weighted avg       0.99      0.99      0.99      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: hospitals\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      7787\n",
      "           1       0.00      0.00      0.00        78\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.50      0.50      0.50      7865\n",
      "weighted avg       0.98      0.99      0.99      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: shops\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      7831\n",
      "           1       0.00      0.00      0.00        34\n",
      "\n",
      "    accuracy                           1.00      7865\n",
      "   macro avg       0.50      0.50      0.50      7865\n",
      "weighted avg       0.99      1.00      0.99      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: aid_centers\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      7766\n",
      "           1       0.00      0.00      0.00        99\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.97      0.99      0.98      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: other_infrastructure\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      7525\n",
      "           1       0.00      0.00      0.00       340\n",
      "\n",
      "    accuracy                           0.96      7865\n",
      "   macro avg       0.48      0.50      0.49      7865\n",
      "weighted avg       0.92      0.96      0.94      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: weather_related\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      1.00      0.85      5734\n",
      "           1       0.93      0.04      0.08      2131\n",
      "\n",
      "    accuracy                           0.74      7865\n",
      "   macro avg       0.83      0.52      0.46      7865\n",
      "weighted avg       0.79      0.74      0.64      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: floods\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      7236\n",
      "           1       0.00      0.00      0.00       629\n",
      "\n",
      "    accuracy                           0.92      7865\n",
      "   macro avg       0.46      0.50      0.48      7865\n",
      "weighted avg       0.85      0.92      0.88      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: storm\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.95      7179\n",
      "           1       0.00      0.00      0.00       686\n",
      "\n",
      "    accuracy                           0.91      7865\n",
      "   macro avg       0.46      0.50      0.48      7865\n",
      "weighted avg       0.83      0.91      0.87      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: fire\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      7778\n",
      "           1       0.00      0.00      0.00        87\n",
      "\n",
      "    accuracy                           0.99      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.98      0.99      0.98      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: earthquake\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.95      7119\n",
      "           1       0.00      0.00      0.00       746\n",
      "\n",
      "    accuracy                           0.91      7865\n",
      "   macro avg       0.45      0.50      0.48      7865\n",
      "weighted avg       0.82      0.91      0.86      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: cold\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      7711\n",
      "           1       0.00      0.00      0.00       154\n",
      "\n",
      "    accuracy                           0.98      7865\n",
      "   macro avg       0.49      0.50      0.50      7865\n",
      "weighted avg       0.96      0.98      0.97      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: other_weather\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7467\n",
      "           1       0.00      0.00      0.00       398\n",
      "\n",
      "    accuracy                           0.95      7865\n",
      "   macro avg       0.47      0.50      0.49      7865\n",
      "weighted avg       0.90      0.95      0.92      7865\n",
      "\n",
      "------------------------------------------------------\n",
      "\n",
      "FEATURE: direct_report\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      1.00      0.89      6357\n",
      "           1       0.00      0.00      0.00      1508\n",
      "\n",
      "    accuracy                           0.81      7865\n",
      "   macro avg       0.40      0.50      0.45      7865\n",
      "weighted avg       0.65      0.81      0.72      7865\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# check metrics\n",
    "y_pred_pd = pd.DataFrame(y_pred, columns = y_test.columns)\n",
    "for column in y_test.columns:\n",
    "    print('------------------------------------------------------\\n')\n",
    "    print('FEATURE: {}\\n'.format(column))\n",
    "    print(classification_report(y_test[column],y_pred_pd[column]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Try improving your model further. Here are a few ideas:\n",
    "* try other machine learning algorithms\n",
    "* add other features besides the TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# skipped that, it took too much time to train new models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Export your model as a pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(pipeline, open('models/classifier.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10. Use this notebook to complete `train.py`\n",
    "Use the template file attached in the Resources folder to write a script that runs the steps above to create a database and export a model based on a new dataset specified by the user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}