gussianNB_classifier_on_Iris_dataset_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load iris Dataset, Perform Train Validation Split And Convert It To Pandas Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.5</td>\n",
       "      <td>2.3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.7</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>1.8</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.6</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "0                5.5               2.3                4.0               1.3   \n",
       "1                5.1               3.7                1.5               0.4   \n",
       "2                6.5               3.0                5.5               1.8   \n",
       "3                5.0               3.5                1.6               0.6   \n",
       "4                6.2               3.4                5.4               2.3   \n",
       "\n",
       "   target  \n",
       "0       1  \n",
       "1       0  \n",
       "2       2  \n",
       "3       0  \n",
       "4       2  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data as numpy array\n",
    "data = load_iris()\n",
    "# train test split\n",
    "df, df_test, target, target_test = train_test_split(data.data, data.target, test_size= 0.2, random_state= 50, stratify= data.target)\n",
    "# convert to dataframe\n",
    "df = pd.DataFrame(df, columns=data.feature_names)\n",
    "df_test = pd.DataFrame(df_test, columns=data.feature_names)\n",
    "# add target column\n",
    "df[\"target\"] = target\n",
    "df_test[\"target\"] = target_test\n",
    "#number of rows\n",
    "n_rows = df.shape[0]\n",
    "# list of possible class\n",
    "class_list = df.target.unique()\n",
    "# record column sequence\n",
    "column_seq = df.columns\n",
    "# sneak peak\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select only continuous data\n",
    "df_continuous = df.select_dtypes(np.float)\n",
    "\n",
    "# select only discrete data\n",
    "df_discrete = df.select_dtypes([int, object]).drop([\"target\"], axis= 1)\n",
    "\n",
    "# list of categorical columns\n",
    "discrete_column_list = df_discrete.columns\n",
    "\n",
    "# list of continuous columns\n",
    "continuous_column_list = df_continuous.columns\n",
    "\n",
    "# add target column\n",
    "df_continuous[\"target\"] = df.target\n",
    "df_discrete[\"target\"] = df.target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply additive smoothing to prevent probability getting 0\n",
    "def apply_additive_smoothing(nume, deno, alpha, n_features):\n",
    "    \"\"\"\n",
    "    Additive smoothing to get around the problem of zero probability\n",
    "    \"\"\"\n",
    "    return (nume + alpha) / (deno + (alpha * n_features))\n",
    "\n",
    "# applying log to probabilities makes computation easy (and transforms multiplication into addition)\n",
    "def apply_log_probability(smoothing_proba):\n",
    "    \"\"\"\n",
    "    Applying log to transforms multiplication into addition for numerical stability\n",
    "    \"\"\"\n",
    "    return np.log(smoothing_proba)\n",
    "\n",
    "# guassian naive bayes for continuous features\n",
    "def compute_mean_std(df_continuous, continuous_column_list):\n",
    "    \"\"\"\n",
    "    This function computes mean and standard deviation of continuous features per class\n",
    "    \"\"\"\n",
    "    \n",
    "    # compute mean and standard deviation (gaussian distribution parameters) per class feature wise\n",
    "    class_feature_pair_mean = df_continuous.groupby(\"target\").agg(\"mean\").reset_index()\n",
    "    class_feature_pair_std = df_continuous.groupby(\"target\").agg(\"std\").reset_index()\n",
    "    \n",
    "    return class_feature_pair_mean, class_feature_pair_std\n",
    "\n",
    "# get the likelihood\n",
    "def get_likelihood(x, class_, feature_name, class_feature_pair_mean, class_feature_pair_std):\n",
    "    \"\"\"\n",
    "    Get the likelihood based on the class-feature pair (per class feature wise)\n",
    "    \"\"\"\n",
    "    # get the mean for current class and current feature\n",
    "    mean = class_feature_pair_mean[(class_feature_pair_mean.target == class_)][feature_name].values[0]\n",
    "    \n",
    "    # get the std for current class and current feature\n",
    "    std = class_feature_pair_std[(class_feature_pair_std.target == class_)][feature_name].values[0]\n",
    "    \n",
    "    # get the likelihood\n",
    "    likelihood = (1/(std * np.sqrt(2 * 3.14))) * (pow(np.e, -pow(x - mean, 2)/ (2 * pow(std, 2))))\n",
    "    \n",
    "    return likelihood\n",
    "\n",
    "def predict(query_point, feature_count, class_feature_pair_mean, class_feature_pair_std):\n",
    "    \"\"\"\n",
    "    Get the prediction, process each feature differently based on the data type.\n",
    "    For non-continuous feature: compute class-feature based probability, apply additive smoothing and log transformation\n",
    "    For continuous feature: compute likelihood and log transformation the likelihood\n",
    "    \"\"\"\n",
    "    # holds log of probabilities for each class\n",
    "    log_proba_class = {}\n",
    "    n_features = query_point.shape[0]\n",
    "    \n",
    "    # for each class-feature pair calculate log of probabilities (or likelihood) and sum them up to get final evidence value\n",
    "    for class_ in class_list:\n",
    "        # initialize probability sum\n",
    "        log_proba = 0\n",
    "        \"\"\"\n",
    "        For each feature, check the data type (continuous or not) and process as needed\n",
    "        \"\"\"\n",
    "        for col_idx, feature in enumerate(query_point):\n",
    "            \n",
    "            # get the name of current column\n",
    "            column_name = column_seq[col_idx]\n",
    "            \n",
    "            # if a feature is continuous, get the likelihood probability\n",
    "            if column_name in continuous_column_list:\n",
    "                # get the likelihood of corresponding feature\n",
    "                likelihood = get_likelihood(feature, class_, column_name, class_feature_pair_mean, class_feature_pair_std)\n",
    "                \n",
    "                # apply log transformation so we don't have to multiply probabilities\n",
    "                log_proba += apply_log_probability(likelihood)\n",
    "                # don't go further\n",
    "                continue\n",
    "                \n",
    "                # this is how we treat discrete feature value\n",
    "                log_proba += apply_log_probability(apply_additive_smoothing(feature_count[class_][feature],\\\n",
    "                                                                        class_count[class_], alpha, n_features))\n",
    "        # lastly add log of prior probability\n",
    "        log_proba += apply_log_probability(apply_additive_smoothing(class_count[class_],\\\n",
    "                                                                    n_rows, alpha, n_features))\n",
    "        \n",
    "        log_proba_class[class_] = log_proba\n",
    "    return log_proba_class\n",
    "\n",
    "# compute accuracy\n",
    "def accuracy(y, y_hat):\n",
    "    \"\"\"\n",
    "    Compute what percentage of data our model got it right\n",
    "    \"\"\"\n",
    "    return accuracy_score(y, y_hat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Phase\n",
    "**For non-continuous features**, we store per class-feature count (and compute probabilities) ie\n",
    "<img src= \"https://miro.medium.com/max/700/1*f09RsCQWhgivap4Smz8QHQ.png\"/>\n",
    "\n",
    "**For continuous features**, we compute statistics based on the distribution. If we choose guassian distribution (guassian naive bayes) we compute mean and standard deviation class-feature wise, which later will be used to compute likelihood for query point,\n",
    "ie\n",
    "\n",
    "Train data: computing statistics\n",
    "<img src= \"https://miro.medium.com/max/700/1*1xTN6yH_hn0zJNgBsNi5NA.png\"/>\n",
    "\n",
    "Test data example:\n",
    "<img src= \"https://miro.medium.com/max/700/1*XOhR0Yud51i7hBrby9911Q.png\"/>\n",
    "\n",
    "Test data: computing likelihood based on the statistics computed on train data\n",
    "<img src= \"https://miro.medium.com/max/700/1*N7bRyi_yYFEBO1rXnMIxKg.png\"/>\n",
    "\n",
    "[Use Naive Bayes Algorithm for Categorical and Numerical data classification](https://medium.com/analytics-vidhya/use-naive-bayes-algorithm-for-categorical-and-numerical-data-classification-935d90ab273f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_feature_pair_count = defaultdict(lambda: defaultdict(int)) # stores class- feature pair count\n",
    "class_count = defaultdict(int) # store frequency of class globally\n",
    "class_proba = defaultdict(int) # prior probability of class (class count / number of rows)\n",
    "class_feature_wise_mean = None\n",
    "class_feature_wise_std = None\n",
    "alpha = 1\n",
    "\n",
    "'''\n",
    "Initally proba_dict will store class count, feature pair count, then turned into probabilities.\n",
    "'''\n",
    "if not df_discrete.empty:\n",
    "    # This is how we will process discrete data\n",
    "    \n",
    "    # for each row, count the occurrence of feature value w.r.t class\n",
    "    for row in df_discrete.values:\n",
    "        class_ = row[-1] # get the label\n",
    "        # iterate through each feature value in a row\n",
    "        for col in (row[:-1]):\n",
    "            class_feature_pair_count[class_][col] += 1 # increment the occurence of a feature in target class\n",
    "\n",
    "for target in df.target:\n",
    "    n_class = len(class_count) # store number of classes\n",
    "    class_count[target] += 1\n",
    "\n",
    "# turn class_count into probabilities\n",
    "for class_, count in class_count.items():\n",
    "    class_proba[class_] = count / n_rows # convert counts to probabilities\n",
    "\n",
    "if not df_continuous.empty:\n",
    "    # This is how we will process continuous data\n",
    "    class_feature_pair_mean = df_continuous.groupby(\"target\").agg(\"mean\").reset_index()\n",
    "    class_feature_pair_std = df_continuous.groupby(\"target\").agg(\"std\").reset_index()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hold predictions by our model\n",
    "y_hat_list = []\n",
    "\n",
    "# for each row in the dataset\n",
    "for row in df_test.values:\n",
    "    \n",
    "    # parse the target\n",
    "    target = row[-1]\n",
    "    \n",
    "    # query point\n",
    "    query = row[:-1]\n",
    "    \n",
    "    # get the log probability distribution accross classes\n",
    "    prob_dist = predict(query, class_feature_pair_count, class_feature_pair_mean, class_feature_pair_std)\n",
    "\n",
    "    # maximum a posteriori to get the class index\n",
    "    pred_class_idx =np.argmax(list(prob_dist.values()))\n",
    "\n",
    "    # get the class label\n",
    "    pred_class_label = list(class_feature_pair_count.keys())[pred_class_idx]\n",
    "    \n",
    "    # append the prediction to the list\n",
    "    y_hat_list.append(pred_class_label)\n",
    "\n",
    "y_hat_list = np.array(y_hat_list, dtype= int)\n",
    "# compute accuracy\n",
    "test_accuracy = accuracy(df_test.target, y_hat_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 Test labels: [0 2 1 1 2 2 2 1 0 0]\r\n",
      "Corresponding predicted labels: [0 2 1 1 2 2 2 1 0 0]\n",
      "Test Accuracy: 0.9666666666666667\n"
     ]
    }
   ],
   "source": [
    "print(f\"10 Test labels: {df_test.target[:10].values}\\r\\nCorresponding predicted labels: {y_hat_list[:10]}\")\n",
    "print(f\"Test Accuracy: {test_accuracy}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}