02b_improved_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification w/ modified dataset\n",
    "The model selection process is similar to that of the previous notebook, however in this attempt the dataset is different. I've added more contextual features as one-hot-encoded variables, I've removed some features that were not useful in differentiating between attack and normal data, and have incorporated standard scaling for the numeric features that remain."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom modules\n",
    "import data_prep as dp\n",
    "import model_abstraction as moda\n",
    "\n",
    "# Data Structures\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "\n",
    "# Preprocessing or data manipulation methods\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "# Modeling methods and selection\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import GaussianNB, MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.svm import OneClassSVM, LinearSVC\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score,\\\n",
    "    train_test_split, StratifiedKFold\n",
    "\n",
    "# Model assessment\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score, classification_report\n",
    "\n",
    "# Visualization\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3265: DtypeWarning: Columns (1,3,47) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3265: DtypeWarning: Columns (3,39,47) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "# Load train and test data\n",
    "X_train, y_train = dp.load_agg_Xy(path='./data/', sample_size=0.35, strat_cat='label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_holdout, y_train, y_holdout = train_test_split(X_train, y_train,\n",
    "                                            test_size=0.25, stratify=y_train)\n",
    "skf = StratifiedKFold(n_splits = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the individual steps\n",
    "ohe_step = ('ohe', OneHotEncoder(sparse=False))\n",
    "ssc_step = ('std_sclr', StandardScaler())\n",
    "\n",
    "# Make the step part of a pipeline\n",
    "ohe_pipe = Pipeline([ohe_step])\n",
    "ssc_pipe = Pipeline([ssc_step])\n",
    "\n",
    "# Columns to transform: categorical columns for encoding, numeric feature columns for standardizing\n",
    "ohe_cols = ['proto', 'state', 'service']\n",
    "binary_cols = ['is_sm_ips_ports', 'is_ftp_login']\n",
    "non_ssc_cols = ohe_cols+binary_cols\n",
    "ssc_cols = [col for col in X_train.columns if col not in non_ssc_cols]\n",
    "\n",
    "# Transformer input: tuple w/ contents ('name', SomeTransformer(Parameters), columns)\n",
    "transformer = [\n",
    "    ('one_hot_encoding', ohe_pipe, ohe_cols),\n",
    "    ('standard_scaling', ssc_pipe, ssc_cols)\n",
    "]\n",
    "ct = ColumnTransformer(transformers=transformer, remainder='passthrough')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Provide classifiers to test for a \"first pass\" assessment using only vanilla models\n",
    "classifiers = {\n",
    "    'knn': KNeighborsClassifier,\n",
    "    'lgr': LogisticRegression,\n",
    "    'gnb': GaussianNB,\n",
    "    'dtc': DecisionTreeClassifier,\n",
    "    'rfc': RandomForestClassifier,\n",
    "    'gbc': GradientBoostingClassifier\n",
    "}\n",
    "\n",
    "default_parameters = {\n",
    "    'knn': {'n_neighbors':9},\n",
    "    'lgr': {'solver':'lbfgs'},\n",
    "    'gnb': {},\n",
    "    'dtc': {},\n",
    "    'rfc': {'n_estimators':100},\n",
    "    'gbc': {}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/data.py:617: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: knn Metric: roc_auc 0.9988598000117236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: lgr Metric: roc_auc 0.999077872047013\n",
      "Model: gnb Metric: roc_auc 0.9685787297973978\n",
      "Model: dtc Metric: roc_auc 0.9819305736550422\n",
      "Model: rfc Metric: roc_auc 0.9997337614871057\n",
      "Model: gbc Metric: roc_auc 0.9996180295992732\n"
     ]
    }
   ],
   "source": [
    "results = moda.cross_val_models(classifiers, ct.fit_transform(X_train), y_train,\n",
    "            use_cv = skf, params=default_parameters, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/data.py:617: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/pipeline.py:451: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  Xt = transform.transform(Xt)\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier(n_estimators=100)\n",
    "rfc.fit(ct.fit_transform(X_train), y_train)\n",
    "y_rfc_pred = rfc.predict(ct.transform(X_holdout))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC score: 0.9839734009808487\n",
      "[[193514    629]\n",
      " [   810  27302]]\n"
     ]
    }
   ],
   "source": [
    "print('AUC score:', roc_auc_score(y_holdout, y_rfc_pred))\n",
    "print(confusion_matrix(y_holdout, y_rfc_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Frankly, after the dropoff experienced in earlier classification trials this is definitely a stunning result. There are several major feature changes that make this possible:\n",
    "* addition of contextual features as one-hot-encoded columns\n",
    "* standard scaling for all features after dataset inspection revealed a need for it\n",
    "* removal of non-predictive values that were orders of magnitude greater than the other features\n",
    "* larger dataset allows for drastically more training data (667k vs 178k in provided training set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.40214146e-03, 2.09958397e-03, 1.24487037e-03, 1.41851796e-03,\n",
       "       8.72902778e-04, 7.07143030e-04, 2.78579762e-04, 3.83763564e-02,\n",
       "       7.36704348e-04, 4.98793342e-03, 2.65258742e-03, 8.22506408e-04,\n",
       "       2.62292665e-02, 1.09492877e-01, 2.47208182e-02, 1.77597419e-01,\n",
       "       2.86384338e-02, 3.17326651e-03, 1.39275324e-03, 4.02553811e-02,\n",
       "       3.00478730e-02, 3.08023216e-03, 3.48176400e-02, 1.20632998e-03,\n",
       "       6.24514049e-04, 6.74212283e-02, 4.12376669e-02, 3.35020960e-04,\n",
       "       8.09483045e-04, 5.07218755e-03, 2.73988333e-03, 9.25096116e-03,\n",
       "       2.06197471e-02, 1.38168383e-02, 1.12243882e-02, 9.92023836e-03,\n",
       "       2.19704907e-01, 3.98875795e-04, 1.88556348e-05, 1.01330347e-02,\n",
       "       9.18165318e-03, 6.01407415e-03, 4.09849256e-03, 6.41705760e-03,\n",
       "       1.14990112e-02, 1.24708515e-02, 7.04136852e-04, 3.47743366e-05])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['proto', 'state', 'dur', 'sbytes', 'dbytes', 'sttl', 'dttl', 'sloss',\n",
       "       'dloss', 'service', 'sload', 'dload', 'spkts', 'dpkts', 'swin', 'dwin',\n",
       "       'smeansz', 'dmeansz', 'trans_depth', 'res_bdy_len', 'sjit', 'djit',\n",
       "       'sintpkt', 'dintpkt', 'tcprtt', 'synack', 'ackdat', 'is_sm_ips_ports',\n",
       "       'ct_state_ttl', 'ct_flw_http_mthd', 'is_ftp_login', 'ct_ftp_cmd',\n",
       "       'ct_srv_src', 'ct_srv_dst', 'ct_dst_ltm', 'ct_src_ltm',\n",
       "       'ct_src_dport_ltm', 'ct_dst_sport_ltm', 'ct_dst_src_ltm'],\n",
       "      dtype='object', name='Name')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the list above, recall that transforming the features re-organizes the dataframe. The first 12 features are the result of one-hot-encoding. The last two are the non-scaled binary columns (both of which turn out to be relatively unimportant at 7e-4 and 3e-5, respectively). From the above, some of the most important features (importance > 0.1) are:\n",
    "* 1.09e-1: sbytes: source to destination transaction bytes\n",
    "* 1.78e-1: sload: source bits per second\n",
    "* 2.19e-1: ct_ftp_cmd: number of flows that has a command in ftp session (how many file transfer protocols are occuring)\n",
    "* 1.12e-2: ct_state_ttl: number for each state according to specific ranges for source/destination ttl values. Most non-attacks (all but 224) have 0 for this value.\n",
    "\n",
    "Further inspection below reveals that most attacks have a ct_state_ttl value >0. The values for this column represent different combinations of source and destination ttl (time-to-live, or hop limit) values for different states. From the above I feel safe in generalizing that attacks, when compared to normal traffic, transfer more data, do so at higher rates, and have more file transfer protocol commands (try to move more data)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.ct_ftp_cmd.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0032488706024639243"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[X_train.ct_state_ttl==0].value_counts()[1]/y_train.value_counts()[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Name</th>\n",
       "      <th>sttl</th>\n",
       "      <th>dttl</th>\n",
       "      <th>ct_state_ttl</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>778908</th>\n",
       "      <td>31</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>583323</th>\n",
       "      <td>31</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315764</th>\n",
       "      <td>31</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522847</th>\n",
       "      <td>31</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250240</th>\n",
       "      <td>31</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Name    sttl  dttl  ct_state_ttl\n",
       "778908    31    29             0\n",
       "583323    31    29             0\n",
       "315764    31    29             0\n",
       "522847    31    29             0\n",
       "250240    31    29             0"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[X_train.ct_state_ttl==0][['sttl', 'dttl', 'ct_state_ttl']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "254.0    1093\n",
       "dtype: int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val = 6\n",
    "(X_train[X_train.ct_state_ttl==val].sttl/(X_train[X_train.ct_state_ttl==val].dttl+1)).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1ac2f792e8>"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAEJxJREFUeJzt3X+s3fVdx/HnW7rNhol0IjekbeyMjQ6p21gDNSTmKgYKLCsmI4HgKIykujDdkiaum3/UDJewGDYlTpIqlaI4JGwLzSjrKtvNYjImZSMU1s3eYB3XVupWxihEl6tv/7ifjtPb03vfPbft93Lu85GcnHPe38/n+/3cT+65r/v9cc6JzESSpIqf6noAkqTXD0NDklRmaEiSygwNSVKZoSFJKjM0JEllhoYkqczQkCSVGRqSpLJFXQ/gVDvvvPNyxYoVA/V95ZVXOPvss0/tgF7HnI/XOBfHcj6ONQzz8eSTT34/M39+tnZDFxorVqxg9+7dA/UdGxtjdHT01A7odcz5eI1zcSzn41jDMB8R8e+Vdh6ekiSVGRqSpDJDQ5JUZmhIksoMDUlSmaEhSSozNCRJZYaGJKnM0JAklQ3dO8Kl2azY9MhJ99m4apKbB+jXa/8d18ypvzQfuKchSSozNCRJZYaGJKnM0JAklRkakqQyQ0OSVGZoSJLKDA1JUpmhIUkqMzQkSWWGhiSpzNCQJJUZGpKkMkNDklRmaEiSygwNSVKZoSFJKjM0JEllft2rtAAM8hW3M6l+/a1fcTt8DA3pDDnVf7ilLsx6eCoilkfEVyNib0Q8GxEfavW3RMSuiNjX7pe0ekTEXRExHhFPR8TFPeta39rvi4j1PfV3RcSe1ueuiIiZtiFJ6kblnMYksDEz3wasAW6LiAuBTcBjmbkSeKw9B7gKWNluG4C7YSoAgM3ApcAlwOaeELi7tT3ab22rn2gbkqQOzBoamXkwM7/ZHr8M7AWWAuuAba3ZNuDa9ngdcF9OeRw4NyIuAK4EdmXm4cx8EdgFrG3LzsnMr2dmAvdNW1e/bUiSOnBS5zQiYgXwTuAbwEhmHoSpYImI81uzpcDzPd0mWm2m+kSfOjNsY/q4NjC1p8LIyAhjY2Mn82P9xJEjRwbuO4yGdT42rpo86T4jiwfrN6yq8zGMvz/9DOtrpZ9yaETEm4HPAR/OzB+10w59m/ap5QD1sszcAmwBWL16dY6Ojp5M958YGxtj0L7DaFjno3LVz3QbV01y5x6vGzmqOh/7bxw9/YOZB4b1tdJP6X0aEfEGpgLj/sz8fCu/0A4t0e4PtfoEsLyn+zLgwCz1ZX3qM21DktSBytVTAdwD7M3MT/Us2g4cvQJqPfBwT/2mdhXVGuCldohpJ3BFRCxpJ8CvAHa2ZS9HxJq2rZumravfNiRJHajsb18GvA/YExFPtdrHgDuAByPiVuB7wHVt2Q7gamAceBW4BSAzD0fE7cATrd3HM/Nwe/wB4F5gMfBouzHDNiRJHZg1NDLzn+l/3gHg8j7tE7jtBOvaCmztU98NXNSn/oN+25AkdcPPnpIklRkakqQyQ0OSVGZoSJLKDA1JUpmhIUkqMzQkSWWGhiSpzNCQJJUZGpKkMkNDklRmaEiSygwNSVKZoSFJKjM0JEllhoYkqczQkCSVGRqSpDJDQ5JUZmhIksoMDUlSmaEhSSozNCRJZYaGJKnM0JAklRkakqQyQ0OSVGZoSJLKDA1JUpmhIUkqMzQkSWWGhiSpzNCQJJUZGpKkMkNDklRmaEiSymYNjYjYGhGHIuKZntqfRMR/RMRT7XZ1z7KPRsR4RHw3Iq7sqa9ttfGI2NRTf2tEfCMi9kXEP0bEG1v9Te35eFu+4lT90JKkwVT2NO4F1vapfzoz39FuOwAi4kLgeuBXW5+/ioizIuIs4DPAVcCFwA2tLcAn27pWAi8Ct7b6rcCLmflLwKdbO0lSh2YNjcz8GnC4uL51wAOZ+T+Z+W/AOHBJu41n5nOZ+WPgAWBdRATwW8BDrf824NqedW1rjx8CLm/tJUkdmcs5jQ9GxNPt8NWSVlsKPN/TZqLVTlT/OeCHmTk5rX7Mutryl1p7SVJHFg3Y727gdiDb/Z3A+4F+ewJJ/3DKGdozy7JjRMQGYAPAyMgIY2NjMwz9xI4cOTJw32E0rPOxcdXk7I2mGVk8WL9hVZ2PYfz96WdYXyv9DBQamfnC0ccR8dfAF9vTCWB5T9NlwIH2uF/9+8C5EbGo7U30tj+6romIWAT8LCc4TJaZW4AtAKtXr87R0dFBfizGxsYYtO8wGtb5uHnTIyfdZ+OqSe7cM+j/WMOnOh/7bxw9/YOZB4b1tdLPQIenIuKCnqe/Axy9smo7cH278umtwErgX4AngJXtSqk3MnWyfHtmJvBV4L2t/3rg4Z51rW+P3wt8pbWXJHVk1n8VIuKzwChwXkRMAJuB0Yh4B1OHi/YDvweQmc9GxIPAt4FJ4LbM/N+2ng8CO4GzgK2Z+WzbxEeAByLiT4FvAfe0+j3A30XEOFN7GNfP+aeVJM3JrKGRmTf0Kd/Tp3a0/SeAT/Sp7wB29Kk/x9TVVdPr/w1cN9v4JElnju8IlySVGRqSpDJDQ5JUZmhIksoMDUlSmaEhSSozNCRJZYaGJKnM0JAklRkakqQyQ0OSVGZoSJLKDA1JUpmhIUkqMzQkSWWGhiSpzNCQJJUZGpKkMkNDklRmaEiSygwNSVKZoSFJKjM0JEllhoYkqczQkCSVGRqSpDJDQ5JUZmhIksoMDUlSmaEhSSozNCRJZYaGJKnM0JAklRkakqQyQ0OSVGZoSJLKDA1JUtmsoRERWyPiUEQ801N7S0Tsioh97X5Jq0dE3BUR4xHxdERc3NNnfWu/LyLW99TfFRF7Wp+7IiJm2oYkqTuVPY17gbXTapuAxzJzJfBYew5wFbCy3TYAd8NUAACbgUuBS4DNPSFwd2t7tN/aWbYhSerIrKGRmV8DDk8rrwO2tcfbgGt76vfllMeBcyPiAuBKYFdmHs7MF4FdwNq27JzM/HpmJnDftHX124YkqSOLBuw3kpkHATLzYESc3+pLged72k202kz1iT71mbZxnIjYwNTeCiMjI4yNjQ30Qx05cmTgvsNoWOdj46rJk+4zsniwfsOqOh/D+PvTz7C+VvoZNDROJPrUcoD6ScnMLcAWgNWrV+fo6OjJrgKY+gUftO8wGtb5uHnTIyfdZ+OqSe7cc6pfLq9f1fnYf+Po6R/MPDCsr5V+Br166oV2aIl2f6jVJ4DlPe2WAQdmqS/rU59pG5KkjgwaGtuBo1dArQce7qnf1K6iWgO81A4x7QSuiIgl7QT4FcDOtuzliFjTrpq6adq6+m1DktSRWfcvI+KzwChwXkRMMHUV1B3AgxFxK/A94LrWfAdwNTAOvArcApCZhyPiduCJ1u7jmXn05PoHmLpCazHwaLsxwzYkSR2ZNTQy84YTLLq8T9sEbjvBerYCW/vUdwMX9an/oN82JEnd8R3hkqQyQ0OSVGZoSJLKDA1JUpmhIUkqMzQkSWWGhiSpzNCQJJUZGpKkMkNDklRmaEiSygwNSVKZoSFJKjM0JEllhoYkqczQkCSVGRqSpDJDQ5JUZmhIksoMDUlSmaEhSSozNCRJZYaGJKnM0JAklRkakqQyQ0OSVGZoSJLKDA1JUpmhIUkqMzQkSWWGhiSpzNCQJJUZGpKkMkNDklRmaEiSygwNSVLZnEIjIvZHxJ6IeCoidrfaWyJiV0Tsa/dLWj0i4q6IGI+IpyPi4p71rG/t90XE+p76u9r6x1vfmMt4JUlzcyr2NH4zM9+Rmavb803AY5m5EnisPQe4CljZbhuAu2EqZIDNwKXAJcDmo0HT2mzo6bf2FIxXkjSg03F4ah2wrT3eBlzbU78vpzwOnBsRFwBXArsy83BmvgjsAta2Zedk5tczM4H7etYlSerAXEMjgS9HxJMRsaHVRjLzIEC7P7/VlwLP9/SdaLWZ6hN96pKkjiyaY//LMvNARJwP7IqI78zQtt/5iBygfvyKpwJrA8DIyAhjY2MzDvpEjhw5MnDfYTSs87Fx1eRJ9xlZPFi/YVWdj2H8/elnWF8r/cwpNDLzQLs/FBFfYOqcxAsRcUFmHmyHmA615hPA8p7uy4ADrT46rT7W6sv6tO83ji3AFoDVq1fn6Ohov2azGhsbY9C+w2hY5+PmTY+cdJ+Nqya5c89c/8caHtX52H/j6OkfzDwwrK+VfgY+PBURZ0fEzxx9DFwBPANsB45eAbUeeLg93g7c1K6iWgO81A5f7QSuiIgl7QT4FcDOtuzliFjTrpq6qWddkqQOzOVfpxHgC+0q2EXAP2TmlyLiCeDBiLgV+B5wXWu/A7gaGAdeBW4ByMzDEXE78ERr9/HMPNwefwC4F1gMPNpukqSODBwamfkc8PY+9R8Al/epJ3DbCda1Fdjap74buGjQMUqSTi3fES5JKjM0JEllhoYkqczQkCSVGRqSpDJDQ5JUZmhIksoMDUlSmaEhSSozNCRJZYaGJKnM0JAklRkakqQyQ0OSVOZXkUk6bVYM8C2Jp8r+O67pbNvDzD0NSVKZoSFJKjM0JEllhoYkqczQkCSVGRqSpDJDQ5JUZmhIksoMDUlSmaEhSSozNCRJZYaGJKnM0JAklRkakqQyQ0OSVGZoSJLKDA1JUpmhIUkqMzQkSWWGhiSpzNCQJJUZGpKksnkfGhGxNiK+GxHjEbGp6/FI0kI2r0MjIs4CPgNcBVwI3BARF3Y7KklauOZ1aACXAOOZ+Vxm/hh4AFjX8ZgkacFa1PUAZrEUeL7n+QRwaUdjkfQ6smLTI2dsWxtXTXJz297+O645Y9vtwnwPjehTy+MaRWwANrSnRyLiuwNu7zzg+wP2HUbOR/OHzsUxnI9j9c5HfLLjwQzuFyqN5ntoTADLe54vAw5Mb5SZW4Atc91YROzOzNVzXc+wcD5e41wcy/k41kKaj/l+TuMJYGVEvDUi3ghcD2zveEyStGDN6z2NzJyMiA8CO4GzgK2Z+WzHw5KkBWtehwZAZu4Adpyhzc35ENeQcT5e41wcy/k41oKZj8g87ryyJEl9zfdzGpKkecTQmCYi/iwivhMRT0fEFyLi3K7HdKb50S2viYjlEfHViNgbEc9GxIe6HlPXIuKsiPhWRHyx67F0LSLOjYiH2t+MvRHx612P6XQzNI63C7goM38N+Ffgox2P54zyo1uOMwlszMy3AWuA2xb4fAB8CNjb9SDmib8AvpSZvwK8nQUwL4bGNJn55cycbE8fZ+q9IQuJH93SIzMPZuY32+OXmfqjsLTbUXUnIpYB1wB/0/VYuhYR5wC/AdwDkJk/zswfdjuq08/QmNn7gUe7HsQZ1u+jWxbsH8leEbECeCfwjW5H0qk/B/4I+L+uBzIP/CLwX8DftsN1fxMRZ3c9qNNtQYZGRPxTRDzT57aup80fM3Vo4v7uRtqJ0ke3LDQR8Wbgc8CHM/NHXY+nCxHxbuBQZj7Z9VjmiUXAxcDdmflO4BVg6M8Bzvv3aZwOmfnbMy2PiPXAu4HLc+Fdk1z66JaFJCLewFRg3J+Zn+96PB26DHhPRFwN/DRwTkT8fWb+bsfj6soEMJGZR/c8H2IBhMaC3NOYSUSsBT4CvCczX+16PB3wo1t6REQwdcx6b2Z+quvxdCkzP5qZyzJzBVO/F19ZwIFBZv4n8HxE/HIrXQ58u8MhnRELck9jFn8JvAnYNfX3gscz8/e7HdKZ40e3HOcy4H3Anoh4qtU+1j6pQPoD4P72D9ZzwC0dj+e08x3hkqQyD09JksoMDUlSmaEhSSozNCRJZYaGJKnM0JAklRkakqQyQ0OSVPb/n+fea7evoGkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "(X_train.sbytes+0.01).apply(np.log10).hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi-class Classification\n",
    "As a "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3265: DtypeWarning: Columns (1,3,47) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3265: DtypeWarning: Columns (3,39,47) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "# Load train and test data\n",
    "X_train_mc, y_train_mc = dp.load_agg_Xy(path='./data/', sample_size=0.35, strat_cat='attack_cat')\n",
    "\n",
    "# Creating numeric class labels and reserving holdout\n",
    "y_train_mc = y_train_mc.apply(lambda x:dp.attack_to_num[x])\n",
    "\n",
    "X_train_mc, X_holdout_mc, y_train_mc, y_holdout_mc = train_test_split(X_train_mc, y_train_mc,\n",
    "                                            test_size=0.25, stratify=y_train_mc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next two blocks of code quickly try out several classifiers in an attempt to identify "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Provide classifiers to test for a \"first pass\" assessment using only vanilla models\n",
    "mc_classifiers = {\n",
    "    'knn': KNeighborsClassifier,\n",
    "    'gnb': GaussianNB,\n",
    "    'dtc': DecisionTreeClassifier,\n",
    "    'rfc': RandomForestClassifier,\n",
    "    'gbc': GradientBoostingClassifier\n",
    "}\n",
    "\n",
    "mc_parameters = {\n",
    "    'knn': {'n_neighbors':9},\n",
    "    'gnb': {},\n",
    "    'dtc': {'random_state':0},\n",
    "    'rfc': {'random_state':0,'n_estimators':100},\n",
    "    'gbc': {'random_state':0}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/data.py:617: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: knn Metric: f1_weighted 0.9736719260079921\n",
      "Model: gnb Metric: f1_weighted 0.8763110028814388\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: dtc Metric: f1_weighted 0.9771683628563779\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: rfc Metric: f1_weighted 0.9789895184832463\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: gbc Metric: f1_weighted 0.9754421421117933\n"
     ]
    }
   ],
   "source": [
    "scores_mc = moda.cross_val_models(mc_classifiers,\n",
    "                                  ct.fit_transform(X_train_mc),\n",
    "                                  y_train_mc,use_cv=skf,\n",
    "                                  params=mc_parameters,\n",
    "                                  metric='f1_weighted',\n",
    "                                 verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/data.py:617: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=150, n_jobs=None,\n",
       "            oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train Random Forest Classifier for a multiclass problem\n",
    "rfc = RandomForestClassifier(n_estimators=150, random_state=0)\n",
    "rfc.fit(ct.fit_transform(X_train_mc),y_train_mc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/data.py:617: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    }
   ],
   "source": [
    "# Indexes: true, Columns: predicted\n",
    "pred_holdout_mc = confusion_matrix(y_holdout_mc, rfc.predict(ct.fit_transform(X_holdout_mc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>normal</th>\n",
       "      <th>analysis</th>\n",
       "      <th>backdoors</th>\n",
       "      <th>dos</th>\n",
       "      <th>exploits</th>\n",
       "      <th>fuzzers</th>\n",
       "      <th>generic</th>\n",
       "      <th>reconnaissance</th>\n",
       "      <th>shellcode</th>\n",
       "      <th>worms</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>normal</th>\n",
       "      <th>0</th>\n",
       "      <td>193596</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>368</td>\n",
       "      <td>168</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analysis</th>\n",
       "      <th>1</th>\n",
       "      <td>42</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>136</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>backdoors</th>\n",
       "      <th>2</th>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>33</td>\n",
       "      <td>113</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dos</th>\n",
       "      <th>3</th>\n",
       "      <td>262</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>147</td>\n",
       "      <td>973</td>\n",
       "      <td>15</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>exploits</th>\n",
       "      <th>4</th>\n",
       "      <td>446</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>202</td>\n",
       "      <td>3037</td>\n",
       "      <td>36</td>\n",
       "      <td>163</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fuzzers</th>\n",
       "      <th>5</th>\n",
       "      <td>1326</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>184</td>\n",
       "      <td>554</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>generic</th>\n",
       "      <th>6</th>\n",
       "      <td>63</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>289</td>\n",
       "      <td>11</td>\n",
       "      <td>18464</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reconnaissance</th>\n",
       "      <th>7</th>\n",
       "      <td>484</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>285</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>429</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shellcode</th>\n",
       "      <th>8</th>\n",
       "      <td>67</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>44</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worms</th>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  normal analysis backdoors  dos exploits fuzzers generic  \\\n",
       "                       0        1         2    3        4       5       6   \n",
       "normal         0  193596        4         1    2      368     168       2   \n",
       "analysis       1      42        3         0   39      136      14       0   \n",
       "backdoors      2      36        0         8   33      113      13       1   \n",
       "dos            3     262        0         0  147      973      15      28   \n",
       "exploits       4     446        0         3  202     3037      36     163   \n",
       "fuzzers        5    1326        0         1   54      184     554       0   \n",
       "generic        6      63        2         0   21      289      11   18464   \n",
       "reconnaissance 7     484        0         0   21      285       4       1   \n",
       "shellcode      8      67        0         0    1       17       0       0   \n",
       "worms          9       0        0         0    0       15       0       0   \n",
       "\n",
       "                 reconnaissance shellcode worms  \n",
       "                              7         8     9  \n",
       "normal         0              0         2     0  \n",
       "analysis       1              0         0     0  \n",
       "backdoors      2              0         0     0  \n",
       "dos            3              3         3     0  \n",
       "exploits       4              7         2     0  \n",
       "fuzzers        5              2         0     0  \n",
       "generic        6              2         3     0  \n",
       "reconnaissance 7            429         0     0  \n",
       "shellcode      8              3        44     0  \n",
       "worms          9              0         0     0  "
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat = pd.DataFrame(pred_holdout_mc, columns = dp.attack_to_num.items(), index=dp.attack_to_num.items())\n",
    "conf_mat.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>normal</th>\n",
       "      <th>analysis</th>\n",
       "      <th>backdoors</th>\n",
       "      <th>dos</th>\n",
       "      <th>exploits</th>\n",
       "      <th>fuzzers</th>\n",
       "      <th>generic</th>\n",
       "      <th>reconnaissance</th>\n",
       "      <th>shellcode</th>\n",
       "      <th>worms</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>normal</th>\n",
       "      <th>0</th>\n",
       "      <td>0.997</td>\n",
       "      <td>0.021</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.087</td>\n",
       "      <td>0.078</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analysis</th>\n",
       "      <th>1</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.023</td>\n",
       "      <td>0.036</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>backdoors</th>\n",
       "      <th>2</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.044</td>\n",
       "      <td>0.022</td>\n",
       "      <td>0.029</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dos</th>\n",
       "      <th>3</th>\n",
       "      <td>0.001</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.123</td>\n",
       "      <td>0.251</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.030</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>exploits</th>\n",
       "      <th>4</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0.167</td>\n",
       "      <td>0.778</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.030</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fuzzers</th>\n",
       "      <th>5</th>\n",
       "      <td>0.007</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.041</td>\n",
       "      <td>0.044</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>generic</th>\n",
       "      <th>6</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.030</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reconnaissance</th>\n",
       "      <th>7</th>\n",
       "      <td>0.003</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.020</td>\n",
       "      <td>0.073</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.337</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shellcode</th>\n",
       "      <th>8</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.311</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worms</th>\n",
       "      <th>9</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 normal analysis backdoors    dos exploits fuzzers generic  \\\n",
       "                      0        1         2      3        4       5       6   \n",
       "normal         0  0.997    0.021     0.005  0.001    0.087   0.078   0.000   \n",
       "analysis       1  0.000    0.017     0.000  0.023    0.036   0.007   0.000   \n",
       "backdoors      2  0.000    0.000     0.044  0.022    0.029   0.006   0.000   \n",
       "dos            3  0.001    0.000     0.000  0.123    0.251   0.007   0.001   \n",
       "exploits       4  0.002    0.000     0.010  0.167    0.778   0.016   0.008   \n",
       "fuzzers        5  0.007    0.000     0.005  0.041    0.044   0.250   0.000   \n",
       "generic        6  0.000    0.017     0.000  0.017    0.069   0.004   0.979   \n",
       "reconnaissance 7  0.003    0.000     0.000  0.020    0.073   0.002   0.000   \n",
       "shellcode      8  0.000    0.000     0.000  0.003    0.003   0.000   0.000   \n",
       "worms          9  0.000    0.000     0.000  0.000    0.004   0.000   0.000   \n",
       "\n",
       "                 reconnaissance shellcode worms  \n",
       "                              7         8     9  \n",
       "normal         0          0.000     0.015   0.0  \n",
       "analysis       1          0.000     0.000   0.0  \n",
       "backdoors      2          0.000     0.000   0.0  \n",
       "dos            3          0.002     0.030   0.0  \n",
       "exploits       4          0.007     0.030   0.0  \n",
       "fuzzers        5          0.001     0.008   0.0  \n",
       "generic        6          0.001     0.030   0.0  \n",
       "reconnaissance 7          0.337     0.000   0.0  \n",
       "shellcode      8          0.002     0.311   0.0  \n",
       "worms          9          0.000     0.000   0.0  "
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# I want to take each column and divide it, element-wise, by the value counts for that cyberattack in the dataset.\n",
    "# Each row of the resulting dataframe should sum to one. Below does not accomplish this.\n",
    "(conf_mat / conf_mat.sum(axis=1)).round(decimals=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### conf_mat.normal.divide(conf_mat.sum(axis=1), level=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}