Crime_features.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Crime Prediction"
      ],
      "metadata": {
        "id": "Ckum4fGs__bD"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Analysis\n",
        "- crime prediction\n",
        "- find features that affect the seriousness of crimes\n",
        "\n",
        "- feature preprocessing \n",
        "  - data preprocessing\n",
        "    - fix cleaning code errors\n",
        "  - dimension reduction \n",
        "  - feature selection \n",
        "  - feature transformation\n",
        "- modeling \n",
        "  - try different models (3-4)\n",
        "  - experiments: tune parameters\n",
        "  - compare results\n",
        "- feature monitoring: feature importance evalution, find common features that have high feature importance\n",
        "\n"
      ],
      "metadata": {
        "id": "f3llgipCCSMy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load Dataset"
      ],
      "metadata": {
        "id": "nJSQAymtGFic"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from datetime import datetime"
      ],
      "metadata": {
        "id": "obe6t2qu57rG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vYlpfSnx5qeu",
        "outputId": "0e371a7a-005a-425a-ab67-d78f19ce809a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n",
            "'Crime Complaints In Different Precincts Per area.jpg'\n",
            " crime_features.csv\n",
            " crime_preprocessed.csv\n",
            "'Crimes in different Boroughs.jpg'\n",
            "'Crime types.jpg'\n",
            "'Crime Type Specific.jpg'\n",
            "'Jurisdiction responsible for incident.jpg'\n",
            "'Level of offense.jpg'\n",
            "'Name of NYCHA housing development of occurrence.jpg'\n",
            " NYPD_Complaint_Data_Current__Year_To_Date_.csv\n",
            "'patrol borough in which the incident occurred.jpg'\n",
            "'Police Precincts.geojson'\n",
            "'Specific location of occurrence in or around the premises.jpg'\n",
            "'whether crime was successfully completed or attempted.jpg'\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')\n",
        "import os\n",
        "# TODO: Put your dataset in your google drive!\n",
        "os.chdir('/content/drive/MyDrive/Crimes/')\n",
        "!ls"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_path = \"/content/drive/MyDrive/Crimes/crime_preprocessed.csv\"\n",
        "df = pd.read_csv(data_path)\n",
        "field_names = list(df.columns)"
      ],
      "metadata": {
        "id": "7ChL5X4Q5vBV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Drop rows with no Level of offense \n",
        "df = df[df['LAW_CAT_CD'].notna()]"
      ],
      "metadata": {
        "id": "ht49B0vjNoJe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (df.columns)\n",
        "print (df.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4FjgyfVLBmw9",
        "outputId": "485bab35-5dac-4875-aee6-456656dac831"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Index(['Unnamed: 0', 'ADDR_PCT_CD', 'BORO_NM', 'CRM_ATPT_CPTD_CD',\n",
            "       'HADEVELOPT', 'HOUSING_PSA', 'JURIS_DESC', 'LAW_CAT_CD',\n",
            "       'LOC_OF_OCCUR_DESC', 'OFNS_DESC', 'PARKS_NM', 'PATROL_BORO', 'PD_CD',\n",
            "       'PD_DESC', 'PREM_TYP_DESC', 'RPT_DT', 'STATION_NAME', 'SUSP_AGE_GROUP',\n",
            "       'SUSP_RACE', 'SUSP_SEX', 'TRANSIT_DISTRICT', 'VIC_AGE_GROUP',\n",
            "       'VIC_RACE', 'VIC_SEX', 'X_COORD_CD', 'Y_COORD_CD', 'Latitude',\n",
            "       'Longitude', 'Lat_Lon', 'New Georeferenced Column', 'FR_TIME',\n",
            "       'TO_TIME', 'duration'],\n",
            "      dtype='object')\n",
            "(449506, 33)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Change categorical features into numerical"
      ],
      "metadata": {
        "id": "oufoOit04ZU0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Drop Unrelated columns \n",
        "Keeping those features which we think are important and start further processing"
      ],
      "metadata": {
        "id": "W3S-b6brPff7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_final = df[['ADDR_PCT_CD', 'JURIS_DESC', 'LOC_OF_OCCUR_DESC', \n",
        "               'OFNS_DESC', 'PREM_TYP_DESC', 'SUSP_AGE_GROUP','SUSP_RACE',\n",
        "               'SUSP_SEX', 'VIC_AGE_GROUP','VIC_RACE','VIC_SEX', 'LAW_CAT_CD',\n",
        "               'FR_TIME','duration']]\n",
        "df_final.dtypes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IYCNwfvIPoHW",
        "outputId": "cf53e105-e4d3-444c-8ea8-551344626339"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ADDR_PCT_CD           int64\n",
              "JURIS_DESC           object\n",
              "LOC_OF_OCCUR_DESC    object\n",
              "OFNS_DESC            object\n",
              "PREM_TYP_DESC        object\n",
              "SUSP_AGE_GROUP       object\n",
              "SUSP_RACE            object\n",
              "SUSP_SEX             object\n",
              "VIC_AGE_GROUP        object\n",
              "VIC_RACE             object\n",
              "VIC_SEX              object\n",
              "LAW_CAT_CD           object\n",
              "FR_TIME              object\n",
              "duration             object\n",
              "dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 340
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### OHE JURISDICTION Description"
      ],
      "metadata": {
        "id": "ri-hkxza9ZmP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_final['JURIS_DESC'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GRvDVXfO9k3W",
        "outputId": "91b78079-9c6e-4bce-975b-de7f66fcc946"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "N.Y. POLICE DEPT                405043\n",
              "N.Y. HOUSING POLICE              32258\n",
              "N.Y. TRANSIT POLICE               8966\n",
              "PORT AUTHORITY                    1473\n",
              "OTHER                              881\n",
              "DEPT OF CORRECTIONS                270\n",
              "NYC PARKS                          198\n",
              "HEALTH & HOSP CORP                 192\n",
              "TRI-BORO BRDG TUNNL                101\n",
              "N.Y. STATE POLICE                   44\n",
              "METRO NORTH                         20\n",
              "LONG ISLAND RAILRD                  12\n",
              "NEW YORK CITY SHERIFF OFFICE        11\n",
              "U.S. PARK POLICE                    10\n",
              "AMTRACK                              8\n",
              "N.Y. STATE PARKS                     8\n",
              "STATN IS RAPID TRANS                 6\n",
              "NYS DEPT TAX AND FINANCE             5\n",
              "Name: JURIS_DESC, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 341
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Since most of the complaints are in top 5 institutes, only going to encode it into numbers. "
      ],
      "metadata": {
        "id": "d3nifl4FQgi3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# import labelencoder\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "# instantiate labelencoder object\n",
        "le = LabelEncoder()\n",
        "\n",
        "# apply le on categorical feature columns\n",
        "df_final['JURIS_DESC'] = le.fit_transform(df_final['JURIS_DESC'])\n",
        "df_final['JURIS_DESC'].head(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KxB-z13SQswl",
        "outputId": "32b31a2b-6fe6-45eb-c53e-760c00eb9c4e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    6\n",
              "1    6\n",
              "2    6\n",
              "3    6\n",
              "4    6\n",
              "5    6\n",
              "6    6\n",
              "7    6\n",
              "8    6\n",
              "9    6\n",
              "Name: JURIS_DESC, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 342
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Three digit offense classification code to OHE"
      ],
      "metadata": {
        "id": "DVn2o2HP9etE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_final[\"OFNS_DESC\"].value_counts()[:30]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d_k5hj5GgO_Z",
        "outputId": "66830529-33d3-4ae4-fd0e-3398a2063953"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PETIT LARCENY                      87091\n",
              "HARRASSMENT 2                      74733\n",
              "ASSAULT 3 & RELATED OFFENSES       48437\n",
              "CRIMINAL MISCHIEF & RELATED OF     44739\n",
              "GRAND LARCENY                      40874\n",
              "FELONY ASSAULT                     22809\n",
              "OFF. AGNST PUB ORD SENSBLTY &      17318\n",
              "MISCELLANEOUS PENAL LAW            14570\n",
              "ROBBERY                            13834\n",
              "BURGLARY                           12791\n",
              "GRAND LARCENY OF MOTOR VEHICLE     10417\n",
              "VEHICLE AND TRAFFIC LAWS            8382\n",
              "DANGEROUS DRUGS                     7744\n",
              "SEX CRIMES                          7053\n",
              "OFFENSES AGAINST PUBLIC ADMINI      5742\n",
              "DANGEROUS WEAPONS                   5436\n",
              "FORGERY                             4503\n",
              "THEFT-FRAUD                         3882\n",
              "FRAUDS                              2611\n",
              "OFFENSES INVOLVING FRAUD            2589\n",
              "INTOXICATED & IMPAIRED DRIVING      2578\n",
              "CRIMINAL TRESPASS                   1930\n",
              "RAPE                                1487\n",
              "UNAUTHORIZED USE OF A VEHICLE       1290\n",
              "POSSESSION OF STOLEN PROPERTY       1226\n",
              "OFFENSES AGAINST THE PERSON         1048\n",
              "ADMINISTRATIVE CODE                  774\n",
              "ARSON                                629\n",
              "OTHER OFFENSES RELATED TO THEF       625\n",
              "MURDER & NON-NEGL. MANSLAUGHTER      483\n",
              "Name: OFNS_DESC, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 343
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Only keeping those entries with classes of higher than 1000 incidence \n",
        "counts_ofns = df_final.groupby(\"OFNS_DESC\")[\"OFNS_DESC\"].transform(len)\n",
        "df_final['ofns_count'] = counts_ofns\n",
        "df_final = df_final.loc[df_final['ofns_count']>1000]\n",
        "df_final = df_final.drop(columns = ['ofns_count'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_cjOvrSHT5Tz",
        "outputId": "13514b1e-3eb0-4c1e-93e4-ffeed4f620a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# apply le on categorical feature columns\n",
        "df_final['OFNS_DESC'] = le.fit_transform(df_final['OFNS_DESC'])\n",
        "df_final.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        },
        "id": "8WjJI-bNmy7M",
        "outputId": "c65e8196-74d1-4c22-edeb-1d55647feb0f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    ADDR_PCT_CD  JURIS_DESC LOC_OF_OCCUR_DESC  OFNS_DESC PREM_TYP_DESC  \\\n",
              "3            52           6               NaN          5        STREET   \n",
              "6            47           6               NaN          5        STREET   \n",
              "19            9           6               NaN          7        STREET   \n",
              "37           43           6               NaN         21        STREET   \n",
              "59           46           6               NaN          4        STREET   \n",
              "\n",
              "   SUSP_AGE_GROUP SUSP_RACE SUSP_SEX VIC_AGE_GROUP VIC_RACE VIC_SEX  \\\n",
              "3             NaN       NaN      NaN       UNKNOWN  UNKNOWN       E   \n",
              "6             NaN       NaN      NaN       UNKNOWN  UNKNOWN       E   \n",
              "19            NaN       NaN      NaN       UNKNOWN  UNKNOWN       E   \n",
              "37        UNKNOWN   UNKNOWN        U         25-44    BLACK       M   \n",
              "59            NaN       NaN      NaN       UNKNOWN  UNKNOWN       E   \n",
              "\n",
              "     LAW_CAT_CD              FR_TIME duration  \n",
              "3        FELONY  2021-12-07 22:49:00      NaN  \n",
              "6        FELONY  2021-12-01 00:01:00      NaN  \n",
              "19       FELONY  2021-09-12 16:55:00      NaN  \n",
              "37       FELONY  2021-07-06 16:15:00      NaN  \n",
              "59  MISDEMEANOR  2021-02-13 15:15:00      NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f73ad73d-c81f-4c63-8860-7423bf2ed3b7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ADDR_PCT_CD</th>\n",
              "      <th>JURIS_DESC</th>\n",
              "      <th>LOC_OF_OCCUR_DESC</th>\n",
              "      <th>OFNS_DESC</th>\n",
              "      <th>PREM_TYP_DESC</th>\n",
              "      <th>SUSP_AGE_GROUP</th>\n",
              "      <th>SUSP_RACE</th>\n",
              "      <th>SUSP_SEX</th>\n",
              "      <th>VIC_AGE_GROUP</th>\n",
              "      <th>VIC_RACE</th>\n",
              "      <th>VIC_SEX</th>\n",
              "      <th>LAW_CAT_CD</th>\n",
              "      <th>FR_TIME</th>\n",
              "      <th>duration</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>52</td>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>5</td>\n",
              "      <td>STREET</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-12-07 22:49:00</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>47</td>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>5</td>\n",
              "      <td>STREET</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-12-01 00:01:00</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>9</td>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>7</td>\n",
              "      <td>STREET</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-09-12 16:55:00</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>43</td>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>21</td>\n",
              "      <td>STREET</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>U</td>\n",
              "      <td>25-44</td>\n",
              "      <td>BLACK</td>\n",
              "      <td>M</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-07-06 16:15:00</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>46</td>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4</td>\n",
              "      <td>STREET</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>MISDEMEANOR</td>\n",
              "      <td>2021-02-13 15:15:00</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f73ad73d-c81f-4c63-8860-7423bf2ed3b7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f73ad73d-c81f-4c63-8860-7423bf2ed3b7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f73ad73d-c81f-4c63-8860-7423bf2ed3b7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 345
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Location of occurance - One Hot Encoding "
      ],
      "metadata": {
        "id": "2ip5BTAIAn2c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_final['LOC_OF_OCCUR_DESC'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bNalm0WGA1su",
        "outputId": "295ebb4d-842a-4805-b1e6-946a9731a719"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "INSIDE         239224\n",
              "FRONT OF       115500\n",
              "OPPOSITE OF      9086\n",
              "REAR OF          7766\n",
              "Name: LOC_OF_OCCUR_DESC, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 346
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Use dummy variables to represent the crime descriptions \n",
        "dummy2 = pd.get_dummies(df_final['LOC_OF_OCCUR_DESC'])\n",
        "# Take a look\n",
        "dummy2.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "6GEpFoHSfIOw",
        "outputId": "4fb4a0c0-a38c-402f-c1b4-ba9b6126f56d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    FRONT OF  INSIDE  OPPOSITE OF  REAR OF\n",
              "3          0       0            0        0\n",
              "6          0       0            0        0\n",
              "19         0       0            0        0\n",
              "37         0       0            0        0\n",
              "59         0       0            0        0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6fe9ea24-552a-4811-b9ff-85fb35e051d4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>FRONT OF</th>\n",
              "      <th>INSIDE</th>\n",
              "      <th>OPPOSITE OF</th>\n",
              "      <th>REAR OF</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6fe9ea24-552a-4811-b9ff-85fb35e051d4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6fe9ea24-552a-4811-b9ff-85fb35e051d4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6fe9ea24-552a-4811-b9ff-85fb35e051d4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 347
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_final = pd.concat([df_final, dummy2], axis=1).drop('LOC_OF_OCCUR_DESC', axis=1)\n",
        "df_final.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461
        },
        "id": "RLJx6aNmnfwe",
        "outputId": "433af916-588c-4541-af11-42fb30322eb3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    ADDR_PCT_CD  JURIS_DESC  OFNS_DESC PREM_TYP_DESC SUSP_AGE_GROUP SUSP_RACE  \\\n",
              "3            52           6          5        STREET            NaN       NaN   \n",
              "6            47           6          5        STREET            NaN       NaN   \n",
              "19            9           6          7        STREET            NaN       NaN   \n",
              "37           43           6         21        STREET        UNKNOWN   UNKNOWN   \n",
              "59           46           6          4        STREET            NaN       NaN   \n",
              "\n",
              "   SUSP_SEX VIC_AGE_GROUP VIC_RACE VIC_SEX   LAW_CAT_CD              FR_TIME  \\\n",
              "3       NaN       UNKNOWN  UNKNOWN       E       FELONY  2021-12-07 22:49:00   \n",
              "6       NaN       UNKNOWN  UNKNOWN       E       FELONY  2021-12-01 00:01:00   \n",
              "19      NaN       UNKNOWN  UNKNOWN       E       FELONY  2021-09-12 16:55:00   \n",
              "37        U         25-44    BLACK       M       FELONY  2021-07-06 16:15:00   \n",
              "59      NaN       UNKNOWN  UNKNOWN       E  MISDEMEANOR  2021-02-13 15:15:00   \n",
              "\n",
              "   duration  FRONT OF  INSIDE  OPPOSITE OF  REAR OF  \n",
              "3       NaN         0       0            0        0  \n",
              "6       NaN         0       0            0        0  \n",
              "19      NaN         0       0            0        0  \n",
              "37      NaN         0       0            0        0  \n",
              "59      NaN         0       0            0        0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-43cb7aa5-df78-445c-8480-40bb46130788\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ADDR_PCT_CD</th>\n",
              "      <th>JURIS_DESC</th>\n",
              "      <th>OFNS_DESC</th>\n",
              "      <th>PREM_TYP_DESC</th>\n",
              "      <th>SUSP_AGE_GROUP</th>\n",
              "      <th>SUSP_RACE</th>\n",
              "      <th>SUSP_SEX</th>\n",
              "      <th>VIC_AGE_GROUP</th>\n",
              "      <th>VIC_RACE</th>\n",
              "      <th>VIC_SEX</th>\n",
              "      <th>LAW_CAT_CD</th>\n",
              "      <th>FR_TIME</th>\n",
              "      <th>duration</th>\n",
              "      <th>FRONT OF</th>\n",
              "      <th>INSIDE</th>\n",
              "      <th>OPPOSITE OF</th>\n",
              "      <th>REAR OF</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>52</td>\n",
              "      <td>6</td>\n",
              "      <td>5</td>\n",
              "      <td>STREET</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-12-07 22:49:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>47</td>\n",
              "      <td>6</td>\n",
              "      <td>5</td>\n",
              "      <td>STREET</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-12-01 00:01:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>9</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "      <td>STREET</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-09-12 16:55:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>43</td>\n",
              "      <td>6</td>\n",
              "      <td>21</td>\n",
              "      <td>STREET</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>U</td>\n",
              "      <td>25-44</td>\n",
              "      <td>BLACK</td>\n",
              "      <td>M</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-07-06 16:15:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>46</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>STREET</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>MISDEMEANOR</td>\n",
              "      <td>2021-02-13 15:15:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-43cb7aa5-df78-445c-8480-40bb46130788')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-43cb7aa5-df78-445c-8480-40bb46130788 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-43cb7aa5-df78-445c-8480-40bb46130788');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 348
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### property type"
      ],
      "metadata": {
        "id": "SS1PJAjeFBIR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_final[\"PREM_TYP_DESC\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cn79KuC0FE6Y",
        "outputId": "34f31162-ab64-4d51-ffad-99ecd07b34bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "STREET                        126659\n",
              "RESIDENCE - APT. HOUSE        107658\n",
              "RESIDENCE-HOUSE                46435\n",
              "RESIDENCE - PUBLIC HOUSING     31996\n",
              "CHAIN STORE                    20318\n",
              "                               ...  \n",
              "CEMETERY                          30\n",
              "PHOTO/COPY                        28\n",
              "LOAN COMPANY                      25\n",
              "DAYCARE FACILITY                  18\n",
              "TRAMWAY                            6\n",
              "Name: PREM_TYP_DESC, Length: 74, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 349
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Due to the large number of different property type, use number encoding instead \n",
        "df_final['PREM_TYP_DESC'] = le.fit_transform(df_final['PREM_TYP_DESC'])\n",
        "df_final.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461
        },
        "id": "-sD7BinKn24V",
        "outputId": "f6a22229-4440-4e66-a6fb-d641c81b18c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    ADDR_PCT_CD  JURIS_DESC  OFNS_DESC  PREM_TYP_DESC SUSP_AGE_GROUP  \\\n",
              "3            52           6          5             62            NaN   \n",
              "6            47           6          5             62            NaN   \n",
              "19            9           6          7             62            NaN   \n",
              "37           43           6         21             62        UNKNOWN   \n",
              "59           46           6          4             62            NaN   \n",
              "\n",
              "   SUSP_RACE SUSP_SEX VIC_AGE_GROUP VIC_RACE VIC_SEX   LAW_CAT_CD  \\\n",
              "3        NaN      NaN       UNKNOWN  UNKNOWN       E       FELONY   \n",
              "6        NaN      NaN       UNKNOWN  UNKNOWN       E       FELONY   \n",
              "19       NaN      NaN       UNKNOWN  UNKNOWN       E       FELONY   \n",
              "37   UNKNOWN        U         25-44    BLACK       M       FELONY   \n",
              "59       NaN      NaN       UNKNOWN  UNKNOWN       E  MISDEMEANOR   \n",
              "\n",
              "                FR_TIME duration  FRONT OF  INSIDE  OPPOSITE OF  REAR OF  \n",
              "3   2021-12-07 22:49:00      NaN         0       0            0        0  \n",
              "6   2021-12-01 00:01:00      NaN         0       0            0        0  \n",
              "19  2021-09-12 16:55:00      NaN         0       0            0        0  \n",
              "37  2021-07-06 16:15:00      NaN         0       0            0        0  \n",
              "59  2021-02-13 15:15:00      NaN         0       0            0        0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d51ed1f8-1f61-4a76-9f9b-2a37b232f149\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ADDR_PCT_CD</th>\n",
              "      <th>JURIS_DESC</th>\n",
              "      <th>OFNS_DESC</th>\n",
              "      <th>PREM_TYP_DESC</th>\n",
              "      <th>SUSP_AGE_GROUP</th>\n",
              "      <th>SUSP_RACE</th>\n",
              "      <th>SUSP_SEX</th>\n",
              "      <th>VIC_AGE_GROUP</th>\n",
              "      <th>VIC_RACE</th>\n",
              "      <th>VIC_SEX</th>\n",
              "      <th>LAW_CAT_CD</th>\n",
              "      <th>FR_TIME</th>\n",
              "      <th>duration</th>\n",
              "      <th>FRONT OF</th>\n",
              "      <th>INSIDE</th>\n",
              "      <th>OPPOSITE OF</th>\n",
              "      <th>REAR OF</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>52</td>\n",
              "      <td>6</td>\n",
              "      <td>5</td>\n",
              "      <td>62</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-12-07 22:49:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>47</td>\n",
              "      <td>6</td>\n",
              "      <td>5</td>\n",
              "      <td>62</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-12-01 00:01:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>9</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "      <td>62</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-09-12 16:55:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>43</td>\n",
              "      <td>6</td>\n",
              "      <td>21</td>\n",
              "      <td>62</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>U</td>\n",
              "      <td>25-44</td>\n",
              "      <td>BLACK</td>\n",
              "      <td>M</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-07-06 16:15:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>46</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>62</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "      <td>MISDEMEANOR</td>\n",
              "      <td>2021-02-13 15:15:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d51ed1f8-1f61-4a76-9f9b-2a37b232f149')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d51ed1f8-1f61-4a76-9f9b-2a37b232f149 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d51ed1f8-1f61-4a76-9f9b-2a37b232f149');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 350
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Suspect and Victim demographic info"
      ],
      "metadata": {
        "id": "ZeVsI47_GlPu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Assign the new group \n",
        "df_pca = df_final[['SUSP_AGE_GROUP','SUSP_RACE','SUSP_SEX', 'VIC_AGE_GROUP','VIC_RACE','VIC_SEX']]\n",
        "df_pca.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "YThgpF7TGsxE",
        "outputId": "f6accb0c-d0a4-43f2-c52d-ddb137d48563"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   SUSP_AGE_GROUP SUSP_RACE SUSP_SEX VIC_AGE_GROUP VIC_RACE VIC_SEX\n",
              "3             NaN       NaN      NaN       UNKNOWN  UNKNOWN       E\n",
              "6             NaN       NaN      NaN       UNKNOWN  UNKNOWN       E\n",
              "19            NaN       NaN      NaN       UNKNOWN  UNKNOWN       E\n",
              "37        UNKNOWN   UNKNOWN        U         25-44    BLACK       M\n",
              "59            NaN       NaN      NaN       UNKNOWN  UNKNOWN       E"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ab4d5ccd-f5b9-47ce-89a9-24bfd641c8ae\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>SUSP_AGE_GROUP</th>\n",
              "      <th>SUSP_RACE</th>\n",
              "      <th>SUSP_SEX</th>\n",
              "      <th>VIC_AGE_GROUP</th>\n",
              "      <th>VIC_RACE</th>\n",
              "      <th>VIC_SEX</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>U</td>\n",
              "      <td>25-44</td>\n",
              "      <td>BLACK</td>\n",
              "      <td>M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>UNKNOWN</td>\n",
              "      <td>E</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ab4d5ccd-f5b9-47ce-89a9-24bfd641c8ae')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ab4d5ccd-f5b9-47ce-89a9-24bfd641c8ae button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ab4d5ccd-f5b9-47ce-89a9-24bfd641c8ae');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 351
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Change UNKNOWN, E to np.Nan\n",
        "print(df_pca['SUSP_SEX'].unique())\n",
        "df_pca['SUSP_SEX'][df_pca['SUSP_SEX'] == 'U'] = np.nan\n",
        "\n",
        "print(df_pca['SUSP_AGE_GROUP'].unique())\n",
        "df_pca['SUSP_AGE_GROUP'][df_pca['SUSP_AGE_GROUP'] == 'UNKNOWN'] = np.nan\n",
        "\n",
        "print(df_pca['SUSP_RACE'].unique())\n",
        "df_pca['SUSP_RACE'][df_pca['SUSP_RACE'] == 'UNKNOWN'] = np.nan\n",
        "\n",
        "print(df_pca['VIC_AGE_GROUP'].unique())\n",
        "df_pca['VIC_AGE_GROUP'][df_pca['VIC_AGE_GROUP'] == 'UNKNOWN'] = np.nan\n",
        "\n",
        "print(df_pca['VIC_RACE'].unique())\n",
        "df_pca['VIC_RACE'][df_pca['VIC_RACE'] == 'UNKNOWN'] = np.nan\n",
        "\n",
        "print(df_pca['VIC_SEX'].unique())\n",
        "df_pca['VIC_SEX'][df_pca['VIC_SEX'] == 'E'] = np.nan\n",
        "df_pca['VIC_SEX'][df_pca['VIC_SEX'] == 'D'] = np.nan"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "haMjOX3m-EAx",
        "outputId": "337050b9-4156-4730-9d54-f6ed4552538b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nan 'U' 'M' 'F']\n",
            "[nan 'UNKNOWN' '25-44' '18-24' '45-64' '65+' '<18' '2021' '-969' '-33'\n",
            " '-60' '953' '-971' '942' '-946' '1032' '938' '-955' '940' '-69' '-941'\n",
            " '-975' '-947' '-973' '1017' '-966']\n",
            "[nan 'UNKNOWN' 'BLACK' 'BLACK HISPANIC' 'WHITE' 'WHITE HISPANIC'\n",
            " 'ASIAN / PACIFIC ISLANDER' 'AMERICAN INDIAN/ALASKAN NATIVE']\n",
            "['UNKNOWN' '25-44' '45-64' '18-24' '65+' '<18' '-4' '-943' '-62' '-3' '-1'\n",
            " '936' '-960' '-921' '-61' '-48' '970' '963' '945' '-935' '-51']\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/generic.py:8870: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  return self._update_inplace(result)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['UNKNOWN' 'BLACK' 'BLACK HISPANIC' 'WHITE' 'WHITE HISPANIC'\n",
            " 'ASIAN / PACIFIC ISLANDER' 'AMERICAN INDIAN/ALASKAN NATIVE' nan]\n",
            "['E' 'M' 'F' 'D']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Use LabelEncoder to convert labels to numbers\n",
        "original = df_pca\n",
        "mask = df_pca.isnull()\n",
        "df_pca = df_pca.astype(str).apply(LabelEncoder().fit_transform)\n",
        "df_pca.where(~mask, original)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "OIf7gwU59lA9",
        "outputId": "d46a0a38-8f2a-4de4-f0ec-70868a0eaf8b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       SUSP_AGE_GROUP SUSP_RACE SUSP_SEX VIC_AGE_GROUP VIC_RACE VIC_SEX\n",
              "3                 NaN       NaN      NaN           NaN      NaN     NaN\n",
              "6                 NaN       NaN      NaN           NaN      NaN     NaN\n",
              "19                NaN       NaN      NaN           NaN      NaN     NaN\n",
              "37                NaN       NaN      NaN            12        2       1\n",
              "59                NaN       NaN      NaN           NaN      NaN     NaN\n",
              "...               ...       ...      ...           ...      ...     ...\n",
              "449501            NaN       NaN      NaN            13        4       0\n",
              "449502            NaN       NaN      NaN           NaN      NaN     NaN\n",
              "449503             16         4        1           NaN      NaN     NaN\n",
              "449504             16         5        1            13        5       0\n",
              "449505            NaN       NaN      NaN            12        2       0\n",
              "\n",
              "[445114 rows x 6 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-601fe609-1486-4b78-a506-28699afd638b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>SUSP_AGE_GROUP</th>\n",
              "      <th>SUSP_RACE</th>\n",
              "      <th>SUSP_SEX</th>\n",
              "      <th>VIC_AGE_GROUP</th>\n",
              "      <th>VIC_RACE</th>\n",
              "      <th>VIC_SEX</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>12</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>449501</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>13</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>449502</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>449503</th>\n",
              "      <td>16</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>449504</th>\n",
              "      <td>16</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>13</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>449505</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>12</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>445114 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-601fe609-1486-4b78-a506-28699afd638b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-601fe609-1486-4b78-a506-28699afd638b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-601fe609-1486-4b78-a506-28699afd638b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 353
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fill Nan with median \n",
        "df_pca = df_pca.fillna(df_pca.median())\n",
        "df_pca.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Qak3sBf6_J7d",
        "outputId": "71db76de-951e-44e7-f98c-edcd807b3716"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    SUSP_AGE_GROUP  SUSP_RACE  SUSP_SEX  VIC_AGE_GROUP  VIC_RACE  VIC_SEX\n",
              "3               24          6         2             20         6        2\n",
              "6               24          6         2             20         6        2\n",
              "19              24          6         2             20         6        2\n",
              "37              24          6         2             12         2        1\n",
              "59              24          6         2             20         6        2"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-be2c3078-a823-4066-a2f6-f53f934c1196\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>SUSP_AGE_GROUP</th>\n",
              "      <th>SUSP_RACE</th>\n",
              "      <th>SUSP_SEX</th>\n",
              "      <th>VIC_AGE_GROUP</th>\n",
              "      <th>VIC_RACE</th>\n",
              "      <th>VIC_SEX</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>20</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>20</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>20</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>12</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>20</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-be2c3078-a823-4066-a2f6-f53f934c1196')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-be2c3078-a823-4066-a2f6-f53f934c1196 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-be2c3078-a823-4066-a2f6-f53f934c1196');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 354
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_final = df_final.drop(columns = ['SUSP_AGE_GROUP','SUSP_RACE','SUSP_SEX', 'VIC_AGE_GROUP','VIC_RACE','VIC_SEX'])\n",
        "df_final = pd.concat([df_final, df_pca], axis = 1)\n",
        "df_final.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461
        },
        "id": "L5qfeX987IBF",
        "outputId": "24037e85-daa8-4fb2-f47c-d4a4a0c5833f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    ADDR_PCT_CD  JURIS_DESC  OFNS_DESC  PREM_TYP_DESC   LAW_CAT_CD  \\\n",
              "3            52           6          5             62       FELONY   \n",
              "6            47           6          5             62       FELONY   \n",
              "19            9           6          7             62       FELONY   \n",
              "37           43           6         21             62       FELONY   \n",
              "59           46           6          4             62  MISDEMEANOR   \n",
              "\n",
              "                FR_TIME duration  FRONT OF  INSIDE  OPPOSITE OF  REAR OF  \\\n",
              "3   2021-12-07 22:49:00      NaN         0       0            0        0   \n",
              "6   2021-12-01 00:01:00      NaN         0       0            0        0   \n",
              "19  2021-09-12 16:55:00      NaN         0       0            0        0   \n",
              "37  2021-07-06 16:15:00      NaN         0       0            0        0   \n",
              "59  2021-02-13 15:15:00      NaN         0       0            0        0   \n",
              "\n",
              "    SUSP_AGE_GROUP  SUSP_RACE  SUSP_SEX  VIC_AGE_GROUP  VIC_RACE  VIC_SEX  \n",
              "3               24          6         2             20         6        2  \n",
              "6               24          6         2             20         6        2  \n",
              "19              24          6         2             20         6        2  \n",
              "37              24          6         2             12         2        1  \n",
              "59              24          6         2             20         6        2  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6aecb779-ca10-4e8a-8e2b-0aa649a53555\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ADDR_PCT_CD</th>\n",
              "      <th>JURIS_DESC</th>\n",
              "      <th>OFNS_DESC</th>\n",
              "      <th>PREM_TYP_DESC</th>\n",
              "      <th>LAW_CAT_CD</th>\n",
              "      <th>FR_TIME</th>\n",
              "      <th>duration</th>\n",
              "      <th>FRONT OF</th>\n",
              "      <th>INSIDE</th>\n",
              "      <th>OPPOSITE OF</th>\n",
              "      <th>REAR OF</th>\n",
              "      <th>SUSP_AGE_GROUP</th>\n",
              "      <th>SUSP_RACE</th>\n",
              "      <th>SUSP_SEX</th>\n",
              "      <th>VIC_AGE_GROUP</th>\n",
              "      <th>VIC_RACE</th>\n",
              "      <th>VIC_SEX</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>52</td>\n",
              "      <td>6</td>\n",
              "      <td>5</td>\n",
              "      <td>62</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-12-07 22:49:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>20</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>47</td>\n",
              "      <td>6</td>\n",
              "      <td>5</td>\n",
              "      <td>62</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-12-01 00:01:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>20</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>9</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "      <td>62</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-09-12 16:55:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>20</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>43</td>\n",
              "      <td>6</td>\n",
              "      <td>21</td>\n",
              "      <td>62</td>\n",
              "      <td>FELONY</td>\n",
              "      <td>2021-07-06 16:15:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>12</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>46</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>62</td>\n",
              "      <td>MISDEMEANOR</td>\n",
              "      <td>2021-02-13 15:15:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>20</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6aecb779-ca10-4e8a-8e2b-0aa649a53555')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6aecb779-ca10-4e8a-8e2b-0aa649a53555 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6aecb779-ca10-4e8a-8e2b-0aa649a53555');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 355
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Complaint time ('FR_TIME')"
      ],
      "metadata": {
        "id": "ji9dngqR_u5m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_final['FR_TIME'] = (pd.to_datetime(df_final['FR_TIME']).dt.hour)"
      ],
      "metadata": {
        "id": "uSvEGhg-_y8G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df_final['FR_TIME'].unique())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UzkpzsUIAoI0",
        "outputId": "7f11a5c8-94fe-4bc6-f0c8-c72a91a03e3b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[22  0 16 15  3 19  4 20 10 13 17  2  9  7 12  1  8 14 18 23 11 21  6  5]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_final['FR_TIME'] = df_final['FR_TIME'].fillna(df_final['FR_TIME'].median())"
      ],
      "metadata": {
        "id": "FlqUvQIZGlEW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Duration ('duration')"
      ],
      "metadata": {
        "id": "8W4jZIcWE_Ba"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert into hours \n",
        "df_final['duration']= pd.to_timedelta(df_final['duration'])/ np.timedelta64(1, 'h')"
      ],
      "metadata": {
        "id": "mnnnerHsFQF3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_final['duration'] = df_final['duration'].fillna(df_final['duration'].median())"
      ],
      "metadata": {
        "id": "Ip8zcqHxFEFo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Convert outcome to numbers "
      ],
      "metadata": {
        "id": "odhC-2PuqSjD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_final['LAW_CAT_CD'] = le.fit_transform(df_final['LAW_CAT_CD'])"
      ],
      "metadata": {
        "id": "t6dqpnITqVyc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Standarize Data\n"
      ],
      "metadata": {
        "id": "IHwu8ouPqHvW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_final['ADDR_PCT_CD'] =( df_final['ADDR_PCT_CD'] - df_final['ADDR_PCT_CD'].mean() ) / df_final['ADDR_PCT_CD'].std()\n",
        "df_final['PREM_TYP_DESC'] =( df_final['PREM_TYP_DESC'] - df_final['PREM_TYP_DESC'].mean() ) / df_final['PREM_TYP_DESC'].std()"
      ],
      "metadata": {
        "id": "1XbEte4hqz8E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Draw feature correlation"
      ],
      "metadata": {
        "id": "QAlb_R3A2PDH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(df_final.corr()[\"LAW_CAT_CD\"][:])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i6tl0TUC8t7e",
        "outputId": "fac362d7-f42a-4c62-b1c0-3ee01f7a7de3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ADDR_PCT_CD       0.028336\n",
            "JURIS_DESC       -0.032275\n",
            "OFNS_DESC         0.105847\n",
            "PREM_TYP_DESC    -0.027790\n",
            "LAW_CAT_CD        1.000000\n",
            "FR_TIME           0.037094\n",
            "duration         -0.012858\n",
            "FRONT OF         -0.031059\n",
            "INSIDE            0.084814\n",
            "OPPOSITE OF      -0.050234\n",
            "REAR OF          -0.024590\n",
            "SUSP_AGE_GROUP   -0.108653\n",
            "SUSP_RACE        -0.145324\n",
            "SUSP_SEX         -0.231874\n",
            "VIC_AGE_GROUP    -0.126339\n",
            "VIC_RACE         -0.107298\n",
            "VIC_SEX          -0.185111\n",
            "Name: LAW_CAT_CD, dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fill Values with Nan with 0 \n",
        "df_final = df_final.fillna(0)"
      ],
      "metadata": {
        "id": "p3yhWJrrzJKX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_final.to_csv('crime_features.csv')\n",
        "!cp data.csv \"/content/drive/MyDrive/Crimes\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dSbAEkGCz3Y7",
        "outputId": "bf93e072-dd18-48e8-909c-fa8c648e9e99"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cp: cannot stat 'data.csv': No such file or directory\n"
          ]
        }
      ]
    }
  ]
}